[repo.or.cz] iwhd.git branch master updated: v0.0-174-g5f3562c
by Jim Meyering
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, master has been updated
via 5f3562c3efe50314a708094b19641ba282b35b6a (commit)
from fc5ba187804f575cd4272d77ec54d0814c9fa6a9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/5f3562c3efe50314a708094b19641ba282b35b6a
commit 5f3562c3efe50314a708094b19641ba282b35b6a
Author: Jim Meyering <meyering(a)redhat.com>
Date: Mon Oct 18 20:23:29 2010 +0200
init.sh: don't comment out cleanup-handling rm -rf
diff --git a/t/init.sh b/t/init.sh
index b93b051..a57de77 100644
--- a/t/init.sh
+++ b/t/init.sh
@@ -225,7 +225,7 @@ remove_tmp_()
cd "$initial_cwd_" || cd / || cd /tmp
chmod -R u+rwx "$test_dir_"
# If removal fails and exit status was to be 0, then change it to 1.
- #rm -rf "$test_dir_" || { test $__st = 0 && __st=1; }
+ rm -rf "$test_dir_" || { test $__st = 0 && __st=1; }
exit $__st
}
-----------------------------------------------------------------------
Summary of changes:
t/init.sh | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months
[repo.or.cz] iwhd.git branch master updated: v0.0-172-gcfa0553
by zaitcev
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, master has been updated
via cfa0553caaeea30e5b542ca9f91b69244d07d709 (commit)
via 57bfee81c42c26eac3299404738bdd06bed5f4f8 (commit)
from d82854a5c8cbc35897fe931376cff1e7f446c953 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/cfa0553caaeea30e5b542ca9f91b69244d07d709
commit cfa0553caaeea30e5b542ca9f91b69244d07d709
Author: Pete Zaitcev <zaitcev(a)yahoo.com>
Date: Sat Oct 16 17:17:44 2010 -0600
add static to parse_config_inner
This suppresses a warning about missing prototype.
diff --git a/setup.c b/setup.c
index 27146ee..6a76506 100644
--- a/setup.c
+++ b/setup.c
@@ -272,7 +272,7 @@ convert_provider (int i, provider_t *out)
return 1;
}
-const char *
+static const char *
parse_config_inner (void)
{
unsigned int nservers;
http://repo.or.cz/w/iwhd.git/commit/57bfee81c42c26eac3299404738bdd06bed5f4f8
commit 57bfee81c42c26eac3299404738bdd06bed5f4f8
Author: Pete Zaitcev <zaitcev(a)yahoo.com>
Date: Sat Oct 16 16:42:12 2010 -0600
switch to atexit(auto_stop)
The main impetus for this change is that query.peg has an error() that
can exit. However, we cannot add an explicit auto_stop in front of it,
because the parser is also linked into tests, which do not have autostart
code and thus throw a link error.
Secondarily, since atexit() is only called when the start of Mongo is
assured, we do not really need checking for PID being valid or any other
scheme to disable auto_stop in the child, etc. We're not changing this
for now, but maybe later.
diff --git a/auto.c b/auto.c
index 01521d6..8390747 100644
--- a/auto.c
+++ b/auto.c
@@ -283,6 +283,12 @@ auto_set_sig (void)
return 0;
}
+static void
+auto_stop (void)
+{
+ auto_kill_mongod(SIGTERM);
+}
+
int
auto_start (int dbport)
{
@@ -326,13 +332,12 @@ auto_start (int dbport)
auto_kill_mongod(SIGTERM);
return -1;
}
+ if (atexit(auto_stop) != 0) {
+ error (0, 0, "atexit failed for auto_stop");
+ auto_kill_mongod(SIGTERM);
+ return -1;
+ }
DPRINTF("mongod listens on port %un", dbport);
return 0;
}
-
-void
-auto_stop (void)
-{
- auto_kill_mongod(SIGTERM);
-}
diff --git a/iwh.h b/iwh.h
index 89d38b2..082356b 100644
--- a/iwh.h
+++ b/iwh.h
@@ -84,3 +84,5 @@ GLOBAL(const char *, me, "here");
#define AUTO_BIN_MONGOD "/usr/bin/mongod"
#define AUTO_MONGOD_LOG "_mongod.log"
#define AUTO_MONGOD_PORT 27018
+
+int auto_start (int dbport);
diff --git a/rest.c b/rest.c
index 91b3552..8f5f148 100644
--- a/rest.c
+++ b/rest.c
@@ -2028,11 +2028,9 @@ args_done:
MHD_OPTION_END);
if (!the_daemon) {
fprintf(stderr,"Could not create daemon.n");
- auto_stop();
return !0;
}
sem_wait(&the_sem);
- auto_stop();
return 0;
}
diff --git a/setup.h b/setup.h
index 0056dbc..8c05040 100644
--- a/setup.h
+++ b/setup.h
@@ -45,8 +45,4 @@ void init_prov_iter (GHashTableIter *iter);
const char *auto_config (void);
-/* These are actually in auto.c, but it doesn't have its own header file. */
-int auto_start (int dbport);
-void auto_stop (void);
-
#endif
-----------------------------------------------------------------------
Summary of changes:
auto.c | 17 +++++++++++------
iwh.h | 2 ++
rest.c | 2 --
setup.c | 2 +-
setup.h | 4 ----
5 files changed, 14 insertions(+), 13 deletions(-)
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months
[repo.or.cz] iwhd.git branch master updated: v0.0-170-gd82854a
by Jim Meyering
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, master has been updated
via d82854a5c8cbc35897fe931376cff1e7f446c953 (commit)
via c947ab61da94a8f85844bc85f061105aa1c6e1b4 (commit)
from 7fb8264a528a19fd999dc4626121c2d6427d437d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/d82854a5c8cbc35897fe931376cff1e7f446c953
commit d82854a5c8cbc35897fe931376cff1e7f446c953
Author: Jim Meyering <meyering(a)redhat.com>
Date: Thu Oct 7 14:02:08 2010 +0200
avoid warning about signed/unsigned comparison
diff --git a/meta.cpp b/meta.cpp
index 651c9aa..de9dfa2 100644
--- a/meta.cpp
+++ b/meta.cpp
@@ -62,7 +62,7 @@ pthread_mutex_t client_lock = PTHREAD_MUTEX_INITIALIZER;
void
dbl_to_str (double *foo, char *optr)
{
- int i;
+ unsigned int i;
unsigned char *iptr = (unsigned char *)foo;
for (i = 0; i < sizeof(*foo); ++i) {
http://repo.or.cz/w/iwhd.git/commit/c947ab61da94a8f85844bc85f061105aa1c6e1b4
commit c947ab61da94a8f85844bc85f061105aa1c6e1b4
Author: Jim Meyering <meyering(a)redhat.com>
Date: Thu Oct 7 14:01:29 2010 +0200
build: avoid warning about unused parameter
diff --git a/meta.cpp b/meta.cpp
index 13abfb2..651c9aa 100644
--- a/meta.cpp
+++ b/meta.cpp
@@ -580,6 +580,8 @@ RepoMeta::GetSize (const char *bucket, const char *key)
BSONObj bo;
const char * data;
+ (void)data;
+
q = QUERY("bucket"<<bucket<<"key"<<key);
curs = GetCursor(q);
-----------------------------------------------------------------------
Summary of changes:
meta.cpp | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months
[repo.or.cz] iwhd.git branch bison-flex updated: v0.0-182-gc8b655f
by Jim Meyering
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, bison-flex has been updated
discards 1c66543ffc83ebf8fd6a99d6c576414746fb8d03 (commit)
discards 2b217ff620298dc2584492a9729ddfc5b1d64ed7 (commit)
discards c2f21c7193a71e0135e01878d25da72d14aee581 (commit)
discards f8a6fed825045cecfad10e6c8d4c85fb93bd2c04 (commit)
discards c7577402deb00b434907a27fb2d869d53b16cf32 (commit)
discards b4d1b13de7d3026e6fc87bd3de872594c5697c88 (commit)
discards 156d035b4cf119d51451cb90cee703812ff37b5e (commit)
discards f9cd1b61a9fe936777cb4011617c7a43e59d01d8 (commit)
discards 7e6b3e5ef9e1a4cada4af4653d95cd6676027e2f (commit)
discards 67f218604c5c5438280a46bf7deb8fb492f2719d (commit)
discards 125c1fc30521d55a51fc0a74c782bdb852e9b3b8 (commit)
via c8b655f76467dadf55b7b8574dbbb5093cab7150 (commit)
via 13d54abce0015041bd2aff388596235d52f62c7f (commit)
via 59b33c3181ccb0adffb1a880a697d8a4a04bd15d (commit)
via d5e631d306b61c6ad9c07124bb5891bf0cf71d43 (commit)
via bc07e03f3a325f0933545636c422ec9b8173b64d (commit)
via 3a848ee0a6421aa5f72bf9beca118b1a1e2cab67 (commit)
via 376e0b6fa1c00121243c1da2b28924214043885f (commit)
via b9670e25d481159101eaf50465669dce02a8e1ff (commit)
via 02b46dcd6b8cf12b306eca5dd7a036964e99cd98 (commit)
via e29ed3162dd00ccdbfb7eeeedf892a3e850c963e (commit)
via 92ce84f08d19187edef1807c98101dc61028d662 (commit)
via 3ce94781571362adb7f66ea6efe7dc9564d93d8d (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (1c66543ffc83ebf8fd6a99d6c576414746fb8d03)
N -- N -- N (c8b655f76467dadf55b7b8574dbbb5093cab7150)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/c8b655f76467dadf55b7b8574dbbb5093cab7150
commit c8b655f76467dadf55b7b8574dbbb5093cab7150
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Fri Oct 15 12:01:20 2010 -0400
Add query.h comments back in.
diff --git a/query.h b/query.h
index b726632..253e19b 100644
--- a/query.h
+++ b/query.h
@@ -21,14 +21,26 @@ extern "C" {
#include "iwhd-qparser.h"
+/*
+ * Comparisons are all the same type to the parser, but when it comes to
+ * evaluation we need to know the difference so we use these subtypes.
+ */
typedef enum {
C_LESSTHAN, C_LESSOREQ,
C_EQUAL, C_DIFFERENT,
C_GREATEROREQ, C_GREATERTHAN
} comp_t;
+/* The actual values are generated by the parser. */
typedef enum yytokentype type_t;
+/*
+ * Universal AST object. T_NUMBER uses as_num, and some day T_DATE might as
+ * well. Several types (T_STRING, T_ID, T_*FIELD) all use as_str. The rest
+ * use as_tree, but there's a caveat. In most cases as_tree.right really is
+ * a value_t, but for T_LINK it's a bare string.
+ * TBD: use a separate as_link union member for T_LINK.
+ */
typedef struct _value {
type_t type;
union {
@@ -43,12 +55,27 @@ typedef struct _value {
const char *resolved; /* saved result for T_OFIELD/T_SFIELD/T_LINK */
} value_t;
+/*
+ * In a higher-level language, this would be a method pointer. It's just
+ * a pointer to a function plus a little piece of the caller's context (in
+ * the replication-policy case it's the current bucket and key) so that we
+ * can do concurrent evaluations with separate contexts.
+ */
typedef struct {
const char *(*func) (void *, const char *);
void *ctx;
} getter_t;
#define CALL_GETTER(g,x) g->func(g->ctx,x)
+/*
+ * In the normal case a caller would invoke parse once, eval multiple times,
+ * and free_value once. print_value is just for debugging/testing.
+ * TBD: make parse reentrant (eval already is, free_value doesn't need to be.
+ * Unfortunately, a quick scan of generated code and information on the web
+ * seems to indicate that even a "reentrant" bison parser only encapsulates
+ * user state and still relies quite a bit on internal globals. That might
+ * mean that we just have to put a lock around it instead.
+ */
int eval (const value_t *expr,
const getter_t *oget, const getter_t *sget);
void free_value (value_t *);
http://repo.or.cz/w/iwhd.git/commit/13d54abce0015041bd2aff388596235d52f62c7f
commit 13d54abce0015041bd2aff388596235d52f62c7f
Author: Jim Meyering <meyering(a)redhat.com>
Date: Fri Oct 15 15:07:19 2010 +0200
build: ensure we use only Flex
diff --git a/qlexer.l b/qlexer.l
index c4d3a63..41232a3 100644
--- a/qlexer.l
+++ b/qlexer.l
@@ -3,6 +3,10 @@
#include <config.h>
#include <stdio.h>
+#ifndef FLEX_SCANNER
+# error This scanner must be made using flex, not lex.
+#endif
+
#include "iwhd-qparser.h"
#undef YY_DECL
#define YY_DECL extern int yylex (YYSTYPE *lvalp)
http://repo.or.cz/w/iwhd.git/commit/59b33c3181ccb0adffb1a880a697d8a4a04bd15d
commit 59b33c3181ccb0adffb1a880a697d8a4a04bd15d
Author: Jim Meyering <meyering(a)redhat.com>
Date: Fri Oct 15 14:24:21 2010 +0200
qlexer.l: simplify
* qlexer.l: Include <config.h> first.
Use a few %options.
Use <<EOF>> rather than yywrap.
diff --git a/qlexer.l b/qlexer.l
index 7ae9e87..c4d3a63 100644
--- a/qlexer.l
+++ b/qlexer.l
@@ -1,9 +1,13 @@
+%option warn nounput noinput noyywrap
%{
+#include <config.h>
#include <stdio.h>
#include "iwhd-qparser.h"
#undef YY_DECL
#define YY_DECL extern int yylex (YYSTYPE *lvalp)
+
+static int at_eof = 0;
%}
%%
@@ -19,15 +23,10 @@
[ tnr]+ { return T_SPACE; }
. { return T_INVALID; }
-%%
+<<EOF>> { at_eof = 1; yyterminate (); }
-static int at_eof = 0;
-int
-yywrap (void)
-{
- return ++at_eof;
-}
+%%
#if defined(UNIT_TEST)
static void
http://repo.or.cz/w/iwhd.git/commit/d5e631d306b61c6ad9c07124bb5891bf0cf71d43
commit d5e631d306b61c6ad9c07124bb5891bf0cf71d43
Author: Jim Meyering <meyering(a)redhat.com>
Date: Fri Oct 15 13:05:44 2010 +0200
avoid undefined-yydecl warning
diff --git a/qlexer.l b/qlexer.l
index bd2e196..7ae9e87 100644
--- a/qlexer.l
+++ b/qlexer.l
@@ -2,6 +2,7 @@
#include <stdio.h>
#include "iwhd-qparser.h"
+#undef YY_DECL
#define YY_DECL extern int yylex (YYSTYPE *lvalp)
%}
diff --git a/qparser.y b/qparser.y
index c52c2ba..bb6e760 100644
--- a/qparser.y
+++ b/qparser.y
@@ -12,6 +12,9 @@
#include "query.h"
#include "iwhd-qparser.h"
+#define YY_DECL int yylex(YYSTYPE *);
+YY_DECL
+
static void
xalloc_die (void)
{
http://repo.or.cz/w/iwhd.git/commit/bc07e03f3a325f0933545636c422ec9b8173b64d
commit bc07e03f3a325f0933545636c422ec9b8173b64d
Author: Jim Meyering <meyering(a)redhat.com>
Date: Thu Oct 14 18:00:11 2010 +0200
use literals in bison
diff --git a/qlexer.l b/qlexer.l
index 33c86a6..bd2e196 100644
--- a/qlexer.l
+++ b/qlexer.l
@@ -11,17 +11,10 @@
~[^~]*~ { return T_DATE; }
[0-9]+ { return T_NUMBER; }
[a-z_]+ { return T_ID; }
-$ { return T_DOLLAR; }
-# { return T_WAFFLE; }
-( { return T_LPAREN; }
-) { return T_RPAREN; }
-< { return T_LESS; }
-> { return T_GREATER; }
-= { return T_EQUAL; }
+[=$#()<>.] { return yytext[0]; }
! { return T_NOT; }
& { return T_AND; }
| { return T_OR; }
-. { return T_DOT; }
[ tnr]+ { return T_SPACE; }
. { return T_INVALID; }
diff --git a/qparser.y b/qparser.y
index 1f3a389..c52c2ba 100644
--- a/qparser.y
+++ b/qparser.y
@@ -163,9 +163,6 @@ yyerror (value_t **result, const char *msg)
%parse-param { value_t **result }
%token T_STRING T_DATE T_NUMBER T_ID
-%token T_DOLLAR T_WAFFLE T_DOT
-%token T_LPAREN T_RPAREN
-%token T_LESS T_GREATER T_EQUAL
%token T_NOT T_AND T_OR
%token T_SPACE T_INVALID
%token T_OFIELD T_SFIELD T_COMP T_LINK
@@ -224,27 +221,27 @@ comp_expr:
// printf("promoting atom to comp_exprn");
$$ = $1;
}|
- atom T_LESS atom {
+ atom '<' atom {
// printf("found LESS THAN expressionn");
$$ = make_comp(C_LESSTHAN,$1,$3);
}|
- atom T_LESS T_EQUAL atom {
+ atom '<' '=' atom {
// printf("found LESS OR EQUAL expressionn");
$$ = make_comp(C_LESSOREQ,$1,$4);
}|
- atom T_EQUAL T_EQUAL atom {
+ atom '=' '=' atom {
// printf("found EQUAL expressionn");
$$ = make_comp(C_EQUAL,$1,$4);
}|
- atom T_NOT T_EQUAL atom {
+ atom T_NOT '=' atom {
// printf("found NOT EQUAL expressionn");
$$ = make_comp(C_DIFFERENT,$1,$4);
}|
- atom T_GREATER T_EQUAL atom {
+ atom '>' '=' atom {
// printf("found GREATER OR EQUAL expressionn");
$$ = make_comp(C_GREATEROREQ,$1,$4);
}|
- atom T_GREATER atom {
+ atom '>' atom {
// printf("found GREATER THAN expressionn");
$$ = make_comp(C_GREATERTHAN,$1,$3);
}|
@@ -278,17 +275,17 @@ link_field:
// printf("promoting field to link_fieldn");
$$ = $1;
}|
- link_field T_DOT T_ID {
+ link_field '.' T_ID {
// printf("found LINK FIELDn");
$$ = make_link($1,yytext);
};
field:
- T_DOLLAR T_ID {
+ '$' T_ID {
// printf("found DOLLAR FIELDn");
$$ = make_string(yytext,T_OFIELD);
}|
- T_WAFFLE T_ID {
+ '#' T_ID {
// printf("found WAFFLE FIELDn");
$$ = make_string(yytext,T_SFIELD);
};
@@ -312,7 +309,7 @@ literal:
};
paren_expr:
- T_LPAREN bbool_expr T_RPAREN {
+ '(' bbool_expr ')' {
// printf("found PAREN expressionn");
$$ = $2;
};
http://repo.or.cz/w/iwhd.git/commit/3a848ee0a6421aa5f72bf9beca118b1a1e2cab67
commit 3a848ee0a6421aa5f72bf9beca118b1a1e2cab67
Author: Jim Meyering <meyering(a)redhat.com>
Date: Thu Oct 14 17:01:43 2010 +0200
static and const
diff --git a/qlexer.l b/qlexer.l
index c90bf15..33c86a6 100644
--- a/qlexer.l
+++ b/qlexer.l
@@ -27,7 +27,7 @@
%%
-int at_eof = 0;
+static int at_eof = 0;
int
yywrap (void)
@@ -37,7 +37,7 @@ yywrap (void)
#if defined(UNIT_TEST)
static void
-yyerror (char *msg)
+yyerror (const char *msg)
{
printf("%s: %sn",__func__,msg);
}
diff --git a/qparser.y b/qparser.y
index 82f9774..1f3a389 100644
--- a/qparser.y
+++ b/qparser.y
@@ -1,3 +1,6 @@
+%define api.pure
+%error-verbose
+
%{
#include <config.h>
#include <error.h>
@@ -59,7 +62,7 @@ xstrdup (char const *string)
}
/* TBD: use separate function to parse dates differently */
-value_t *
+static value_t *
make_number (const char *text)
{
value_t *tmp = malloc(sizeof(*tmp));
@@ -73,7 +76,7 @@ make_number (const char *text)
return tmp;
}
-value_t *
+static value_t *
make_string (const char *text, type_t t)
{
value_t *tmp = malloc(sizeof(*tmp));
@@ -87,23 +90,23 @@ make_string (const char *text, type_t t)
return tmp;
}
-value_t *
-make_tree (type_t t, value_t *left, value_t *right)
+static value_t *
+make_tree (type_t t, const value_t *left, const value_t *right)
{
value_t *tmp = malloc(sizeof(*tmp));
if (tmp) {
tmp->type = t;
- tmp->as_tree.left = left;
- tmp->as_tree.right = right;
+ tmp->as_tree.left = (value_t *) left;
+ tmp->as_tree.right = (value_t *) right;
tmp->resolved = NULL;
}
return tmp;
}
-value_t *
-make_comp (comp_t c, value_t *left, value_t *right)
+static value_t *
+make_comp (comp_t c, const value_t *left, const value_t *right)
{
value_t *tmp = make_tree(T_COMP,left,right);
@@ -114,8 +117,8 @@ make_comp (comp_t c, value_t *left, value_t *right)
return tmp;
}
-value_t *
-make_link (value_t *left, char *right)
+static value_t *
+make_link (value_t *left, const char *right)
{
char *copy;
@@ -157,7 +160,6 @@ yyerror (value_t **result, const char *msg)
%}
-%define api.pure
%parse-param { value_t **result }
%token T_STRING T_DATE T_NUMBER T_ID
@@ -318,7 +320,7 @@ paren_expr:
%%
#if defined PARSER_UNIT_TEST
-struct { char *name; char *value; } hacked_obj_fields[] = {
+static const struct { char *name; char *value; } hacked_obj_fields[] = {
/* Fake object fields for generic unit testing. */
{ "a", "2" }, { "b", "7" }, { "c", "11" },
/* This one's here to test links (e.g. $template.owner.name). */
@@ -327,7 +329,7 @@ struct { char *name; char *value; } hacked_obj_fields[] = {
};
/* Fake out the eval code for unit testing. */
-const char *
+static const char *
unit_oget_func (void * notused, const char *text)
{
int i;
@@ -340,18 +342,18 @@ unit_oget_func (void * notused, const char *text)
return NULL;
}
-getter_t unit_oget = { unit_oget_func };
+static const getter_t unit_oget = { unit_oget_func };
/*
* Same as above, but the site-field stuff is so similar to the object-field
* stuff that it's not worth exercising too much separately.
*/
-const char *
+static const char *
unit_sget_func (void * notused, const char *text)
{
return "never";
}
-getter_t unit_sget = { unit_sget_func };
+static const getter_t unit_sget = { unit_sget_func };
/* Fake links from an object/key tuple to an object/key string. */
struct { char *obj; char *key; char *value; } hacked_links[] = {
@@ -360,7 +362,7 @@ struct { char *obj; char *key; char *value; } hacked_links[] = {
{ NULL }
};
-char *
+static char *
follow_link (const char *object, const char *key)
{
unsigned int i;
@@ -381,7 +383,7 @@ follow_link (const char *object, const char *key)
extern char *follow_link (const char *object, const char *key);
#endif
-void
+static void
_print_value (const value_t *v, int level)
{
if (!v) {
@@ -505,7 +507,7 @@ parse (const char *text)
* and booleans.
*/
static const char *
-string_value (value_t *v, getter_t *oget, getter_t *sget)
+string_value (value_t *v, const getter_t *oget, const getter_t *sget)
{
const char *left;
@@ -537,7 +539,7 @@ string_value (value_t *v, getter_t *oget, getter_t *sget)
* Check whether a string looks like a simple decimal number. There's
* probably a library function for this somewhere.
*/
-int
+static int
is_ok_number (const char *a_str)
{
const char *p;
@@ -563,9 +565,9 @@ is_ok_number (const char *a_str)
* but the code is actually structured a different way to allow re-use of
* common operator-specific code at the end for both cases.
*/
-int
+static int
compare (value_t *left, comp_t op, value_t *right,
- getter_t *oget, getter_t *sget)
+ const getter_t *oget, const getter_t *sget)
{
const char *lstr;
const char *rstr;
@@ -652,7 +654,7 @@ compare (value_t *left, comp_t op, value_t *right,
*/
int
-eval (const value_t *v, getter_t *oget, getter_t *sget)
+eval (const value_t *v, const getter_t *oget, const getter_t *sget)
{
int res;
const char *str;
diff --git a/query.h b/query.h
index f7823e2..b726632 100644
--- a/query.h
+++ b/query.h
@@ -49,7 +49,8 @@ typedef struct {
} getter_t;
#define CALL_GETTER(g,x) g->func(g->ctx,x)
-int eval (const value_t *expr, getter_t *oget, getter_t *sget);
+int eval (const value_t *expr,
+ const getter_t *oget, const getter_t *sget);
void free_value (value_t *);
void print_value (const value_t *);
http://repo.or.cz/w/iwhd.git/commit/376e0b6fa1c00121243c1da2b28924214043885f
commit 376e0b6fa1c00121243c1da2b28924214043885f
Author: Jim Meyering <meyering(a)redhat.com>
Date: Thu Oct 14 16:43:06 2010 +0200
cast away "const" on free argument
diff --git a/qparser.y b/qparser.y
index c23c03e..82f9774 100644
--- a/qparser.y
+++ b/qparser.y
@@ -459,7 +459,7 @@ free_value (value_t *v)
printf("freeing resolved string "%s" (%p)n",
v->resolved, v->resolved);
}
- free(v->resolved);
+ free((void *)v->resolved);
switch (v->type) {
case T_STRING:
http://repo.or.cz/w/iwhd.git/commit/b9670e25d481159101eaf50465669dce02a8e1ff
commit b9670e25d481159101eaf50465669dce02a8e1ff
Author: Jim Meyering <meyering(a)redhat.com>
Date: Thu Oct 14 16:41:36 2010 +0200
qparser.y: Include <ctype.h> for use of isdigit
diff --git a/qparser.y b/qparser.y
index 81b2ef6..c23c03e 100644
--- a/qparser.y
+++ b/qparser.y
@@ -4,6 +4,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <ctype.h>
#define YYSTYPE value_t *
#include "query.h"
#include "iwhd-qparser.h"
http://repo.or.cz/w/iwhd.git/commit/02b46dcd6b8cf12b306eca5dd7a036964e99cd98
commit 02b46dcd6b8cf12b306eca5dd7a036964e99cd98
Author: Jim Meyering <meyering(a)redhat.com>
Date: Thu Oct 14 16:16:24 2010 +0200
apply Jeff's link-following change
diff --git a/qparser.y b/qparser.y
index d028098..81b2ef6 100644
--- a/qparser.y
+++ b/qparser.y
@@ -519,12 +519,14 @@ string_value (value_t *v, getter_t *oget, getter_t *sget)
case T_SFIELD:
return sget ? CALL_GETTER(sget,v->as_str) : NULL;
case T_LINK:
- left = string_value(v->as_tree.left,oget,sget);
- if (left) {
- return follow_link((char *)left,
- (char *)v->as_tree.right);
+ if (!v->resolved) {
+ left = string_value(v->as_tree.left,oget,sget);
+ if (left) {
+ v->resolved = follow_link((char *)left,
+ (char *)v->as_tree.right);
+ }
}
- /* Fall through. */
+ return v->resolved;
default:
return NULL;
}
http://repo.or.cz/w/iwhd.git/commit/e29ed3162dd00ccdbfb7eeeedf892a3e850c963e
commit e29ed3162dd00ccdbfb7eeeedf892a3e850c963e
Author: Jim Meyering <meyering(a)redhat.com>
Date: Thu Oct 14 16:07:33 2010 +0200
apply jeffs leak-fixing patch from master (manually merge 3 failed hunks)
diff --git a/qparser.y b/qparser.y
index 7902c90..d028098 100644
--- a/qparser.y
+++ b/qparser.y
@@ -66,6 +66,7 @@ make_number (const char *text)
if (tmp) {
tmp->type = T_NUMBER;
tmp->as_num = strtoll(text,NULL,10);
+ tmp->resolved = NULL;
}
return tmp;
@@ -79,6 +80,7 @@ make_string (const char *text, type_t t)
if (tmp) {
tmp->type = t;
tmp->as_str = xstrdup(text);
+ tmp->resolved = NULL;
}
return tmp;
@@ -93,6 +95,7 @@ make_tree (type_t t, value_t *left, value_t *right)
tmp->type = t;
tmp->as_tree.left = left;
tmp->as_tree.right = right;
+ tmp->resolved = NULL;
}
return tmp;
@@ -115,7 +118,7 @@ make_link (value_t *left, char *right)
{
char *copy;
- copy = strdup(right);
+ copy = xstrdup(right);
if (!copy) {
return NULL;
}
@@ -451,6 +454,12 @@ free_value (value_t *v)
return;
}
+ if (v->resolved) {
+ printf("freeing resolved string "%s" (%p)n",
+ v->resolved, v->resolved);
+ }
+ free(v->resolved);
+
switch (v->type) {
case T_STRING:
case T_OFIELD:
@@ -504,8 +513,7 @@ string_value (value_t *v, getter_t *oget, getter_t *sget)
return v->as_str;
case T_OFIELD:
if (!v->resolved) {
- v->resolved = oget
- ? CALL_GETTER(oget,v->as_str) : NULL;
+ v->resolved = oget ? CALL_GETTER(oget,v->as_str) : NULL;
}
return v->resolved;
case T_SFIELD:
http://repo.or.cz/w/iwhd.git/commit/92ce84f08d19187edef1807c98101dc61028d662
commit 92ce84f08d19187edef1807c98101dc61028d662
Author: Jim Meyering <meyering(a)redhat.com>
Date: Thu Oct 14 16:02:08 2010 +0200
parse-test now passes
diff --git a/qparser.y b/qparser.y
index 32e7aac..7902c90 100644
--- a/qparser.y
+++ b/qparser.y
@@ -170,7 +170,6 @@ yyerror (value_t **result, const char *msg)
policy:
bbool_expr {
- printf("policy completen");
if (syntax_error) {
printf("bad policy!n");
}
@@ -314,6 +313,70 @@ paren_expr:
%%
+#if defined PARSER_UNIT_TEST
+struct { char *name; char *value; } hacked_obj_fields[] = {
+ /* Fake object fields for generic unit testing. */
+ { "a", "2" }, { "b", "7" }, { "c", "11" },
+ /* This one's here to test links (e.g. $template.owner.name). */
+ { "template", "templates/the_tmpl" },
+ { NULL }
+};
+
+/* Fake out the eval code for unit testing. */
+const char *
+unit_oget_func (void * notused, const char *text)
+{
+ int i;
+
+ for (i = 0; hacked_obj_fields[i].name; ++i) {
+ if (!strcmp(hacked_obj_fields[i].name,text)) {
+ return xstrdup(hacked_obj_fields[i].value);
+ }
+ }
+
+ return NULL;
+}
+getter_t unit_oget = { unit_oget_func };
+
+/*
+ * Same as above, but the site-field stuff is so similar to the object-field
+ * stuff that it's not worth exercising too much separately.
+ */
+const char *
+unit_sget_func (void * notused, const char *text)
+{
+ return "never";
+}
+getter_t unit_sget = { unit_sget_func };
+
+/* Fake links from an object/key tuple to an object/key string. */
+struct { char *obj; char *key; char *value; } hacked_links[] = {
+ { "templates/the_tmpl", "owner", "users/the_user" },
+ { "users/the_user", "name", "Jeff Darcy" },
+ { NULL }
+};
+
+char *
+follow_link (const char *object, const char *key)
+{
+ unsigned int i;
+
+ for (i = 0; hacked_links[i].obj; ++i) {
+ if (strcmp(object,hacked_links[i].obj)) {
+ continue;
+ }
+ if (strcmp(key,hacked_links[i].key)) {
+ continue;
+ }
+ return hacked_links[i].value;
+ }
+
+ return NULL;
+}
+#else
+extern char *follow_link (const char *object, const char *key);
+#endif
+
void
_print_value (const value_t *v, int level)
{
@@ -330,7 +393,7 @@ _print_value (const value_t *v, int level)
printf("%*sSTRING %sn",level,"",v->as_str);
break;
case T_OFIELD:
-#if defined(UNIT_TEST)
+#if defined PARSER_UNIT_TEST
printf("%*sOBJECT FIELD %s (%s)n",level,"",v->as_str,
unit_oget_func(NULL,v->as_str));
#else
@@ -338,7 +401,7 @@ _print_value (const value_t *v, int level)
#endif
break;
case T_SFIELD:
-#if defined(UNIT_TEST)
+#if defined PARSER_UNIT_TEST
printf("%*sSERVER FIELD %s (%s)n",level,"",v->as_str,
unit_sget_func(NULL,v->as_str));
#else
@@ -426,10 +489,208 @@ parse (const char *text)
return r;
}
+/*
+ * Return the string value of an expression for comparison or display, iff
+ * all component parts are string-valued themselves. That excludes numbers
+ * and booleans.
+ */
+static const char *
+string_value (value_t *v, getter_t *oget, getter_t *sget)
+{
+ const char *left;
+
+ switch (v->type) {
+ case T_STRING:
+ return v->as_str;
+ case T_OFIELD:
+ if (!v->resolved) {
+ v->resolved = oget
+ ? CALL_GETTER(oget,v->as_str) : NULL;
+ }
+ return v->resolved;
+ case T_SFIELD:
+ return sget ? CALL_GETTER(sget,v->as_str) : NULL;
+ case T_LINK:
+ left = string_value(v->as_tree.left,oget,sget);
+ if (left) {
+ return follow_link((char *)left,
+ (char *)v->as_tree.right);
+ }
+ /* Fall through. */
+ default:
+ return NULL;
+ }
+}
+
+/*
+ * Check whether a string looks like a simple decimal number. There's
+ * probably a library function for this somewhere.
+ */
+int
+is_ok_number (const char *a_str)
+{
+ const char *p;
+
+ if (!a_str) {
+ return 0;
+ }
+
+ for (p = a_str; *p; ++p) {
+ if (!isdigit(*p)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+/*
+ * Comparisons are a bit messy. If both sides are numbers, strings that look
+ * like numbers, or expressions that evaluate to numbers (booleans evaluate
+ * to 0/1), then we do a numeric comparison. Otherwise, if both sides
+ * evaluate to strings, we attempt a string comparison. That's the logic,
+ * but the code is actually structured a different way to allow re-use of
+ * common operator-specific code at the end for both cases.
+ */
+int
+compare (value_t *left, comp_t op, value_t *right,
+ getter_t *oget, getter_t *sget)
+{
+ const char *lstr;
+ const char *rstr;
+ int lval = 0; // solely to placate gcc
+ int rval;
+ int num_ok = 1;
+
+ lstr = string_value(left,oget,sget);
+ rstr = string_value(right,oget,sget);
+
+ if (left->type == T_NUMBER) {
+ lval = left->as_num;
+ }
+ else if (lstr) {
+ if (is_ok_number(lstr)) {
+ lval = strtoll(lstr,NULL,0);
+ }
+ else {
+ num_ok = 0;
+ }
+ }
+ else {
+ lval = eval(left,oget,sget);
+ if (lval < 0) {
+ return lval;
+ }
+ }
+
+ if (right->type == T_NUMBER) {
+ rval = right->as_num;
+ }
+ else if (rstr) {
+ if (is_ok_number(rstr)) {
+ rval = strtoll(rstr,NULL,0);
+ }
+ else {
+ num_ok = 0;
+ }
+ }
+ else {
+ rval = eval(right,oget,sget);
+ if (rval < 0) {
+ return rval;
+ }
+ }
+
+ /*
+ * Strcmp returns -1/0/1, but -1 for us would mean an error and
+ * which of 0/1 we return depends on which comparison operatoer
+ * we're dealing with. Therefore, we stick the strcmp result on
+ * the left side and let the switch below do an operator-appropriate
+ * compare against zero on the right.
+ */
+ if (!num_ok) {
+ if (!lstr || !rstr) {
+ return -1;
+ }
+ lval = strcmp(lstr,rstr);
+ rval = 0;
+ }
+
+ switch (op) {
+ case C_LESSTHAN: return (lval < rval);
+ case C_LESSOREQ: return (lval <= rval);
+ case C_EQUAL: return (lval == rval);
+ case C_DIFFERENT: return (lval != rval);
+ case C_GREATEROREQ: return (lval >= rval);
+ case C_GREATERTHAN: return (lval > rval);
+ default:
+ return -1;
+ }
+}
+
+/*
+ * Evaluate an AST in the current context to one of:
+ * true=1
+ * false=0
+ * error=-1
+ * It's up to the caller whether error is functionally the same as false.
+ * Note that even T_NUMBER gets squeezed down to these three values. The
+ * only thing numbers are used for is comparing against other numbers to
+ * yield a boolean for the query or replication-policy code. If you want
+ * something that returns a number, this is the wrong language for it.
+ */
+
+int
+eval (const value_t *v, getter_t *oget, getter_t *sget)
+{
+ int res;
+ const char *str;
+
+ switch (v->type) {
+ case T_NUMBER:
+ return v->as_num != 0;
+ case T_STRING:
+ return v->as_str && *v->as_str;
+ case T_OFIELD:
+ str = CALL_GETTER(oget,v->as_str);
+ return str && *str;
+ case T_SFIELD:
+ str = CALL_GETTER(sget,v->as_str);
+ return str && *str;
+ case T_LINK:
+ str = string_value(v->as_tree.left,oget,sget);
+ if (str) {
+ str = follow_link(str,(char *)v->as_tree.right);
+ }
+ return str && *str;
+ case T_COMP:
+ return compare(v->as_tree.left,(comp_t)v->as_tree.op,
+ v->as_tree.right, oget, sget);
+ case T_NOT:
+ res = eval(v->as_tree.left,oget,sget);
+ return (res >= 0) ? !res : res;
+ case T_AND:
+ res = eval(v->as_tree.left,oget,sget);
+ if (res > 0) {
+ res = eval(v->as_tree.right,oget,sget);
+ }
+ return res;
+ case T_OR:
+ res = eval(v->as_tree.left,oget,sget);
+ if (res > 0) {
+ return res;
+ }
+ return eval(v->as_tree.right,oget,sget);
+ default:
+ return -1;
+ }
+}
+
#ifdef PARSER_UNIT_TEST
int
main (int argc, char **argv)
{
+ int fail = 0;
unsigned int i;
for (i = 1; i < argc; ++i)
{
@@ -437,22 +698,21 @@ main (int argc, char **argv)
if (!expr)
{
printf ("could not parse '%s'n", argv[i]);
+ fail = 1;
continue;
}
print_value (expr);
-#if 0
- char *str = string_value (expr, &unit_oget, &unit_sget);
+
+ const char *str = string_value (expr, &unit_oget, &unit_sget);
if (str)
{
printf ("s= %sn", str);
continue;
}
- res = eval (expr, &unit_oget, &unit_sget);
- printf ("d= %dn", res);
-#endif
+ printf ("d= %dn", eval (expr, &unit_oget, &unit_sget));
}
- return 0;
+ return fail;
}
#endif
http://repo.or.cz/w/iwhd.git/commit/3ce94781571362adb7f66ea6efe7dc9564d93d8d
commit 3ce94781571362adb7f66ea6efe7dc9564d93d8d
Author: Jim Meyering <meyering(a)redhat.com>
Date: Fri Oct 15 14:50:59 2010 +0200
build: rearrange things to use Bison/Flex; adapt; clean up
diff --git a/.gitignore b/.gitignore
index e032047..c495020 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,11 @@ config.log
config.status
configure
iwhd
-query-orig.c
-query.c
+iwhd-*.tar.gz
+iwhd-qparser.c
+iwhd-qparser.h
+qlexer.c
stamp-h1
+t/*.log
+t/parser
+t/parser.c
diff --git a/Makefile.am b/Makefile.am
index ac88336..60dce9a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,11 +18,14 @@ AM_CFLAGS = -W -Wall -Wextra -Wno-unused -Wformat-security
AM_CPPFLAGS = -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include
+iwhd_YFLAGS = -d
+
SUBDIRS = t
ACLOCAL_AMFLAGS = -I ax
bin_PROGRAMS = iwhd
-EXTRA_DIST = query.leg
+EXTRA_DIST = iwhd-qparser.h
+BUILT_SOURCES = iwhd-qparser.h qlexer.c
# iwhd is short for Image WareHouse Daemon.
@@ -37,7 +40,7 @@ iwhd_SOURCES = mpipe.h replica.c replica.h - query.c + qparser.y query.h rest.c setup.c @@ -46,6 +49,8 @@ iwhd_SOURCES = template.c template.h
+EXTRA_iwhd_SOURCES = qlexer.l
+
iwhd_CPPFLAGS = $(HAIL_CFLAGS)
iwhd_LDADD = -lmongoclient @@ -60,18 +65,6 @@ iwhd_LDADD = $(GLIB2_LIB) $(HAIL_LIBS)
-# leg is a lexer/parser-generator
-# http://piumarta.com/software/peg/peg-0.1.4.tar.gz
-query-orig.c: query.leg
- leg < $(srcdir)/query.leg > $@-t && mv $@-t $@
-
-# Apply this patch to fix NULL-deref bugs in the generated query-orig.c:
-query.c: query-orig.c query.c.diff
- patch --fuzz=0 query-orig.c $(srcdir)/query.c.diff --output=$@-t
- mv $@-t $@
-
-BUILT_SOURCES = query.c
-CLEANFILES = query-orig.c
-MOSTLYCLEANFILES = query.c query-orig.c-t
-MAINTAINERCLEANFILES = query.c query-orig.c
-EXTRA_DIST += query.c.diff
+MOSTLYCLEANFILES = qlexer.c
+MAINTAINERCLEANFILES = qlexer.c
+EXTRA_DIST += qlexer.c
diff --git a/configure.ac b/configure.ac
index bd0609b..afbe4c3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -28,10 +28,8 @@ AM_SILENT_RULES([yes]) # make --enable-silent-rules the default.
AC_PROG_CXX
AC_PROG_CC
AM_PROG_CC_C_O
-
-leg -V > /dev/null 2>&1 ||
- AC_MSG_ERROR([you need the leg program to build this code;
- <http://piumarta.com/software/peg/>])
+AC_PROG_YACC
+AC_PROG_LEX
AC_CHECK_LIB([curl], [curl_easy_init],
[CURL_LIB=-lcurl],
diff --git a/qlexer.l b/qlexer.l
index 73e685f..c90bf15 100644
--- a/qlexer.l
+++ b/qlexer.l
@@ -1,22 +1,13 @@
%{
#include <stdio.h>
-#if defined(UNIT_TEST)
-enum {
- T_STRING, T_DATE, T_NUMBER, T_ID,
- T_DOLLAR, T_WAFFLE, T_LPAREN, T_RPAREN,
- T_LESS, T_GREATER, T_EQUAL, T_NOT,
- T_AND, T_OR, T_DOT, T_SPACE,
- T_INVALID
-};
-#else
-#include "qparser.tab.h"
-#endif
+#include "iwhd-qparser.h"
+#define YY_DECL extern int yylex (YYSTYPE *lvalp)
%}
%%
-"[^"]*" { return T_STRING; }
+"[^"]*" { yytext[yyleng-1] = 0; yytext++; return T_STRING; }
~[^~]*~ { return T_DATE; }
[0-9]+ { return T_NUMBER; }
[a-z_]+ { return T_ID; }
@@ -54,12 +45,12 @@ yyerror (char *msg)
int
main (int argc, char **argv)
{
- int t;
yy_scan_string(""!@#" ~xxx~ 123 abc $#()<>=!&|.");
do {
- t = yylex();
+ YYSTYPE v;
+ int t = yylex(&v);
switch (t) {
case T_STRING:
if (!yytext[0]) {
diff --git a/qparser.y b/qparser.y
index 7df27a1..32e7aac 100644
--- a/qparser.y
+++ b/qparser.y
@@ -4,9 +4,9 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include "query.h"
#define YYSTYPE value_t *
-#include "qparser.h"
+#include "query.h"
+#include "iwhd-qparser.h"
static void
xalloc_die (void)
@@ -133,9 +133,6 @@ make_link (value_t *left, char *right)
*/
extern char *yytext;
-/* In a sane world, yyparse would return this. */
-value_t *cur_val;
-
/*
* IMO it's wrong for us to get into the bbool_expr=policy rule when there's
* a syntax error, but we do. The good news is that it's easy to free the
@@ -144,17 +141,21 @@ value_t *cur_val;
* ourselves) to figure out whether we got a valid tree or not.
* No, yynerrs doesn't seem to give the right answer.
*/
-int syntax_error = 0;
+static int syntax_error = 0;
void
-yyerror (char *msg)
+yyerror (value_t **result, const char *msg)
{
- printf("%s: %sn",__func__,msg);
- syntax_error = 1;
+ // error (0, 0, "parse error: %sn", msg);
+ // FIXME do this via param, not file-global
+ syntax_error = 1;
}
%}
+%define api.pure
+%parse-param { value_t **result }
+
%token T_STRING T_DATE T_NUMBER T_ID
%token T_DOLLAR T_WAFFLE T_DOT
%token T_LPAREN T_RPAREN
@@ -174,21 +175,21 @@ policy:
printf("bad policy!n");
}
else {
- cur_val = $1;
+ *result = $1;
}
};
bbool_expr:
ubool_expr {
- printf("promoting ubool_expr to bbool_exprn");
+ // printf("promoting ubool_expr to bbool_exprn");
$$ = $1;
}|
bbool_expr T_AND T_AND ubool_expr {
- printf("found AND expressionn");
+ // printf("found AND expressionn");
$$ = make_tree(T_AND,$1,$4);
}|
bbool_expr T_OR T_OR ubool_expr {
- printf("found OR expressionn");
+ // printf("found OR expressionn");
$$ = make_tree(T_OR,$1,$4);
}|
bbool_expr T_SPACE {
@@ -199,11 +200,11 @@ bbool_expr:
ubool_expr:
comp_expr {
- printf("promoting comp_expr to ubool_exprn");
+ // printf("promoting comp_expr to ubool_exprn");
$$ = $1;
}|
T_NOT comp_expr {
- printf("found NOT expressionn");
+ // printf("found NOT expressionn");
$$ = make_tree(T_NOT,$2,NULL);
}|
ubool_expr T_SPACE {
@@ -215,31 +216,31 @@ ubool_expr:
comp_expr:
atom {
- printf("promoting atom to comp_exprn");
+ // printf("promoting atom to comp_exprn");
$$ = $1;
}|
atom T_LESS atom {
- printf("found LESS THAN expressionn");
+ // printf("found LESS THAN expressionn");
$$ = make_comp(C_LESSTHAN,$1,$3);
}|
atom T_LESS T_EQUAL atom {
- printf("found LESS OR EQUAL expressionn");
+ // printf("found LESS OR EQUAL expressionn");
$$ = make_comp(C_LESSOREQ,$1,$4);
}|
atom T_EQUAL T_EQUAL atom {
- printf("found EQUAL expressionn");
+ // printf("found EQUAL expressionn");
$$ = make_comp(C_EQUAL,$1,$4);
}|
atom T_NOT T_EQUAL atom {
- printf("found NOT EQUAL expressionn");
+ // printf("found NOT EQUAL expressionn");
$$ = make_comp(C_DIFFERENT,$1,$4);
}|
atom T_GREATER T_EQUAL atom {
- printf("found GREATER OR EQUAL expressionn");
+ // printf("found GREATER OR EQUAL expressionn");
$$ = make_comp(C_GREATEROREQ,$1,$4);
}|
atom T_GREATER atom {
- printf("found GREATER THAN expressionn");
+ // printf("found GREATER THAN expressionn");
$$ = make_comp(C_GREATERTHAN,$1,$3);
}|
comp_expr T_SPACE {
@@ -250,15 +251,15 @@ comp_expr:
atom:
link_field {
- printf("promoting link_field to atomn");
+ // printf("promoting link_field to atomn");
$$ = $1;
}|
literal {
- printf("promoting literal to atomn");
+ // printf("promoting literal to atomn");
$$ = $1;
}|
paren_expr {
- printf("promoting paren_expr to atomn");
+ // printf("promoting paren_expr to atomn");
$$ = $1;
}|
atom T_SPACE {
@@ -269,45 +270,45 @@ atom:
link_field:
field {
- printf("promoting field to link_fieldn");
+ // printf("promoting field to link_fieldn");
$$ = $1;
}|
link_field T_DOT T_ID {
- printf("found LINK FIELDn");
+ // printf("found LINK FIELDn");
$$ = make_link($1,yytext);
};
field:
T_DOLLAR T_ID {
- printf("found DOLLAR FIELDn");
+ // printf("found DOLLAR FIELDn");
$$ = make_string(yytext,T_OFIELD);
}|
T_WAFFLE T_ID {
- printf("found WAFFLE FIELDn");
+ // printf("found WAFFLE FIELDn");
$$ = make_string(yytext,T_SFIELD);
};
literal:
T_NUMBER {
- printf("found NUMBER %sn",yytext);
+ // printf("found NUMBER %sn",yytext);
$$ = make_number(yytext);
}|
T_STRING {
- printf("found STRING %sn",yytext);
+ // printf("found STRING %sn",yytext);
$$ = make_string(yytext,T_STRING);
}|
T_DATE {
- printf("found DATEn");
+ // printf("found DATEn");
$$ = make_string(yytext,T_DATE);
}|
T_ID {
- printf("found ID %sn",yytext);
+ // printf("found ID %sn",yytext);
$$ = make_string(yytext,T_ID);
};
paren_expr:
T_LPAREN bbool_expr T_RPAREN {
- printf("found PAREN expressionn");
+ // printf("found PAREN expressionn");
$$ = $2;
};
@@ -413,39 +414,45 @@ free_value (value_t *v)
}
}
+#include "qlexer.c"
+
+value_t *
+parse (const char *text)
+{
+ yy_scan_string(text);
+ value_t *result;
+ value_t *r = yyparse (&result) == 0 ? result : NULL;
+ yylex_destroy();
+ return r;
+}
+
+#ifdef PARSER_UNIT_TEST
int
main (int argc, char **argv)
{
- char *data = "#a=="fubar"&&!($b.c.d<3)";
- char *edata = "gobbledy-gook";
-
- yy_scan_string(data);
- yyparse();
- if (cur_val) {
- if (!syntax_error) {
- print_value(cur_val);
- }
- else {
- printf("discarding erroneous parse treen");
- }
- free_value(cur_val);
- cur_val = NULL;
+ unsigned int i;
+ for (i = 1; i < argc; ++i)
+ {
+ value_t *expr = parse (argv[i]);
+ if (!expr)
+ {
+ printf ("could not parse '%s'n", argv[i]);
+ continue;
}
- yylex_destroy();
- yy_scan_string(edata);
- yyparse();
- if (cur_val) {
- if (!syntax_error) {
- print_value(cur_val);
- }
- else {
- printf("discarding erroneous parse treen");
- }
- free_value(cur_val);
- cur_val = NULL;
+ print_value (expr);
+#if 0
+ char *str = string_value (expr, &unit_oget, &unit_sget);
+ if (str)
+ {
+ printf ("s= %sn", str);
+ continue;
}
- yylex_destroy();
+ res = eval (expr, &unit_oget, &unit_sget);
+ printf ("d= %dn", res);
+#endif
+ }
- return 0;
+ return 0;
}
+#endif
diff --git a/query.h b/query.h
index 8189633..f7823e2 100644
--- a/query.h
+++ b/query.h
@@ -19,16 +19,15 @@
extern "C" {
#endif
+#include "iwhd-qparser.h"
+
typedef enum {
C_LESSTHAN, C_LESSOREQ,
C_EQUAL, C_DIFFERENT,
C_GREATEROREQ, C_GREATERTHAN
} comp_t;
-typedef enum {
- T_NUMBER, T_STRING, T_OFIELD, T_SFIELD,
- T_COMP, T_NOT, T_AND, T_OR, T_LINK
-} type_t;
+typedef enum yytokentype type_t;
typedef struct _value {
type_t type;
@@ -50,12 +49,12 @@ typedef struct {
} getter_t;
#define CALL_GETTER(g,x) g->func(g->ctx,x)
-
-value_t *parse (const char *text);
int eval (const value_t *expr, getter_t *oget, getter_t *sget);
void free_value (value_t *);
void print_value (const value_t *);
+value_t *parse (const char *text);
+
#if defined(__CPLUSPLUS__) || defined(__cplusplus)
}
#endif
diff --git a/t/Makefile.am b/t/Makefile.am
index 91f0ad4..c2a0af6 100644
--- a/t/Makefile.am
+++ b/t/Makefile.am
@@ -34,8 +34,8 @@ parser_CPPFLAGS = -I $(top_srcdir)
parser.c: Makefile.am
rm -f $@-t $@
- printf '#define UNIT_TEST 1n' > $@-t
- printf '#include "query.c"n' >> $@-t
+ printf '#define PARSER_UNIT_TEST 1n' > $@-t
+ printf '#include "iwhd-qparser.c"n' >> $@-t
chmod a=r $@-t
mv $@-t $@
-----------------------------------------------------------------------
Summary of changes:
Makefile.am | 3 +--
query.h | 27 +++++++++++++++++++++++++++
2 files changed, 28 insertions(+), 2 deletions(-)
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months
[repo.or.cz] iwhd.git branch master updated: v0.0-168-g7fb8264
by Jeff Darcy
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, master has been updated
via 7fb8264a528a19fd999dc4626121c2d6427d437d (commit)
from 8f3bc25341e2a212e3497028d59ff6a5f3527445 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/7fb8264a528a19fd999dc4626121c2d6427d437d
commit 7fb8264a528a19fd999dc4626121c2d6427d437d
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Wed Oct 13 16:08:39 2010 -0400
Add link-following implementation (not just syntax).
Also fixed parser main() so that mainline can pass tests modified to suit
an unposted version.
diff --git a/query.leg b/query.leg
index 2b9cf59..403f0d0 100644
--- a/query.leg
+++ b/query.leg
@@ -230,12 +230,14 @@ string_value (value_t *v, getter_t *oget, getter_t *sget)
case T_SFIELD:
return sget ? CALL_GETTER(sget,v->as_str) : NULL;
case T_LINK:
- left = string_value(v->as_tree.left,oget,sget);
- if (left) {
- return follow_link((char *)left,
- (char *)v->as_tree.right);
+ if (!v->resolved) {
+ left = string_value(v->as_tree.left,oget,sget);
+ if (left) {
+ v->resolved = follow_link((char *)left,
+ (char *)v->as_tree.right);
+ }
}
- /* Fall through. */
+ return v->resolved;
default:
return NULL;
}
@@ -591,11 +593,13 @@ main (int argc, char **argv)
value_t *expr;
int res;
const char *str;
+ unsigned int failed = 0;
for (i = 1; i < argc; ++i) {
expr = parse(argv[i]);
if (!expr) {
printf("could not parse '%s'n",argv[i]);
+ ++failed;
continue;
}
print_value(expr);
@@ -608,6 +612,6 @@ main (int argc, char **argv)
printf("d= %dn",res);
}
- return 0;
+ return failed;
}
#endif
diff --git a/replica.c b/replica.c
index 07589bd..8bdcb53 100644
--- a/replica.c
+++ b/replica.c
@@ -757,7 +757,6 @@ replicate (const char *url, size_t size, const char *policy)
static void
replicate_namespace_action (const char *name, repl_t action)
{
- unsigned int i;
repl_item *item;
GHashTableIter iter;
gpointer key;
@@ -768,7 +767,8 @@ replicate_namespace_action (const char *name, repl_t action)
if (!strcmp(key,me)) {
continue;
}
- DPRINTF("replicating delete(%s) on %un",name,i);
+ DPRINTF("replicating delete(%s) on %sn",name,
+ ((provider_t *)value)->name);
item = malloc(sizeof(*item));
if (!item) {
error(0,errno,"could not create repl_item for %s",
@@ -812,12 +812,22 @@ replicate_bcreate (const char *name)
/* Part of our API to the query module. */
char *
-follow_link (char *object, char *key)
+follow_link (char *object, const char *key)
{
- (void)object;
- (void)key;
+ char *slash;
+ char *value = NULL;
- return "no_such_object";
+ slash = strchr(object,'/');
+ if (!slash) {
+ return NULL;
+ }
+
+ *(slash++) = '0';
+ (void)meta_get_value(object,slash,key,&value);
+ *(--slash) = '/';
+
+ DPRINTF("%s: %s:%s => %sn",__func__,object,key,value);
+ return value;
}
int
-----------------------------------------------------------------------
Summary of changes:
query.leg | 16 ++++++++++------
replica.c | 22 ++++++++++++++++------
2 files changed, 26 insertions(+), 12 deletions(-)
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months
[repo.or.cz] iwhd.git branch master updated: v0.0-167-g8f3bc25
by Jim Meyering
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, master has been updated
via 8f3bc25341e2a212e3497028d59ff6a5f3527445 (commit)
from bb9ae00fcc2c84440108054ceb0ea33bafda5cf4 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/8f3bc25341e2a212e3497028d59ff6a5f3527445
commit 8f3bc25341e2a212e3497028d59ff6a5f3527445
Author: Jim Meyering <meyering(a)redhat.com>
Date: Wed Oct 13 21:30:12 2010 +0200
tests: add missing "Exit $fail"
diff --git a/t/parser-test b/t/parser-test
index 0005436..cf006c6 100755
--- a/t/parser-test
+++ b/t/parser-test
@@ -34,3 +34,5 @@ compare e1.exp e1.out || fail=1
parser '1 = = 1' > e2.out 2>&1; test $? = 1 || fail=1
printf "could not parse '1 = = 1'n" > e2.exp
compare e2.exp e2.out || fail=1
+
+Exit $fail
-----------------------------------------------------------------------
Summary of changes:
t/parser-test | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months
[repo.or.cz] iwhd.git branch master updated: v0.0-166-gbb9ae00
by Jim Meyering
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, master has been updated
via bb9ae00fcc2c84440108054ceb0ea33bafda5cf4 (commit)
from 436ce690c6dca730f9fdf6e3cd831d53c48ab474 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/bb9ae00fcc2c84440108054ceb0ea33bafda5cf4
commit bb9ae00fcc2c84440108054ceb0ea33bafda5cf4
Author: Jim Meyering <meyering(a)redhat.com>
Date: Wed Oct 13 21:22:49 2010 +0200
tests: parser-test: don't ignore "compare" failure
diff --git a/t/parser-test b/t/parser-test
index 9d0a6c8..0005436 100755
--- a/t/parser-test
+++ b/t/parser-test
@@ -20,17 +20,17 @@ d= 0
EOF
parser '#a=="fubar"&&!($b.c.d<3)' > features 2> err || fail=1
-compare features f.exp
-compare /dev/null err
+compare features f.exp || fail=1
+compare /dev/null err || fail=1
parser '#a == "fubar" && ! ( $b.c.d < 3)' > white-space 2> err || fail=1
-compare white-space f.exp
-compare /dev/null err
+compare white-space f.exp || fail=1
+compare /dev/null err || fail=1
parser '$ a' > e1.out 2>&1; test $? = 1 || fail=1
printf "could not parse '$ a'n" > e1.exp
-compare e1.exp e1.out
+compare e1.exp e1.out || fail=1
parser '1 = = 1' > e2.out 2>&1; test $? = 1 || fail=1
printf "could not parse '1 = = 1'n" > e2.exp
-compare e2.exp e2.out
+compare e2.exp e2.out || fail=1
-----------------------------------------------------------------------
Summary of changes:
t/parser-test | 12 ++++++------
1 files changed, 6 insertions(+), 6 deletions(-)
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months
[repo.or.cz] iwhd.git branch master updated: v0.0-165-g436ce69
by Jeff Darcy
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, master has been updated
via 436ce690c6dca730f9fdf6e3cd831d53c48ab474 (commit)
from 1b561174b1eca8fe2fca2586313d324d19e08dd9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/436ce690c6dca730f9fdf6e3cd831d53c48ab474
commit 436ce690c6dca730f9fdf6e3cd831d53c48ab474
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Wed Oct 13 14:06:00 2010 -0400
New tests for autostart and "headless operation"
t/Makefile.am: added new "auto" test
t/auto: new
t/basic: added headless test, fixed bug in waiting for mongod to start
t/init.cfg: changed ports we use to avoid the mongo/auto defaults
auto.c: added --pidfilepath so tests can get mongod PID
diff --git a/auto.c b/auto.c
index a0e0b21..01521d6 100644
--- a/auto.c
+++ b/auto.c
@@ -30,6 +30,7 @@ static char *auto_arg_mongod[] = {
/* "--fork", */ /* chdirs god knows where, we cannot use this. */
/* "--logpath", AUTO_MONGOD_LOG, */ /* required by --fork */
/* "--logappend", */
+ "--pidfilepath", "mongo.pid",
NULL
};
@@ -39,6 +40,7 @@ static char *auto_arg_mongod_quiet[] = {
"--port", auto_arg_port,
"--dbpath", AUTO_DIR_DB,
"--logpath", AUTO_MONGOD_LOG,
+ "--pidfilepath", "mongo.pid",
NULL
};
diff --git a/t/Makefile.am b/t/Makefile.am
index 31a0e92..91f0ad4 100644
--- a/t/Makefile.am
+++ b/t/Makefile.am
@@ -17,7 +17,8 @@
TESTS = basic replication - parser-test
+ parser-test + auto
EXTRA_DIST = $(TESTS) diff --git a/t/auto b/t/auto
new file mode 100644
index 0000000..86efade
--- /dev/null
+++ b/t/auto
@@ -0,0 +1,57 @@
+#!/bin/sh
+# Test autostart functionality.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+mkdir iwhd || framework_failure_ mkdir failed
+
+port=9094
+m_port=27018 # auto-start default
+
+printf '[{"path": "FS", "type": "fs", "name": "primary"}]n' + > iwhd.cfg || fail=1
+
+# Make sure specifying both -a and -c fails.
+iwhd -v -a -c iwhd.cfg && fail=1
+
+# Make sure specifying *neither* fails.
+iwhd -v && fail=1
+
+# Start for real.
+iwhd -v -p $port -a &
+iwhd_pid=$!
+cleanup_() { kill $iwhd_pid; }
+
+# Wait for up to 3 seconds for mongod to begin listening.
+wait_for .1 30 'mongo localhost:$m_port < /dev/null' + || framework_failure_ mongod failed to start
+# Wait for up to 3 seconds for iwhd to begin listening on $port.
+wait_for .1 30 "curl -s http://localhost:$port" + || framework_failure_ iwhd failed to start
+
+mongo_pid=$(cat mongo.pid)
+echo "mongo_pid = $mongo_pid"
+cleanup_() { kill $iwhd_pid; kill -9 $mongo_pid; }
+
+# Create an empty bucket.
+curl -X PUT http://localhost:$port/b1 || fail=1
+test -d _fs/b1 || fail=1
+
+# Create a file in that bucket.
+echo foo | curl -T - http://localhost:$port/b1/f1 || fail=1
+# Test presence in the FS.
+test -f _fs/b1/f1 || fail=1
+test "$(cat _fs/b1/f1)" = foo || fail=1
+# Test through the API as well.
+curl http://localhost:$port/b1/f1 > f1copy
+test "$(cat f1copy)" = foo || fail=1
+
+# Delete an object.
+curl -X DELETE http://localhost:$port/b1/f1 || fail=1
+# Ensure that it's gone from the file system.
+test -f _fs/b1/f1 && fail=1
+# Ensure that it's gone in the API as well.
+curl -f http://localhost:$port/b1/f1 > /dev/null 2> del_object.err
+grep ' 404$' del_object.err || fail=1
+
+Exit $fail
diff --git a/t/basic b/t/basic
index 4dac5e9..2ea2fcd 100644
--- a/t/basic
+++ b/t/basic
@@ -12,7 +12,7 @@ mongo_pid=$!
cleanup_() { kill -9 $mongo_pid; }
# Wait for up to 3 seconds for mongod to begin listening.
-wait_for .1 30 'mongo < /dev/null' +wait_for .1 30 'mongo localhost:$m_port < /dev/null' || framework_failure_ mongod failed to start
port=9091
@@ -264,6 +264,12 @@ for attr in bucket key date etag loc; do
grep ' 400$' bad_attr.err || fail=1
done
+# Test "headless" operation (no access to metadata DB).
+kill -9 $mongo_pid
+cleanup_() { kill $iwhd_pid; }
+curl $bucket/f3 > f3copy
+test "$(cat f3copy)" = "nothing" || fail=1
+
# TBD: add attribute-delete tests when that functionality is implemented
# TBD: add white-box tests for attributes in mongo
diff --git a/t/init.cfg b/t/init.cfg
index 1fa2bb6..bffe669 100644
--- a/t/init.cfg
+++ b/t/init.cfg
@@ -15,7 +15,8 @@ curl() { env curl -q -s -S "$@"; }
# m_port=$(expr $mongo_base_port + DDD)
# And then start mongod on that port and specify iwhd's DB host
# using --db=localhost:$m_port.
-mongo_base_port=27017
+# Avoid the built-in and auto-start defaults (27017 and 27018 respectively)
+mongo_base_port=27019
wait_for()
{
-----------------------------------------------------------------------
Summary of changes:
auto.c | 2 ++
t/Makefile.am | 3 ++-
t/auto | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
t/basic | 8 +++++++-
t/init.cfg | 3 ++-
5 files changed, 70 insertions(+), 3 deletions(-)
create mode 100644 t/auto
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months
[repo.or.cz] iwhd.git branch master updated: v0.0-164-g1b56117
by Jeff Darcy
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, master has been updated
via 1b561174b1eca8fe2fca2586313d324d19e08dd9 (commit)
from 0f82ac4e7c2cfbfe18a58f89b5543ca8e64290ba (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/1b561174b1eca8fe2fca2586313d324d19e08dd9
commit 1b561174b1eca8fe2fca2586313d324d19e08dd9
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Wed Oct 13 15:12:27 2010 -0400
Fix rollup which didn't include renamed/added files.
diff --git a/replica.c b/replica.c
new file mode 100644
index 0000000..07589bd
--- /dev/null
+++ b/replica.c
@@ -0,0 +1,827 @@
+/* Copyright (C) 2010 Red Hat, Inc.
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
+
+#include <config.h>
+
+#include <errno.h>
+#include <error.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <poll.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <unistd.h>
+#include <assert.h>
+
+#include <microhttpd.h>
+#include <curl/curl.h>
+#include <hstor.h>
+
+#include "iwh.h"
+#include "setup.h"
+#include "query.h"
+#include "meta.h"
+
+/* Sizes for internal string buffers. */
+#define ADDR_SIZE 1024
+#define SVC_ACC_SIZE 128
+#define HEADER_SIZE 64
+
+typedef enum {
+ REPL_PUT, /* store an object */
+ REPL_ODELETE, /* delete an object */
+ REPL_BCREATE, /* create a bucket */
+ /* TBD: bucket deletion, others? */
+} repl_t;
+
+typedef struct _repl_item {
+ struct _repl_item *next;
+ repl_t type;
+ char *path;
+ provider_t *server;
+ size_t size;
+ int pipes[2];
+} repl_item;
+
+typedef struct {
+ char *cur_bucket;
+ char *cur_key;
+ provider_t *cur_server;
+} query_ctx_t;
+
+static repl_item *queue_head = NULL;
+static repl_item *queue_tail = NULL;
+static pthread_mutex_t queue_lock;
+static sem_t queue_sema;
+static int rep_count = 0;
+
+static size_t
+junk_writer (/* const */ void *ptr, size_t size, size_t nmemb, void *stream)
+{
+ size_t n;
+
+ n = fwrite(ptr,size,nmemb,stream);
+ if (n != nmemb)
+ error(0, 0, "warning: write failed");
+ if (fflush(stream))
+ error(0, 0, "warning: write failed");
+ DPRINTF("in %s(%zu,%zu) => %zun",__func__,size,nmemb,n);
+
+ return n;
+}
+
+static void *
+proxy_repl_prod_fs (void *ctx)
+{
+ repl_item *item = ctx;
+ int ifd;
+ int ofd;
+ char buf[1<<16];
+ ssize_t ibytes;
+ ssize_t obytes;
+ ssize_t offset;
+
+ DPRINTF("replicating from %s (FS)n",item->path);
+
+ ifd = open(item->path,O_RDONLY);
+ if (ifd < 0) {
+ error(0,errno,"ifd open");
+ return THREAD_FAILED;
+ }
+ ofd = item->pipes[1];
+
+ for (;;) {
+ ibytes = read(ifd,buf,sizeof(buf));
+ if (ibytes <= 0) {
+ if (ibytes < 0) {
+ error(0,errno,"%s: read failed", item->path);
+ }
+ else {
+ DPRINTF("EOF on ifdn");
+ }
+ break;
+ }
+ offset = 0;
+ do {
+ obytes = write(ofd,buf+offset,ibytes);
+ if (obytes <= 0) {
+ if (obytes < 0) {
+ error(0,errno,"ofd write");
+ }
+ else {
+ DPRINTF("zero-length write on ofdn");
+ }
+ break;
+ }
+ ibytes -= obytes;
+ offset += obytes;
+ } while (ibytes > 0);
+ }
+
+ close(ifd);
+ close(ofd);
+
+ DPRINTF("%s returningn",__func__);
+ close(item->pipes[1]);
+ return NULL;
+}
+
+static void *
+proxy_repl_prod (void *ctx)
+{
+ repl_item *item = ctx;
+ FILE *fp = fdopen(item->pipes[1],"w");
+ char addr[ADDR_SIZE];
+ CURL *curl;
+ char svc_acc[SVC_ACC_SIZE];
+ struct hstor_client *hstor;
+ char *bucket;
+ char *key;
+ char *stctx;
+ char *myurl;
+ int chars;
+
+ if (fp == NULL) {
+ error(0, errno, "%s: fdopen failed", __func__);
+ return NULL;
+ }
+
+ chars = snprintf(addr,ADDR_SIZE,
+ "http://%s:%u/%s",proxy_host,proxy_port,item->path);
+ if (chars >= ADDR_SIZE) {
+ error(0,0,"path too long in %s",__func__);
+ goto done;
+ }
+ DPRINTF("replicating from %sn",addr);
+
+ if (s3mode) {
+ chars = snprintf(svc_acc,SVC_ACC_SIZE,"%s:%u",
+ proxy_host,proxy_port);
+ if (chars >= SVC_ACC_SIZE) {
+ error(0,0,"svc_acc too long in %s",__func__);
+ goto done;
+ }
+ hstor = hstor_new(svc_acc,proxy_host,
+ proxy_key,proxy_secret);
+ /* Blech. Can't conflict with consumer, though. */
+ myurl = strdup(item->path);
+ assert (myurl);
+ bucket = strtok_r(myurl,"/",&stctx);
+ key = strtok_r(NULL,"/",&stctx);
+ hstor_get(hstor,bucket,key,
+ junk_writer,fp,0);
+ hstor_free(hstor);
+ free(myurl);
+ }
+ else {
+ curl = curl_easy_init();
+ curl_easy_setopt(curl,CURLOPT_URL,addr);
+ curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION,junk_writer);
+ curl_easy_setopt(curl,CURLOPT_WRITEDATA,fp);
+ DPRINTF("%s calling curl_easy_performn",__func__);
+ curl_easy_perform(curl);
+ curl_easy_cleanup(curl);
+ }
+
+done:
+ DPRINTF("%s returningn",__func__);
+ /* Closing should signal to the consumer that we're finished. */
+ fclose(fp);
+ return NULL;
+}
+
+static size_t
+junk_reader (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+ size_t n;
+
+ n = fread(ptr,size,nmemb,stream);
+ printf("in %s(%zu,%zu) => %zun",__func__,size,nmemb,n);
+ return n;
+}
+
+static size_t
+cf_writer (void *ptr ATTRIBUTE_UNUSED, size_t size, size_t nmemb,
+ void *stream ATTRIBUTE_UNUSED)
+{
+ return size * nmemb;
+}
+
+static size_t
+cf_header (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+ char *next;
+ char *sctx;
+ provider_t *server = (provider_t *)stream;
+
+ next = strtok_r(ptr,":",&sctx);
+ if (next) {
+ if (!strcasecmp(next,"X-Storage-Url")) {
+ next = strtok_r(NULL," nr",&sctx);
+ if (next) {
+ DPRINTF("got CF URL %sn",next);
+ /* NB: after this, original "host" is gone. */
+ free((char *)server->host);
+ server->host = strdup(next);
+ }
+ }
+ else if (!strcasecmp(next,"X-Storage-Token")) {
+ next = strtok_r(NULL," nr",&sctx);
+ if (next) {
+ DPRINTF("got CF token %sn",next);
+ server->token = strdup(next);
+ }
+ }
+ }
+ return size * nmemb;
+}
+
+static const char *
+get_cloudfiles_token (provider_t *server, const char *host, unsigned int port,
+ const char * user, const char * key)
+{
+ CURL *curl;
+ char addr[ADDR_SIZE];
+ char auth_user[HEADER_SIZE];
+ char auth_key[HEADER_SIZE];
+ char *token;
+ struct curl_slist *slist;
+ int chars;
+
+ token = server->token;
+ if (token) {
+ return token;
+ }
+
+ chars = snprintf(addr,ADDR_SIZE,"https://%s:%u/v1.0",host,port);
+ if (chars >= ADDR_SIZE) {
+ error(0,0,"API URL too long in %s",__func__);
+ return NULL;
+ }
+
+ chars = snprintf(auth_user,HEADER_SIZE,"X-Auth-User: %s",user);
+ if (chars >= HEADER_SIZE) {
+ error(0,0,"auth_user too long in %s",__func__);
+ return NULL;
+ }
+
+ chars = snprintf(auth_key,HEADER_SIZE,"X-Auth-Key: %s",key);
+ if (chars >= HEADER_SIZE) {
+ error(0,0,"auth_key too long in %s",__func__);
+ return NULL;
+ }
+
+ curl = curl_easy_init();
+ curl_easy_setopt(curl,CURLOPT_URL,addr);
+ curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION,cf_writer);
+ curl_easy_setopt(curl,CURLOPT_HEADERFUNCTION,cf_header);
+ curl_easy_setopt(curl,CURLOPT_WRITEHEADER,server);
+ slist = curl_slist_append(NULL,auth_user);
+ slist = curl_slist_append(slist,auth_key);
+ curl_easy_setopt(curl,CURLOPT_HTTPHEADER,slist);
+ curl_easy_perform(curl);
+ curl_easy_cleanup(curl);
+ curl_slist_free_all(slist);
+
+ return server->token;
+}
+
+static void *
+proxy_repl_cons (void *ctx)
+{
+ repl_item *item = ctx;
+ FILE *fp = fdopen(item->pipes[0],"r");
+ char addr[ADDR_SIZE];
+ CURL *curl;
+ provider_t *server;
+ char svc_acc[SVC_ACC_SIZE];
+ char auth_hdr[HEADER_SIZE];
+ struct hstor_client *hstor;
+ char *bucket;
+ char *key;
+ char *stctx;
+ const char *s_host;
+ unsigned int s_port;
+ const char *s_key;
+ const char *s_secret;
+ const char *s_type;
+ const char *s_name;
+ struct curl_slist *slist;
+ char *myurl;
+ int chars;
+
+ if (fp == NULL) {
+ error(0, errno, "%s: fdopen failed", __func__);
+ return THREAD_FAILED;
+ }
+
+ server = item->server;
+ s_host = server->host;
+ s_port = server->port;
+ s_key = server->username;
+ s_secret = server->password;
+ s_type = server->type;
+ s_name = server->name;
+
+ myurl = strdup(item->path);
+ assert (myurl);
+ bucket = strtok_r(myurl,"/",&stctx);
+ key = strtok_r(NULL,"/",&stctx);
+
+ if (!strcasecmp(s_type,"s3")) {
+ DPRINTF("replicating %zu to %s/%s (S3)n",item->size,s_host,
+ item->path);
+ chars = snprintf(svc_acc,SVC_ACC_SIZE,"%s:%u",s_host,s_port);
+ if (chars >= SVC_ACC_SIZE) {
+ error(0,0,"svc_acc too long in %s",__func__);
+ return THREAD_FAILED;
+ }
+ hstor = hstor_new(svc_acc,s_host,s_key,s_secret);
+ /* Blech. Can't conflict with producer, though. */
+ hstor_put(hstor,bucket,key,
+ junk_reader,item->size,fp,NULL);
+ hstor_free(hstor);
+ }
+ else {
+ const char *token_str = NULL;
+ if (!strcasecmp(s_type,"cf")) {
+ token_str = get_cloudfiles_token(server,s_host,s_port,
+ s_key, s_secret);
+ if (!token_str) {
+ DPRINTF("could not get CF tokenn");
+ return THREAD_FAILED;
+ }
+ /* Re-fetch as this might have changed. */
+ s_host = server->host;
+ chars = snprintf(addr,ADDR_SIZE,"%s/%s",
+ s_host,item->path);
+ if (chars >= ADDR_SIZE) {
+ error(0,0,"CF path too long in %s",__func__);
+ return THREAD_FAILED;
+ }
+ DPRINTF("replicating %zu to %s (CF)n",item->size,
+ addr);
+ }
+ else {
+ chars = snprintf(addr,ADDR_SIZE,"http://%s:%u/%s",
+ s_host,s_port,item->path);
+ if (chars >= ADDR_SIZE) {
+ error(0,0,"HTTP path too long in %s",
+ __func__);
+ return THREAD_FAILED;
+ }
+ DPRINTF("replicating %zu to %s (repod)n",item->size,
+ addr);
+ }
+ curl = curl_easy_init();
+ curl_easy_setopt(curl,CURLOPT_URL,addr);
+ curl_easy_setopt(curl,CURLOPT_UPLOAD,1);
+ curl_easy_setopt(curl,CURLOPT_INFILESIZE_LARGE,
+ (curl_off_t)item->size);
+ curl_easy_setopt(curl,CURLOPT_READFUNCTION,junk_reader);
+ if (!strcasecmp(s_type,"cf")) {
+ chars = snprintf(auth_hdr,HEADER_SIZE,
+ "X-Auth-Token: %s",token_str);
+ if (chars >= HEADER_SIZE) {
+ error(0,0,"auth_token too long in %s",
+ __func__);
+ return THREAD_FAILED;
+ }
+ slist = curl_slist_append(NULL,auth_hdr);
+ /*
+ * Rackspace doesn't clearly document that you'll get
+ * 412 (Precondition Failed) if you omit this.
+ */
+ slist = curl_slist_append(slist,
+ "Content-Type: binary/octet-stream");
+ }
+ else {
+ slist = curl_slist_append(NULL,"X-redhat-role: master");
+ }
+ curl_easy_setopt(curl,CURLOPT_HTTPHEADER,slist);
+ curl_easy_setopt(curl,CURLOPT_READDATA,fp);
+ DPRINTF("%s calling curl_easy_performn",__func__);
+ curl_easy_perform(curl);
+ curl_easy_cleanup(curl);
+ curl_slist_free_all(slist);
+ }
+
+ DPRINTF("%s returningn",__func__);
+ fclose(fp);
+ meta_got_copy(bucket,key,s_name);
+ free(myurl);
+ return NULL;
+}
+
+static void
+repl_worker_del (const repl_item *item)
+{
+ provider_t *server;
+ const char *s_host;
+ unsigned int s_port;
+ const char *s_key;
+ const char *s_secret;
+ const char *s_type;
+ char svc_acc[SVC_ACC_SIZE];
+ struct hstor_client *hstor;
+ char addr[ADDR_SIZE];
+ CURL *curl;
+ char *bucket;
+ char *key;
+ char *stctx;
+ int chars;
+
+ server = item->server;
+ s_host = server->host;
+ s_port = server->port;
+ s_key = server->username;
+ s_secret = server->password;
+ s_type = server->type;
+
+ if (!strcasecmp(s_type,"s3")) {
+ DPRINTF("%s replicating delete of %s on %s:%u (S3)n",__func__,
+ item->path, s_host, s_port);
+ chars = snprintf(svc_acc,SVC_ACC_SIZE,"%s:%u",s_host,s_port);
+ if (chars >= SVC_ACC_SIZE) {
+ error(0,0,"svc_acc too long in %s",__func__);
+ return;
+ }
+ /* TBD: check return */
+ hstor = hstor_new(svc_acc,s_host,s_key,s_secret);
+ assert (item->path);
+ bucket = strtok_r(item->path,"/",&stctx);
+ key = strtok_r(NULL,"/",&stctx);
+ (void)hstor_del(hstor,bucket,key);
+ hstor_free(hstor);
+ }
+ else {
+ DPRINTF("%s replicating delete of %s on %s:%u (HTTP)n",
+ __func__, item->path, s_host, s_port);
+ chars = snprintf(addr,ADDR_SIZE,"http://%s:%d%s",
+ s_host,s_port,item->path);
+ if (chars >= ADDR_SIZE) {
+ error(0,0,"path too long in %s",__func__);
+ return;
+ }
+ curl = curl_easy_init();
+ curl_easy_setopt(curl,CURLOPT_URL,addr);
+ curl_easy_setopt(curl,CURLOPT_CUSTOMREQUEST,"DELETE");
+ curl_easy_perform(curl);
+ curl_easy_cleanup(curl);
+ }
+
+ DPRINTF("%s returningn",__func__);
+}
+
+static void
+repl_worker_bcreate (repl_item *item)
+{
+ provider_t *server;
+ const char *s_host;
+ unsigned int s_port;
+ const char *s_key;
+ const char *s_secret;
+ const char *s_type;
+ char svc_acc[SVC_ACC_SIZE];
+ struct hstor_client *hstor;
+ char addr[ADDR_SIZE];
+ CURL *curl;
+ int chars;
+
+ server = item->server;
+ s_host = server->host;
+ s_port = server->port;
+ s_key = server->username;
+ s_secret = server->password;
+ s_type = server->type;
+
+ if (!strcasecmp(s_type,"s3")) {
+ DPRINTF("%s replicating create of bucket %s on %s:%u (S3)n",
+ __func__, item->path, s_host, s_port);
+ chars = snprintf(svc_acc,SVC_ACC_SIZE,"%s:%u",s_host,s_port);
+ if (chars >= SVC_ACC_SIZE) {
+ error(0,0,"svc_acc too long in %s",__func__);
+ return;
+ }
+ /* TBD: check return */
+ hstor = hstor_new(svc_acc,s_host,s_key,s_secret);
+ assert (item->path);
+ if (!hstor_add_bucket(hstor,item->path)) {
+ error(0,0,"bucket create failed for %s",
+ item->path);
+ }
+ hstor_free(hstor);
+ }
+ else {
+ DPRINTF("%s replicating create of bucket %s on %s:%u (HTTP)n",
+ __func__, item->path, s_host, s_port);
+ chars = snprintf(addr,ADDR_SIZE,"http://%s:%d/%s",
+ s_host,s_port,item->path);
+ if (chars >= ADDR_SIZE) {
+ error(0,0,"path too long in %s",__func__);
+ return;
+ }
+ curl = curl_easy_init();
+ curl_easy_setopt(curl,CURLOPT_URL,addr);
+ curl_easy_setopt(curl,CURLOPT_CUSTOMREQUEST,"PUT");
+ curl_easy_perform(curl);
+ curl_easy_cleanup(curl);
+ }
+
+ DPRINTF("%s returningn",__func__);
+}
+
+/* Use this to diagnose failed thread creation. */
+#define xpthread_create(thread, start_routine, item, msg) + do { + int err = pthread_create (thread, NULL, start_routine, item); + if (err) { + error (0, err, msg); + return NULL; + } + } while (0)
+
+static void *
+repl_worker (void *notused ATTRIBUTE_UNUSED)
+{
+ repl_item *item;
+ pthread_t cons;
+ pthread_t prod;
+
+ for (;;) {
+ sem_wait(&queue_sema);
+ pthread_mutex_lock(&queue_lock);
+ item = queue_head;
+ queue_head = item->next;
+ if (!queue_head) {
+ queue_tail = NULL;
+ }
+ pthread_mutex_unlock(&queue_lock);
+
+ switch (item->type) {
+ case REPL_PUT:
+ if (pipe(item->pipes) >= 0) {
+ xpthread_create(&prod, (proxy_host
+ ? proxy_repl_prod
+ : proxy_repl_prod_fs),
+ item,
+ "failed to start producer thread");
+ xpthread_create(&cons,proxy_repl_cons,item,
+ "failed to start consumer thread");
+ pthread_join(prod,NULL);
+ pthread_join(cons,NULL);
+ }
+ else {
+ error(0,errno,"pipe");
+ }
+ break;
+ case REPL_ODELETE:
+ repl_worker_del(item);
+ break;
+ case REPL_BCREATE:
+ repl_worker_bcreate(item);
+ break;
+ default:
+ error(0,0,"bad repl type %d (url=%s) skipped",
+ item->type, item->path);
+ }
+ free(item->path);
+ free(item);
+ /* No atomic dec without test? Lame. */
+ (void)g_atomic_int_dec_and_test(&rep_count);
+ }
+}
+
+void
+repl_init (void)
+{
+ pthread_t tid;
+
+ sem_init(&queue_sema,0,0);
+ pthread_mutex_init(&queue_lock,NULL);
+ pthread_create(&tid,NULL,repl_worker,NULL);
+}
+
+static const char *
+repl_oget (void *ctx, const char *id)
+{
+ query_ctx_t *qctx = ctx;
+ char *cur_value = NULL;
+
+ (void)meta_get_value(qctx->cur_bucket,qctx->cur_key,id,&cur_value);
+
+ return cur_value;
+}
+
+static const char *
+repl_sget (void *ctx, const char *id)
+{
+ query_ctx_t *qctx = ctx;
+ provider_t *prov = qctx->cur_server;
+
+ if (!strcmp(id,"name")) {
+ return prov->name;
+ }
+ if (!strcmp(id,"type")) {
+ return prov->type;
+ }
+ if (!strcmp(id,"host")) {
+ return prov->host;
+ }
+ if (!strcmp(id,"key")) {
+ return prov->username;
+ }
+ if (!strcmp(id,"secret")) {
+ return prov->password;
+ }
+ if (!strcmp(id,"path")) {
+ return prov->path;
+ }
+
+ return g_hash_table_lookup(prov->attrs,id);
+}
+
+void
+replicate (const char *url, size_t size, const char *policy)
+{
+ repl_item *item;
+ value_t *expr;
+ int res;
+ char *url2;
+ char *stctx;
+ query_ctx_t qctx;
+ getter_t oget;
+ getter_t sget;
+ GHashTableIter iter;
+ gpointer key;
+ gpointer value;
+ provider_t *prov;
+
+ url2 = strdup(url);
+ if (!url2) {
+ error(0,0,"could not parse url %s",url);
+ return;
+ }
+ qctx.cur_bucket = strtok_r(url2,"/",&stctx);
+ qctx.cur_key = strtok_r(NULL,"/",&stctx);
+
+ if (!size) {
+ size = meta_get_size(qctx.cur_bucket,qctx.cur_key);
+ DPRINTF("fetched size %zu for %sn",size,url);
+ }
+
+ if (policy) {
+ DPRINTF("--- policy = %sn",policy);
+ expr = parse(policy);
+ }
+ else {
+ expr = NULL;
+ }
+
+ oget.func = repl_oget;
+ oget.ctx = &qctx;
+ sget.func = repl_sget;
+ sget.ctx = &qctx;
+
+ init_prov_iter(&iter);
+ while (g_hash_table_iter_next(&iter,&key,&value)) {
+ if (!strcmp(key,me)) {
+ continue;
+ }
+ prov = (provider_t *)value;
+ if (expr) {
+ qctx.cur_server = prov;
+ res = eval(expr,&oget,&sget);
+ }
+ else {
+ res = 0;
+ }
+ if (res <= 0) {
+ DPRINTF("skipping %s for %sn",prov->name,url);
+ continue;
+ }
+ DPRINTF("REPLICATING %s to %sn",url,prov->name);
+ item = malloc(sizeof(*item));
+ if (!item) {
+ error(0,errno,"could not create repl_item for %s",
+ url);
+ break;
+ }
+ item->type = REPL_PUT;
+ item->path = strdup(url);
+ if (!item->path) {
+ error(0,errno,"could not create repl_item for %s",
+ url);
+ break;
+ }
+ item->server = prov;
+ item->size = size;
+ pthread_mutex_lock(&queue_lock);
+ if (queue_tail) {
+ item->next = queue_tail->next;
+ queue_tail->next = item;
+ }
+ else {
+ item->next = NULL;
+ queue_head = item;
+ }
+ queue_tail = item;
+ pthread_mutex_unlock(&queue_lock);
+ g_atomic_int_inc(&rep_count);
+ sem_post(&queue_sema);
+ }
+
+ if (expr) {
+ free_value(expr);
+ }
+ free(url2);
+}
+
+static void
+replicate_namespace_action (const char *name, repl_t action)
+{
+ unsigned int i;
+ repl_item *item;
+ GHashTableIter iter;
+ gpointer key;
+ gpointer value;
+
+ init_prov_iter(&iter);
+ while (g_hash_table_iter_next(&iter,&key,&value)) {
+ if (!strcmp(key,me)) {
+ continue;
+ }
+ DPRINTF("replicating delete(%s) on %un",name,i);
+ item = malloc(sizeof(*item));
+ if (!item) {
+ error(0,errno,"could not create repl_item for %s",
+ name);
+ return;
+ }
+ item->type = action;
+ item->path = strdup(name);
+ if (!item->path) {
+ free(item);
+ return;
+ }
+ item->server = (provider_t *)value;
+ pthread_mutex_lock(&queue_lock);
+ if (queue_tail) {
+ item->next = queue_tail->next;
+ queue_tail->next = item;
+ }
+ else {
+ item->next = NULL;
+ queue_head = item;
+ }
+ queue_tail = item;
+ pthread_mutex_unlock(&queue_lock);
+ g_atomic_int_inc(&rep_count);
+ sem_post(&queue_sema);
+ }
+}
+
+void
+replicate_delete (const char *name)
+{
+ replicate_namespace_action(name,REPL_ODELETE);
+}
+
+void
+replicate_bcreate (const char *name)
+{
+ replicate_namespace_action(name,REPL_BCREATE);
+}
+
+/* Part of our API to the query module. */
+char *
+follow_link (char *object, char *key)
+{
+ (void)object;
+ (void)key;
+
+ return "no_such_object";
+}
+
+int
+get_rep_count (void)
+{
+ return g_atomic_int_get(&rep_count);
+}
diff --git a/replica.h b/replica.h
new file mode 100644
index 0000000..430f4bb
--- /dev/null
+++ b/replica.h
@@ -0,0 +1,26 @@
+/* Copyright (C) 2010 Red Hat, Inc.
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
+
+#if !defined(_REPLICA_H)
+#define _REPLICA_H
+
+void repl_init (void);
+void replicate (const char *url, size_t size,
+ const char *policy);
+void replicate_delete (const char *url);
+void replicate_bcreate (const char *bucket);
+int get_rep_count (void);
+
+#endif
diff --git a/setup.c b/setup.c
new file mode 100644
index 0000000..27146ee
--- /dev/null
+++ b/setup.c
@@ -0,0 +1,449 @@
+/* Copyright (C) 2010 Red Hat, Inc.
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
+
+#include <config.h>
+
+#include <errno.h>
+#include <error.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <poll.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <unistd.h>
+#include <assert.h>
+
+#include <jansson.h>
+
+#include "iwh.h"
+#include "setup.h"
+#include "query.h"
+#include "meta.h"
+
+/*
+ * A config consists of a JSON array of objects, where each object includes:
+ *
+ * name string
+ * type string "s3" or "cf" or "http" (case insensitive)
+ * host string
+ * port integer
+ * key string S3 key or (optional TBD) HTTP user
+ * secret string S3 secret or (optional TBD) HTTP password
+ *
+ * The above fields are all used to implement replication once we've decided to
+ * do it. There may be other fields as well, to help us make that decision.
+ * For example, there might be one or more fields to describe geographic
+ * location, an array of supported image-format names, etc. This information
+ * is deliberately left "schema-free" so that users may add whatever fields
+ * they like to both the config and to replication attributes on objects.
+ */
+
+/* Bitfield for things to check in validate_server */
+#define NEED_NONE 0
+#define NEED_SERVER 0x00000001
+#define NEED_CREDS 0x00000002
+#define NEED_PATH 0x00000004
+#define NEED_ALL ~0
+
+extern backend_func_tbl bad_func_tbl;
+extern backend_func_tbl s3_func_tbl;
+extern backend_func_tbl curl_func_tbl;
+extern backend_func_tbl fs_func_tbl;
+
+static json_t *config = NULL;
+static GHashTable *prov_hash = NULL;
+
+static int
+validate_server (unsigned int i)
+{
+ json_t *server;
+ json_t *elem;
+ const char *name;
+ const char *type;
+ unsigned int needs = NEED_ALL;
+
+ server = json_array_get(config,i);
+ if (!json_is_object(server)) {
+ error(0,0,"config elem %u: missing object",i);
+ return 0;
+ }
+
+ elem = json_object_get(server,"name");
+ if (!json_is_string(elem)) {
+ error(0,0,"config elem %u: missing name",i);
+ return 0;
+ }
+ name = json_string_value(elem);
+
+ elem = json_object_get(server,"type");
+ if (!json_is_string(elem)) {
+ error(0,0,"config elem %u (%s): missing type",i,name);
+ return 0;
+ }
+ type = json_string_value(elem);
+
+ if (!strcasecmp(type,"s3") || !strcasecmp(type,"cf")) {
+ needs = NEED_SERVER | NEED_CREDS;
+ }
+ else if (!strcasecmp(type,"http")) {
+ needs = NEED_SERVER;
+ }
+ else if (!strcasecmp(type,"fs")) {
+ needs = NEED_PATH;
+ }
+ else {
+ error(0,0,"config elem %u (%s): bad type",i,name);
+ return 0;
+ }
+
+ if (needs & NEED_SERVER) {
+ elem = json_object_get(server,"host");
+ if (!json_is_string(elem)) {
+ error(0,0,"config elem %u (%s): missing host",
+ i,name);
+ return 0;
+ }
+ elem = json_object_get(server,"port");
+ if (!json_is_integer(elem)) {
+ error(0,0,"config elem %u (%s): missing port",
+ i,name);
+ return 0;
+ }
+ }
+
+ if (needs & NEED_CREDS) {
+ elem = json_object_get(server,"key");
+ if (!json_is_string(elem)) {
+ error(0,0,"config elem %u (%s): missing key",
+ i, name);
+ return 0;
+ }
+ elem = json_object_get(server,"secret");
+ if (!json_is_string(elem)) {
+ error(0,0, "config elem %u (%s): missing secret",
+ i, name);
+ return 0;
+ }
+ }
+
+ if (needs & NEED_PATH) {
+ elem = json_object_get(server,"path");
+ if (!json_is_string(elem)) {
+ error(0,0,"config elem %u (%s): missing path",
+ i, name);
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+/* We've already validated, so minimal checking here. */
+static const char *
+set_config (provider_t *prov)
+{
+ if (strcasecmp(prov->type,"fs")) {
+ proxy_host = prov->host;
+ proxy_port = prov->port;
+ if (!strcasecmp(prov->type,"s3")) {
+ s3mode = 1;
+ proxy_key = prov->username;
+ proxy_secret = prov->password;
+ }
+ else {
+ s3mode = 0;
+ }
+ }
+ else {
+ local_path = prov->path;
+ }
+
+ return prov->name;
+}
+
+static const char *
+dup_json_string (json_t *obj, char *field)
+{
+ const char *tmp;
+
+ tmp = json_string_value(json_object_get(obj,field));
+ if (tmp) {
+ tmp = strdup(tmp);
+ }
+
+ return tmp;
+}
+
+static int
+is_reserved_attr (const char *name)
+{
+ static const char const *rsvd[] = {
+ "name", "type", "host", "port", "key", "secret", "path",
+ NULL
+ };
+ const char *const *r;
+
+ for (r = rsvd; *r; ++r) {
+ if (!strcasecmp(*r,name)) {
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static int
+convert_provider (int i, provider_t *out)
+{
+ json_t *server;
+ void *iter;
+ const char *key;
+ const char *value;
+
+ server = json_array_get(config,i);
+ if (!server) {
+ DPRINTF("no such entry %dn",i);
+ return 0;
+ }
+
+ out->name = dup_json_string(server,"name");
+ out->type = dup_json_string(server,"type");
+ out->host = dup_json_string(server,"host");
+ out->port = json_integer_value(json_object_get(server,"port"));
+ /* TBD: change key/secret field names to username/password */
+ out->username = dup_json_string(server,"key");
+ out->password = dup_json_string(server,"secret");
+ out->path = dup_json_string(server,"path");
+
+ /* TBD: do this a cleaner way. */
+ if (!strcasecmp(out->type,"s3")) {
+ out->func_tbl = &s3_func_tbl;
+ }
+ else if (!strcasecmp(out->type,"http")) {
+ out->func_tbl = &curl_func_tbl;
+ }
+ else if (!strcasecmp(out->type,"fs")) {
+ out->func_tbl = &fs_func_tbl;
+ }
+ else {
+ out->func_tbl = &bad_func_tbl;
+ }
+
+ out->attrs = g_hash_table_new_full(g_str_hash,g_str_equal,free,free);
+ iter = json_object_iter(server);
+ while (iter) {
+ key = json_object_iter_key(iter);
+ if (!is_reserved_attr(key)) {
+ value = json_string_value(json_object_iter_value(iter));
+ if (value) {
+ value = strdup(value);
+ }
+ if (value) {
+ DPRINTF("%p.%s = %sn",out,key,value);
+ g_hash_table_insert(out->attrs,
+ strdup((char *)key), (char *)value);
+ }
+ else {
+ error(0,0,"could not extract %u.%s",i,key);
+ }
+ }
+ iter = json_object_iter_next(server,iter);
+ }
+
+ out->token = NULL;
+
+ return 1;
+}
+
+const char *
+parse_config_inner (void)
+{
+ unsigned int nservers;
+ unsigned int i;
+ json_t *server;
+ const char *new_key;
+ provider_t *new_prov;
+ const char *primary = NULL;
+
+ if (json_typeof(config) != JSON_ARRAY) {
+ error(0,0,"config should be a JSON array");
+ goto err;
+ }
+
+ nservers = json_array_size(config);
+ if (!nservers) {
+ goto err;
+ }
+
+ for (i = 0; i < nservers; ++i) {
+ if (!validate_server(i)) {
+ goto err;
+ }
+ }
+
+ /* Everything looks OK. */
+ printf("%u replication servers definedn",nservers-1);
+ prov_hash = g_hash_table_new_full(g_str_hash,g_str_equal,free,free);
+ if (!prov_hash) {
+ error(0,0,"could not allocate provider hash");
+ goto err;
+ }
+ for (i = 0; i < nservers; ++i) {
+ server = json_array_get(config,i);
+ if (!server) {
+ error(0,0,"could not get pointer to provider %u",i);
+ goto err_free_hash;
+ }
+ new_key = dup_json_string(server,"name");
+ if (!new_key) {
+ error(0,errno,"could not copy key %u",i);
+ goto err_free_hash;
+ }
+ new_prov = (provider_t *)malloc(sizeof(*new_prov));
+ if (!new_prov) {
+ error(0,errno,"could not allocate provider %u",i);
+ free((char *)new_key);
+ goto err_free_hash;
+ }
+ if (!convert_provider(i,new_prov)) {
+ error(0,0,"could not add provider %u",i);
+ free((char *)new_key);
+ free(new_prov);
+ }
+ g_hash_table_insert(prov_hash,(char *)new_key,new_prov);
+ if (!i) {
+ primary = set_config(new_prov);
+ }
+ }
+ return primary;
+
+err_free_hash:
+ g_hash_table_destroy(prov_hash);
+ prov_hash = NULL;
+err:
+ return 0;
+}
+
+const char *
+parse_config (char *cfg_file)
+{
+ json_error_t err;
+ const char *primary = NULL;
+
+ if (access(cfg_file,R_OK) < 0) {
+ error(0,errno,"failed to open %s for reading", cfg_file);
+ return NULL;
+ }
+
+ config = json_load_file(cfg_file,&err);
+ if (!config) {
+ error(0,0,"JSON error on line %d: %s",err.line,err.text);
+ return NULL;
+ }
+
+ primary = parse_config_inner();
+
+ json_decref(config);
+ config = NULL;
+
+ return primary;
+}
+
+const char *
+auto_config(void)
+{
+ static const char auto_json[] = {
+ "[ {n"
+ " "name": "fs_autostart",n"
+ " "type": "fs",n"
+ " "path": "" AUTO_DIR_FS ""n"
+ "} ]n",
+ };
+ json_error_t err;
+ const char *primary = NULL;
+
+ if (auto_start(db_port) != 0) {
+ return NULL;
+ }
+
+ config = json_loads(auto_json,&err);
+ if (!config) {
+ fprintf(stderr,"JSON error on line %d: %sn",err.line,err.text);
+ return NULL;
+ }
+
+ primary = parse_config_inner();
+ if (primary) {
+ printf("auto-start in effectn");
+ }
+ else {
+ error(0, 0, "invalid autostart configuration (internal error)");
+ }
+
+ json_decref(config);
+ config = NULL;
+
+ return primary;
+}
+
+const provider_t *
+get_provider (const char *name)
+{
+ if (!prov_hash || !name || (*name == '0')) {
+ return NULL;
+ }
+ return g_hash_table_lookup(prov_hash,name);
+}
+
+const char *
+get_provider_value (const provider_t *prov, const char *fname)
+{
+ if (!prov || !fname || (*fname == '0')) {
+ return NULL;
+ }
+ return g_hash_table_lookup(prov->attrs,fname);
+}
+
+void
+init_prov_iter (GHashTableIter *iter)
+{
+ g_hash_table_iter_init(iter,prov_hash);
+}
+
+void
+update_provider (const char *provname, const char *username,
+ const char *password)
+{
+ provider_t *prov;
+
+ DPRINTF("updating %s username=%s password=%sn",
+ provname, username, password);
+
+ prov = (provider_t *)get_provider(provname);
+ if (!prov) {
+ DPRINTF(" could not find provider %sn",provname);
+ return; /* TBD: return actual HTTP status for user */
+ }
+
+ free((char *)prov->username);
+ prov->username = strdup(username);
+ free((char *)prov->password);
+ prov->password = strdup(password);
+}
diff --git a/setup.h b/setup.h
new file mode 100644
index 0000000..0056dbc
--- /dev/null
+++ b/setup.h
@@ -0,0 +1,52 @@
+/* Copyright (C) 2010 Red Hat, Inc.
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
+
+#if !defined(_SETUP_H)
+#define _SETUP_H
+
+#include <glib.h>
+#include <curl/curl.h> /* needed by stuff in state_defs.h (from backend.h) */
+#include <microhttpd.h> /* ditto */
+#include "backend.h"
+
+typedef struct _provider {
+ const char *name;
+ const char *type;
+ const char *host;
+ int port;
+ const char *username;
+ const char *password;
+ const char *path;
+ backend_func_tbl *func_tbl;
+ GHashTable *attrs;
+ char *token;
+} provider_t;
+
+const char *parse_config (char *);
+const provider_t *get_provider (const char *name);
+void update_provider (const char *provname,
+ const char *username,
+ const char *password);
+const char *get_provider_value (const provider_t *prov,
+ const char *fname);
+void init_prov_iter (GHashTableIter *iter);
+
+const char *auto_config (void);
+
+/* These are actually in auto.c, but it doesn't have its own header file. */
+int auto_start (int dbport);
+void auto_stop (void);
+
+#endif
-----------------------------------------------------------------------
Summary of changes:
replica.c | 827 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
replica.h | 26 ++
setup.c | 449 +++++++++++++++++++++++++++++++++
setup.h | 52 ++++
4 files changed, 1354 insertions(+), 0 deletions(-)
create mode 100644 replica.c
create mode 100644 replica.h
create mode 100644 setup.c
create mode 100644 setup.h
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months
[repo.or.cz] iwhd.git branch master updated: v0.0-163-g0f82ac4
by Jeff Darcy
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project iwhd.git.
The branch, master has been updated
via 0f82ac4e7c2cfbfe18a58f89b5543ca8e64290ba (commit)
from c63dfdc7d21bc0bd221777621a7649552a8aa40f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://repo.or.cz/w/iwhd.git/commit/0f82ac4e7c2cfbfe18a58f89b5543ca8e64290ba
commit 0f82ac4e7c2cfbfe18a58f89b5543ca8e64290ba
Author: Jeff Darcy <jdarcy(a)redhat.com>
Date: Wed Oct 13 10:16:03 2010 -0400
Dynamic config, steps 1-3
(rollup of previous patches to keep commits buildable)
Generate provider_t structures at init instead of on demand
Change non-setup code to use provider_t instead of JSON calls
get_provider in rest.c (provider list/update)
get_provider_value in backend.c (EC2 bundling)
get_provider_value in query.leg (policy eval)
many in proxy.c (replication)
Split proxy.c into setup.c and replica.c
Propagate const changes where needed
Reconcile with autostart changes
diff --git a/Makefile.am b/Makefile.am
index ccf5afc..ac88336 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -35,11 +35,13 @@ iwhd_SOURCES = meta.h mpipe.c mpipe.h - proxy.c - proxy.h + replica.c + replica.h query.c query.h rest.c + setup.c + setup.h state_defs.h template.c template.h
diff --git a/auto.c b/auto.c
index b4ff5e9..a0e0b21 100644
--- a/auto.c
+++ b/auto.c
@@ -16,11 +16,9 @@
#include <stdarg.h> /* for microhttpd.h (bug in there) */
#include <stdint.h> /* for microhttpd.h (bug in there) */
-#include <microhttpd.h> /* for proxy.h */
-#include <curl/curl.h> /* for proxy.h */
-#include <glib.h> /* for pfoxy.h */
+#include <jansson.h>
-#include "proxy.h"
+#include "setup.h"
#include "iwh.h"
static char auto_arg_port[10];
@@ -146,8 +144,9 @@ struct server_node {
} a;
};
-static int node_resolve(struct server_node *sn,
- const char *hostname, const char *portstr)
+static int
+node_resolve(struct server_node *sn,
+ const char *hostname, const char *portstr)
{
struct addrinfo hints;
struct addrinfo *res, *res0;
@@ -182,7 +181,8 @@ static int node_resolve(struct server_node *sn,
return -1;
}
-static int auto_test_mongod(void)
+static int
+auto_test_mongod(void)
{
struct server_node snode, *sn = &snode;
int sfd;
diff --git a/backend.c b/backend.c
index 1ad376a..b3551ca 100644
--- a/backend.c
+++ b/backend.c
@@ -41,7 +41,7 @@
#define GLOBALS_IMPL
#include "iwh.h"
#include "meta.h"
-#include "proxy.h"
+#include "setup.h"
#include "template.h"
#include "mpipe.h"
#include "backend.h"
@@ -86,10 +86,10 @@ bad_init (void)
void *
bad_get_child (void * ctx)
{
- my_state *ms = (my_state *)ctx;
+ my_state *ms = (my_state *)ctx;
DPRINTF("*** bad call to %sn",__func__);
- pipe_prod_siginit(&ms->pipe,-1);
+ pipe_prod_siginit(&ms->pipe,-1);
return NULL;
}
@@ -224,8 +224,8 @@ s3_get_child (void * ctx)
{
my_state *ms = ctx;
- /* TBD: check existence before calling siginit */
- pipe_prod_siginit(&ms->pipe,0);
+ /* TBD: check existence before calling siginit */
+ pipe_prod_siginit(&ms->pipe,0);
hstor_get(hstor,ms->bucket,ms->key,http_get_prod,&ms->pipe,0);
/* TBD: check return value */
@@ -294,7 +294,7 @@ s3_bcreate (const char *bucket)
return MHD_HTTP_OK;
}
-char *
+static const char *
s3_init_tmpfile (char *value)
{
char *path;
@@ -344,9 +344,9 @@ s3_register (my_state *ms, const provider_t *prov, const char *next,
char *ramdisk = g_hash_table_lookup(args,"ramdisk");
char *api_key;
char *api_secret;
- char *ami_cert;
- char *ami_key;
- char *ami_uid;
+ const char *ami_cert;
+ const char *ami_key;
+ const char *ami_uid;
const char *argv[12];
int argc = 0;
pid_t pid;
@@ -404,7 +404,7 @@ s3_register (my_state *ms, const provider_t *prov, const char *next,
}
}
else {
- ami_cert = get_provider_value(prov->index,"ami-cert");
+ ami_cert = get_provider_value(prov,"ami-cert");
if (!ami_cert) {
error (0, 0, "missing EC2 AMI cert");
goto cleanup;
@@ -419,7 +419,7 @@ s3_register (my_state *ms, const provider_t *prov, const char *next,
}
}
else {
- ami_key = get_provider_value(prov->index,"ami-key");
+ ami_key = get_provider_value(prov,"ami-key");
if (!ami_key) {
error (0, 0, "missing EC2 AMI key");
goto cleanup;
@@ -428,7 +428,7 @@ s3_register (my_state *ms, const provider_t *prov, const char *next,
ami_uid = g_hash_table_lookup(args,"ami-uid");
if (!ami_uid) {
- ami_uid = get_provider_value(prov->index,"ami-uid");
+ ami_uid = get_provider_value(prov,"ami-uid");
if (!ami_uid) {
error (0, 0, "missing EC2 AMI uid");
goto cleanup;
@@ -538,12 +538,12 @@ cleanup:
*/
if (cval && ami_cert) {
unlink(ami_cert);
- free(ami_cert);
+ free((char *)ami_cert);
}
/* Same reasoning as above, with kval/ami_key. */
if (kval && ami_key) {
unlink(ami_key);
- free(ami_key);
+ free((char *)ami_key);
}
(void)meta_set_value(ms->bucket,ms->key,"ami-id",ami_id_buf);
@@ -566,7 +566,7 @@ curl_get_child (void * ctx)
ms->curl = curl_easy_init();
if (!ms->curl) {
- pipe_prod_siginit(&ms->pipe,-1);
+ pipe_prod_siginit(&ms->pipe,-1);
return NULL; /* TBD: flag error somehow */
}
ms->cleanup |= CLEANUP_CURL;
@@ -582,7 +582,7 @@ curl_get_child (void * ctx)
curl_easy_setopt(ms->curl,CURLOPT_WRITEFUNCTION,
http_get_prod);
curl_easy_setopt(ms->curl,CURLOPT_WRITEDATA,&ms->pipe);
- pipe_prod_siginit(&ms->pipe,0);
+ pipe_prod_siginit(&ms->pipe,0);
curl_easy_perform(ms->curl);
curl_easy_getinfo(ms->curl,CURLINFO_RESPONSE_CODE,&ms->rc);
@@ -796,12 +796,12 @@ fs_get_child (void * ctx)
fd = open(file, O_RDONLY);
if (fd < 0) {
- pipe_prod_siginit(&ms->pipe,-1);
- pipe_prod_finish(&ms->pipe);
+ pipe_prod_siginit(&ms->pipe,-1);
+ pipe_prod_finish(&ms->pipe);
return THREAD_FAILED;
}
- pipe_prod_siginit(&ms->pipe,0);
+ pipe_prod_siginit(&ms->pipe,0);
for (;;) {
bytes = read(fd,buf,sizeof(buf));
diff --git a/iwh.h b/iwh.h
index f3c091e..89d38b2 100644
--- a/iwh.h
+++ b/iwh.h
@@ -38,7 +38,6 @@
*/
GLOBAL(int, verbose, 0);
-GLOBAL(char *, cfg_file, NULL);
GLOBAL(const char *, proxy_host, NULL); /* repod/S3 */
GLOBAL(unsigned short, proxy_port, MY_PORT+1); /* repod/S3 */
GLOBAL(const char *, proxy_key, "foo"); /* S3 only */
@@ -49,7 +48,7 @@ GLOBAL(unsigned int, s3mode, 0); /* repod/S3 */
GLOBAL(const char *, local_path, "/tmp"); /* FS */
GLOBAL(const char *, db_host, "localhost");
GLOBAL(unsigned short, db_port, 0);
-GLOBAL(char *, me, "here");
+GLOBAL(const char *, me, "here");
#define DPRINTF(fmt,args...) do { if (verbose) { diff --git a/meta.cpp b/meta.cpp
index 550e4e4..13abfb2 100644
--- a/meta.cpp
+++ b/meta.cpp
@@ -455,7 +455,7 @@ meta_query_stop (void * qobj)
CLIENT_UNLOCK;
}
-extern "C" char *
+extern "C" const char *
query_getter (void *ctx, const char *id)
{
BSONObj *cur_bo = (BSONObj *)ctx;
diff --git a/proxy.c b/proxy.c
deleted file mode 100644
index 9e50f7c..0000000
--- a/proxy.c
+++ /dev/null
@@ -1,1122 +0,0 @@
-/* Copyright (C) 2010 Red Hat, Inc.
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>. */
-
-#include <config.h>
-
-#include <errno.h>
-#include <error.h>
-#include <fcntl.h>
-#include <getopt.h>
-#include <poll.h>
-#include <pthread.h>
-#include <semaphore.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <strings.h>
-#include <unistd.h>
-#include <assert.h>
-
-#include <microhttpd.h>
-#include <curl/curl.h>
-#include <hstor.h>
-#include <jansson.h>
-
-#include "iwh.h"
-#include "proxy.h"
-#include "query.h"
-#include "meta.h"
-
-/*
- * A config consists of a JSON array of objects, where each object includes:
- *
- * name string
- * type string "s3" or "cf" or "http" (case insensitive)
- * host string
- * port integer
- * key string S3 key or (optional TBD) HTTP user
- * secret string S3 secret or (optional TBD) HTTP password
- *
- * The above fields are all used to implement replication once we've decided to
- * do it. There may be other fields as well, to help us make that decision.
- * For example, there might be one or more fields to describe geographic
- * location, an array of supported image-format names, etc. This information
- * is deliberately left "schema-free" so that users may add whatever fields
- * they like to both the config and to replication attributes on objects.
- */
-
-/* Sizes for internal string buffers. */
-#define ADDR_SIZE 1024
-#define SVC_ACC_SIZE 128
-#define HEADER_SIZE 64
-
-/* Bitfield for things to check in validate_server */
-#define NEED_NONE 0
-#define NEED_SERVER 0x00000001
-#define NEED_CREDS 0x00000002
-#define NEED_PATH 0x00000004
-#define NEED_ALL ~0
-
-extern backend_func_tbl bad_func_tbl;
-extern backend_func_tbl s3_func_tbl;
-extern backend_func_tbl curl_func_tbl;
-extern backend_func_tbl fs_func_tbl;
-
-typedef enum {
- REPL_PUT, /* store an object */
- REPL_ODELETE, /* delete an object */
- REPL_BCREATE, /* create a bucket */
- /* TBD: bucket deletion, others? */
-} repl_t;
-
-typedef struct _repl_item {
- struct _repl_item *next;
- repl_t type;
- char *path;
- unsigned int server;
- size_t size;
- int pipes[2];
-} repl_item;
-
-typedef struct {
- char *cur_bucket;
- char *cur_key;
- json_t *cur_server;
-} query_ctx_t;
-
-static repl_item *queue_head = NULL;
-static repl_item *queue_tail = NULL;
-static pthread_mutex_t queue_lock;
-static sem_t queue_sema;
-static json_t *config = NULL;
-static int rep_count = 0;
-
-static int
-validate_server (unsigned int i)
-{
- json_t *server;
- json_t *elem;
- const char *name;
- const char *type;
- unsigned int needs = NEED_ALL;
-
- server = json_array_get(config,i);
- if (!json_is_object(server)) {
- error(0,0,"config elem %u: missing object",i);
- return 0;
- }
-
- elem = json_object_get(server,"name");
- if (!json_is_string(elem)) {
- error(0,0,"config elem %u: missing name",i);
- return 0;
- }
- name = json_string_value(elem);
-
- elem = json_object_get(server,"type");
- if (!json_is_string(elem)) {
- error(0,0,"config elem %u (%s): missing type",i,name);
- return 0;
- }
- type = json_string_value(elem);
-
- if (!strcasecmp(type,"s3") || !strcasecmp(type,"cf")) {
- needs = NEED_SERVER | NEED_CREDS;
- }
- else if (!strcasecmp(type,"http")) {
- needs = NEED_SERVER;
- }
- else if (!strcasecmp(type,"fs")) {
- needs = NEED_PATH;
- }
- else {
- error(0,0,"config elem %u (%s): bad type",i,name);
- return 0;
- }
-
- if (needs & NEED_SERVER) {
- elem = json_object_get(server,"host");
- if (!json_is_string(elem)) {
- error(0,0,"config elem %u (%s): missing host",
- i,name);
- return 0;
- }
- elem = json_object_get(server,"port");
- if (!json_is_integer(elem)) {
- error(0,0,"config elem %u (%s): missing port",
- i,name);
- return 0;
- }
- }
-
- if (needs & NEED_CREDS) {
- elem = json_object_get(server,"key");
- if (!json_is_string(elem)) {
- error(0,0,"config elem %u (%s): missing key",
- i, name);
- return 0;
- }
- elem = json_object_get(server,"secret");
- if (!json_is_string(elem)) {
- error(0,0, "config elem %u (%s): missing secret",
- i, name);
- return 0;
- }
- }
-
- if (needs & NEED_PATH) {
- elem = json_object_get(server,"path");
- if (!json_is_string(elem)) {
- error(0,0,"config elem %u (%s): missing path",
- i, name);
- return 0;
- }
- }
-
- return 1;
-}
-
-/* We've already validated, so minimal checking here. */
-char *
-set_config (void)
-{
- json_t *server;
- const char *type;
-
- server = json_array_get(config,0);
- type = json_string_value(json_object_get(server,"type"));
- if (strcasecmp(type,"fs")) {
- proxy_host = json_string_value(
- json_object_get(server,"host"));
- proxy_port = json_integer_value(
- json_object_get(server,"port"));
- if (!strcasecmp(type,"s3")) {
- s3mode = 1;
- proxy_key = json_string_value(
- json_object_get(server,"key"));
- proxy_secret = json_string_value(
- json_object_get(server,"secret"));
- }
- else {
- s3mode = 0;
- }
- }
- else {
- local_path = json_string_value(
- json_object_get(server,"path"));
- }
-
- return (char *)json_string_value(json_object_get(server,"name"));
-}
-
-char *
-parse_config (void)
-{
- json_error_t err;
- unsigned int nservers;
- unsigned int i;
-
- if (access(cfg_file,R_OK) < 0) {
- error(0,errno,"failed to open %s for reading", cfg_file);
- return NULL;
- }
-
- config = json_load_file(cfg_file,&err);
- if (!config) {
- error(0,0,"JSON error on line %d: %s",err.line,err.text);
- return NULL;
- }
-
- if (json_typeof(config) != JSON_ARRAY) {
- error(0,0,"config should be a JSON array");
- goto err;
- }
-
- nservers = json_array_size(config);
- if (!nservers) {
- goto err;
- }
-
- for (i = 0; i < nservers; ++i) {
- if (!validate_server(i)) {
- goto err;
- }
- }
-
- /* Everything looks OK. */
- printf("%u replication servers definedn",nservers-1);
- return set_config();
-
-err:
- json_decref(config);
- config = NULL;
- return 0;
-}
-
-char *
-auto_config(void)
-{
- static const char auto_json[] = {
- "[ {n"
- " "name": "fs_autostart",n"
- " "type": "fs",n"
- " "path": "" AUTO_DIR_FS ""n"
- "} ]n",
- };
- json_error_t err;
-
- if (auto_start(db_port) != 0) {
- return NULL;
- }
-
- config = json_loads(auto_json,&err);
- if (!config) {
- fprintf(stderr,"JSON error on line %d: %sn",err.line,err.text);
- return NULL;
- }
-
- if (json_typeof(config) != JSON_ARRAY) {
- goto out_ie;
- }
-
- if (json_array_size(config) != 1) {
- goto out_ie;
- }
-
- if (!validate_server(0)) {
- goto out_ie;
- }
-
- printf("no replication servers defined, auto-start in effectn");
- return set_config();
-
-out_ie:
- error(0, 0, "invalid autostart configuration (internal error)");
- return NULL;
-}
-
-static size_t
-junk_writer (/* const */ void *ptr, size_t size, size_t nmemb, void *stream)
-{
- size_t n;
-
- n = fwrite(ptr,size,nmemb,stream);
- if (n != nmemb)
- error(0, 0, "warning: write failed");
- if (fflush(stream))
- error(0, 0, "warning: write failed");
- DPRINTF("in %s(%zu,%zu) => %zun",__func__,size,nmemb,n);
-
- return n;
-}
-
-static void *
-proxy_repl_prod_fs (void *ctx)
-{
- repl_item *item = ctx;
- int ifd;
- int ofd;
- char buf[1<<16];
- ssize_t ibytes;
- ssize_t obytes;
- ssize_t offset;
-
- DPRINTF("replicating from %s (FS)n",item->path);
-
- ifd = open(item->path,O_RDONLY);
- if (ifd < 0) {
- error(0,errno,"ifd open");
- return THREAD_FAILED;
- }
- ofd = item->pipes[1];
-
- for (;;) {
- ibytes = read(ifd,buf,sizeof(buf));
- if (ibytes <= 0) {
- if (ibytes < 0) {
- error(0,errno,"%s: read failed", item->path);
- }
- else {
- DPRINTF("EOF on ifdn");
- }
- break;
- }
- offset = 0;
- do {
- obytes = write(ofd,buf+offset,ibytes);
- if (obytes <= 0) {
- if (obytes < 0) {
- error(0,errno,"ofd write");
- }
- else {
- DPRINTF("zero-length write on ofdn");
- }
- break;
- }
- ibytes -= obytes;
- offset += obytes;
- } while (ibytes > 0);
- }
-
- close(ifd);
- close(ofd);
-
- DPRINTF("%s returningn",__func__);
- close(item->pipes[1]);
- return NULL;
-}
-
-static void *
-proxy_repl_prod (void *ctx)
-{
- repl_item *item = ctx;
- FILE *fp = fdopen(item->pipes[1],"w");
- char addr[ADDR_SIZE];
- CURL *curl;
- char svc_acc[SVC_ACC_SIZE];
- struct hstor_client *hstor;
- char *bucket;
- char *key;
- char *stctx;
- char *myurl;
- int chars;
-
- if (fp == NULL) {
- error(0, errno, "%s: fdopen failed", __func__);
- return NULL;
- }
-
- chars = snprintf(addr,ADDR_SIZE,
- "http://%s:%u/%s",proxy_host,proxy_port,item->path);
- if (chars >= ADDR_SIZE) {
- error(0,0,"path too long in %s",__func__);
- goto done;
- }
- DPRINTF("replicating from %sn",addr);
-
- if (s3mode) {
- chars = snprintf(svc_acc,SVC_ACC_SIZE,"%s:%u",
- proxy_host,proxy_port);
- if (chars >= SVC_ACC_SIZE) {
- error(0,0,"svc_acc too long in %s",__func__);
- goto done;
- }
- hstor = hstor_new(svc_acc,proxy_host,
- proxy_key,proxy_secret);
- /* Blech. Can't conflict with consumer, though. */
- myurl = strdup(item->path);
- assert (myurl);
- bucket = strtok_r(myurl,"/",&stctx);
- key = strtok_r(NULL,"/",&stctx);
- hstor_get(hstor,bucket,key,
- junk_writer,fp,0);
- hstor_free(hstor);
- free(myurl);
- }
- else {
- curl = curl_easy_init();
- curl_easy_setopt(curl,CURLOPT_URL,addr);
- curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION,junk_writer);
- curl_easy_setopt(curl,CURLOPT_WRITEDATA,fp);
- DPRINTF("%s calling curl_easy_performn",__func__);
- curl_easy_perform(curl);
- curl_easy_cleanup(curl);
- }
-
-done:
- DPRINTF("%s returningn",__func__);
- /* Closing should signal to the consumer that we're finished. */
- fclose(fp);
- return NULL;
-}
-
-static size_t
-junk_reader (void *ptr, size_t size, size_t nmemb, void *stream)
-{
- size_t n;
-
- n = fread(ptr,size,nmemb,stream);
- printf("in %s(%zu,%zu) => %zun",__func__,size,nmemb,n);
- return n;
-}
-
-static size_t
-cf_writer (void *ptr ATTRIBUTE_UNUSED, size_t size, size_t nmemb,
- void *stream ATTRIBUTE_UNUSED)
-{
- return size * nmemb;
-}
-
-static size_t
-cf_header (void *ptr, size_t size, size_t nmemb, void *stream)
-{
- char *next;
- char *sctx;
- json_t *server = (json_t *)stream;
-
- next = strtok_r(ptr,":",&sctx);
- if (next) {
- if (!strcasecmp(next,"X-Storage-Url")) {
- next = strtok_r(NULL," nr",&sctx);
- if (next) {
- DPRINTF("got CF URL %sn",next);
- /* NB: after this, original "host" is gone. */
- json_object_set_new(server,"host",
- json_string(next));
- }
- }
- else if (!strcasecmp(next,"X-Storage-Token")) {
- next = strtok_r(NULL," nr",&sctx);
- if (next) {
- DPRINTF("got CF token %sn",next);
- json_object_set_new(server,"token",
- json_string(next));
- }
- }
- }
- return size * nmemb;
-}
-
-static const char *
-get_cloudfiles_token (json_t *server, const char *host, unsigned int port,
- const char * user, const char * key)
-{
- CURL *curl;
- char addr[ADDR_SIZE];
- char auth_user[HEADER_SIZE];
- char auth_key[HEADER_SIZE];
- json_t *token_obj;
- struct curl_slist *slist;
- int chars;
-
- token_obj = json_object_get(server,"token");
- if (token_obj) {
- return json_string_value(token_obj);
- }
-
- chars = snprintf(addr,ADDR_SIZE,"https://%s:%u/v1.0",host,port);
- if (chars >= ADDR_SIZE) {
- error(0,0,"API URL too long in %s",__func__);
- return NULL;
- }
-
- chars = snprintf(auth_user,HEADER_SIZE,"X-Auth-User: %s",user);
- if (chars >= HEADER_SIZE) {
- error(0,0,"auth_user too long in %s",__func__);
- return NULL;
- }
-
- chars = snprintf(auth_key,HEADER_SIZE,"X-Auth-Key: %s",key);
- if (chars >= HEADER_SIZE) {
- error(0,0,"auth_key too long in %s",__func__);
- return NULL;
- }
-
- curl = curl_easy_init();
- curl_easy_setopt(curl,CURLOPT_URL,addr);
- curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION,cf_writer);
- curl_easy_setopt(curl,CURLOPT_HEADERFUNCTION,cf_header);
- curl_easy_setopt(curl,CURLOPT_WRITEHEADER,server);
- slist = curl_slist_append(NULL,auth_user);
- slist = curl_slist_append(slist,auth_key);
- curl_easy_setopt(curl,CURLOPT_HTTPHEADER,slist);
- curl_easy_perform(curl);
- curl_easy_cleanup(curl);
- curl_slist_free_all(slist);
-
- token_obj = json_object_get(server,"token");
- return token_obj ? json_string_value(token_obj) : NULL;
-}
-
-static void *
-proxy_repl_cons (void *ctx)
-{
- repl_item *item = ctx;
- FILE *fp = fdopen(item->pipes[0],"r");
- char addr[ADDR_SIZE];
- CURL *curl;
- json_t *server;
- char svc_acc[SVC_ACC_SIZE];
- char auth_hdr[HEADER_SIZE];
- struct hstor_client *hstor;
- char *bucket;
- char *key;
- char *stctx;
- const char *s_host;
- unsigned int s_port;
- const char *s_key;
- const char *s_secret;
- const char *s_type;
- const char *s_name;
- struct curl_slist *slist;
- char *myurl;
- int chars;
-
- if (fp == NULL) {
- error(0, errno, "%s: fdopen failed", __func__);
- return THREAD_FAILED;
- }
-
- server = json_array_get(config,item->server);
- s_host = json_string_value(json_object_get(server,"host"));
- s_port = json_integer_value(json_object_get(server,"port"));
- s_key = json_string_value(json_object_get(server,"key"));
- s_secret = json_string_value(json_object_get(server,"secret"));
- s_type = json_string_value(json_object_get(server,"type"));
- s_name = json_string_value(json_object_get(server,"name"));
-
- myurl = strdup(item->path);
- assert (myurl);
- bucket = strtok_r(myurl,"/",&stctx);
- key = strtok_r(NULL,"/",&stctx);
-
- if (!strcasecmp(s_type,"s3")) {
- DPRINTF("replicating %zu to %s/%s (S3)n",item->size,s_host,
- item->path);
- chars = snprintf(svc_acc,SVC_ACC_SIZE,"%s:%u",s_host,s_port);
- if (chars >= SVC_ACC_SIZE) {
- error(0,0,"svc_acc too long in %s",__func__);
- return THREAD_FAILED;
- }
- hstor = hstor_new(svc_acc,s_host,s_key,s_secret);
- /* Blech. Can't conflict with producer, though. */
- hstor_put(hstor,bucket,key,
- junk_reader,item->size,fp,NULL);
- hstor_free(hstor);
- }
- else {
- const char *token_str = NULL;
- if (!strcasecmp(s_type,"cf")) {
- token_str = get_cloudfiles_token(server,s_host,s_port,
- s_key, s_secret);
- if (!token_str) {
- DPRINTF("could not get CF tokenn");
- return THREAD_FAILED;
- }
- /* Re-fetch as this might have changed. */
- s_host = json_string_value(json_object_get(server,
- "host"));
- chars = snprintf(addr,ADDR_SIZE,"%s/%s",
- s_host,item->path);
- if (chars >= ADDR_SIZE) {
- error(0,0,"CF path too long in %s",__func__);
- return THREAD_FAILED;
- }
- DPRINTF("replicating %zu to %s (CF)n",item->size,
- addr);
- }
- else {
- chars = snprintf(addr,ADDR_SIZE,"http://%s:%u/%s",
- s_host,s_port,item->path);
- if (chars >= ADDR_SIZE) {
- error(0,0,"HTTP path too long in %s",
- __func__);
- return THREAD_FAILED;
- }
- DPRINTF("replicating %zu to %s (repod)n",item->size,
- addr);
- }
- curl = curl_easy_init();
- curl_easy_setopt(curl,CURLOPT_URL,addr);
- curl_easy_setopt(curl,CURLOPT_UPLOAD,1);
- curl_easy_setopt(curl,CURLOPT_INFILESIZE_LARGE,
- (curl_off_t)item->size);
- curl_easy_setopt(curl,CURLOPT_READFUNCTION,junk_reader);
- if (!strcasecmp(s_type,"cf")) {
- chars = snprintf(auth_hdr,HEADER_SIZE,
- "X-Auth-Token: %s",token_str);
- if (chars >= HEADER_SIZE) {
- error(0,0,"auth_token too long in %s",
- __func__);
- return THREAD_FAILED;
- }
- slist = curl_slist_append(NULL,auth_hdr);
- /*
- * Rackspace doesn't clearly document that you'll get
- * 412 (Precondition Failed) if you omit this.
- */
- slist = curl_slist_append(slist,
- "Content-Type: binary/octet-stream");
- }
- else {
- slist = curl_slist_append(NULL,"X-redhat-role: master");
- }
- curl_easy_setopt(curl,CURLOPT_HTTPHEADER,slist);
- curl_easy_setopt(curl,CURLOPT_READDATA,fp);
- DPRINTF("%s calling curl_easy_performn",__func__);
- curl_easy_perform(curl);
- curl_easy_cleanup(curl);
- curl_slist_free_all(slist);
- }
-
- DPRINTF("%s returningn",__func__);
- fclose(fp);
- meta_got_copy(bucket,key,s_name);
- free(myurl);
- return NULL;
-}
-
-static void
-repl_worker_del (const repl_item *item)
-{
- json_t *server;
- const char *s_host;
- unsigned int s_port;
- const char *s_key;
- const char *s_secret;
- const char *s_type;
- char svc_acc[SVC_ACC_SIZE];
- struct hstor_client *hstor;
- char addr[ADDR_SIZE];
- CURL *curl;
- char *bucket;
- char *key;
- char *stctx;
- int chars;
-
- server = json_array_get(config,item->server);
- s_host = json_string_value(json_object_get(server,"host"));
- s_port = json_integer_value(json_object_get(server,"port"));
- s_key = json_string_value(json_object_get(server,"key"));
- s_secret = json_string_value(json_object_get(server,"secret"));
- s_type = json_string_value(json_object_get(server,"type"));
-
- if (!strcasecmp(s_type,"s3")) {
- DPRINTF("%s replicating delete of %s on %s:%u (S3)n",__func__,
- item->path, s_host, s_port);
- chars = snprintf(svc_acc,SVC_ACC_SIZE,"%s:%u",s_host,s_port);
- if (chars >= SVC_ACC_SIZE) {
- error(0,0,"svc_acc too long in %s",__func__);
- return;
- }
- /* TBD: check return */
- hstor = hstor_new(svc_acc,s_host,s_key,s_secret);
- assert (item->path);
- bucket = strtok_r(item->path,"/",&stctx);
- key = strtok_r(NULL,"/",&stctx);
- (void)hstor_del(hstor,bucket,key);
- hstor_free(hstor);
- }
- else {
- DPRINTF("%s replicating delete of %s on %s:%u (HTTP)n",
- __func__, item->path, s_host, s_port);
- chars = snprintf(addr,ADDR_SIZE,"http://%s:%d%s",
- s_host,s_port,item->path);
- if (chars >= ADDR_SIZE) {
- error(0,0,"path too long in %s",__func__);
- return;
- }
- curl = curl_easy_init();
- curl_easy_setopt(curl,CURLOPT_URL,addr);
- curl_easy_setopt(curl,CURLOPT_CUSTOMREQUEST,"DELETE");
- curl_easy_perform(curl);
- curl_easy_cleanup(curl);
- }
-
- DPRINTF("%s returningn",__func__);
-}
-
-static void
-repl_worker_bcreate (repl_item *item)
-{
- json_t *server;
- const char *s_host;
- unsigned int s_port;
- const char *s_key;
- const char *s_secret;
- const char *s_type;
- char svc_acc[SVC_ACC_SIZE];
- struct hstor_client *hstor;
- char addr[ADDR_SIZE];
- CURL *curl;
- int chars;
-
- server = json_array_get(config,item->server);
- s_host = json_string_value(json_object_get(server,"host"));
- s_port = json_integer_value(json_object_get(server,"port"));
- s_key = json_string_value(json_object_get(server,"key"));
- s_secret = json_string_value(json_object_get(server,"secret"));
- s_type = json_string_value(json_object_get(server,"type"));
-
- if (!strcasecmp(s_type,"s3")) {
- DPRINTF("%s replicating create of bucket %s on %s:%u (S3)n",
- __func__, item->path, s_host, s_port);
- chars = snprintf(svc_acc,SVC_ACC_SIZE,"%s:%u",s_host,s_port);
- if (chars >= SVC_ACC_SIZE) {
- error(0,0,"svc_acc too long in %s",__func__);
- return;
- }
- /* TBD: check return */
- hstor = hstor_new(svc_acc,s_host,s_key,s_secret);
- assert (item->path);
- if (!hstor_add_bucket(hstor,item->path)) {
- error(0,0,"bucket create failed for %s",
- item->path);
- }
- hstor_free(hstor);
- }
- else {
- DPRINTF("%s replicating create of bucket %s on %s:%u (HTTP)n",
- __func__, item->path, s_host, s_port);
- chars = snprintf(addr,ADDR_SIZE,"http://%s:%d/%s",
- s_host,s_port,item->path);
- if (chars >= ADDR_SIZE) {
- error(0,0,"path too long in %s",__func__);
- return;
- }
- curl = curl_easy_init();
- curl_easy_setopt(curl,CURLOPT_URL,addr);
- curl_easy_setopt(curl,CURLOPT_CUSTOMREQUEST,"PUT");
- curl_easy_perform(curl);
- curl_easy_cleanup(curl);
- }
-
- DPRINTF("%s returningn",__func__);
-}
-
-/* Use this to diagnose failed thread creation. */
-#define xpthread_create(thread, start_routine, item, msg) - do { - int err = pthread_create (thread, NULL, start_routine, item); - if (err) { - error (0, err, msg); - return NULL; - } - } while (0)
-
-static void *
-repl_worker (void *notused ATTRIBUTE_UNUSED)
-{
- repl_item *item;
- pthread_t cons;
- pthread_t prod;
-
- for (;;) {
- sem_wait(&queue_sema);
- pthread_mutex_lock(&queue_lock);
- item = queue_head;
- queue_head = item->next;
- if (!queue_head) {
- queue_tail = NULL;
- }
- pthread_mutex_unlock(&queue_lock);
-
- switch (item->type) {
- case REPL_PUT:
- if (pipe(item->pipes) >= 0) {
- xpthread_create(&prod, (proxy_host
- ? proxy_repl_prod
- : proxy_repl_prod_fs),
- item,
- "failed to start producer thread");
- xpthread_create(&cons,proxy_repl_cons,item,
- "failed to start consumer thread");
- pthread_join(prod,NULL);
- pthread_join(cons,NULL);
- }
- else {
- error(0,errno,"pipe");
- }
- break;
- case REPL_ODELETE:
- repl_worker_del(item);
- break;
- case REPL_BCREATE:
- repl_worker_bcreate(item);
- break;
- default:
- error(0,0,"bad repl type %d (url=%s) skipped",
- item->type, item->path);
- }
- free(item->path);
- free(item);
- /* No atomic dec without test? Lame. */
- (void)g_atomic_int_dec_and_test(&rep_count);
- }
-}
-
-void
-repl_init (void)
-{
- pthread_t tid;
-
- sem_init(&queue_sema,0,0);
- pthread_mutex_init(&queue_lock,NULL);
- pthread_create(&tid,NULL,repl_worker,NULL);
-}
-
-static char *
-repl_oget (void *ctx, const char *id)
-{
- query_ctx_t *qctx = ctx;
- char *cur_value = NULL;
-
- (void)meta_get_value(qctx->cur_bucket,qctx->cur_key,id,&cur_value);
-
- return cur_value;
-}
-
-static char *
-repl_sget (void *ctx, const char *id)
-{
- query_ctx_t *qctx = ctx;
- json_t *elem = json_object_get(qctx->cur_server,id);
-
- return elem ? (char *)json_string_value(elem) : NULL;
-}
-
-void
-replicate (const char *url, size_t size, const char *policy)
-{
- unsigned int i;
- repl_item *item;
- value_t *expr;
- int res;
- char *url2;
- char *stctx;
- query_ctx_t qctx;
- getter_t oget;
- getter_t sget;
-
- url2 = strdup(url);
- if (!url2) {
- error(0,0,"could not parse url %s",url);
- return;
- }
- qctx.cur_bucket = strtok_r(url2,"/",&stctx);
- qctx.cur_key = strtok_r(NULL,"/",&stctx);
-
- if (!size) {
- size = meta_get_size(qctx.cur_bucket,qctx.cur_key);
- DPRINTF("fetched size %zu for %sn",size,url);
- }
-
- if (policy) {
- DPRINTF("--- policy = %sn",policy);
- expr = parse(policy);
- }
- else {
- expr = NULL;
- }
-
- oget.func = repl_oget;
- oget.ctx = &qctx;
- sget.func = repl_sget;
- sget.ctx = &qctx;
-
- for (i = 1; i < json_array_size(config); ++i) {
- if (expr) {
- qctx.cur_server = json_array_get(config,i);
- res = eval(expr,&oget,&sget);
- }
- else {
- res = 0;
- }
- if (res <= 0) {
- DPRINTF("skipping %u for %sn",i,url);
- continue;
- }
- DPRINTF("REPLICATING %s to %un",url,i);
- item = malloc(sizeof(*item));
- if (!item) {
- error(0,errno,"could not create repl_item for %s",
- url);
- break;
- }
- item->type = REPL_PUT;
- item->path = strdup(url);
- if (!item->path) {
- error(0,errno,"could not create repl_item for %s",
- url);
- break;
- }
- item->server = i;
- item->size = size;
- pthread_mutex_lock(&queue_lock);
- if (queue_tail) {
- item->next = queue_tail->next;
- queue_tail->next = item;
- }
- else {
- item->next = NULL;
- queue_head = item;
- }
- queue_tail = item;
- pthread_mutex_unlock(&queue_lock);
- g_atomic_int_inc(&rep_count);
- sem_post(&queue_sema);
- }
-
- if (expr) {
- free_value(expr);
- }
- free(url2);
-}
-
-static void
-replicate_namespace_action (const char *name, repl_t action)
-{
- unsigned int i;
- repl_item *item;
-
- for (i = 1; i < json_array_size(config); ++i) {
- DPRINTF("replicating delete(%s) on %un",name,i);
- item = malloc(sizeof(*item));
- if (!item) {
- error(0,errno,"could not create repl_item for %s",
- name);
- return;
- }
- item->type = action;
- item->path = strdup(name);
- if (!item->path) {
- free(item);
- return;
- }
- item->server = i;
- pthread_mutex_lock(&queue_lock);
- if (queue_tail) {
- item->next = queue_tail->next;
- queue_tail->next = item;
- }
- else {
- item->next = NULL;
- queue_head = item;
- }
- queue_tail = item;
- pthread_mutex_unlock(&queue_lock);
- g_atomic_int_inc(&rep_count);
- sem_post(&queue_sema);
- }
-}
-
-void
-replicate_delete (const char *name)
-{
- replicate_namespace_action(name,REPL_ODELETE);
-}
-
-void
-replicate_bcreate (const char *name)
-{
- replicate_namespace_action(name,REPL_BCREATE);
-}
-
-int
-get_provider (int i, provider_t *out)
-{
- json_t *server;
-
- server = json_array_get(config,i);
- if (!server) {
- DPRINTF("no such entry %dn",i);
- return 0;
- }
-
- out->index = i;
- out->name = json_string_value(json_object_get(server,"name"));
- out->type = json_string_value(json_object_get(server,"type"));
- out->host = json_string_value(json_object_get(server,"host"));
- out->port = json_integer_value(json_object_get(server,"port"));
- /* TBD: change key/secret field names to username/password */
- out->username = json_string_value(json_object_get(server,"key"));
- out->password = json_string_value(json_object_get(server,"secret"));
- out->path = json_string_value(json_object_get(server,"path"));
-
- /* Use empty strings instead of NULL. */
- if (!out->username) out->username = "";
- if (!out->password) out->password = "";
-
- /* TBD: do this a cleaner way. */
- if (!strcasecmp(out->type,"s3")) {
- out->func_tbl = &s3_func_tbl;
- }
- else if (!strcasecmp(out->type,"http")) {
- out->func_tbl = &curl_func_tbl;
- }
- else if (!strcasecmp(out->type,"fs")) {
- out->func_tbl = &fs_func_tbl;
- }
- else {
- out->func_tbl = &bad_func_tbl;
- }
-
- return 1;
-}
-
-char *
-get_provider_value (int i, const char *fname)
-{
- json_t *server;
- json_t *field;
-
- server = json_array_get(config,i);
- if (!server) {
- DPRINTF("no such entry %dn",i);
- return NULL;
- }
-
- field = json_object_get(server,fname);
- if (!field) {
- DPRINTF("no such field %s on server %dn",fname,i);
- return NULL;
- }
-
- return (char *)json_string_value(field);
-}
-
-void
-update_provider (const char *provider, const char *username, const char *password)
-{
- int i;
- json_t *server;
- const char *s_name;
-
- DPRINTF("updating %s username=%s password=%sn",
- provider, username, password);
-
- for (i = 0; ; ++i) {
- server = json_array_get(config,i);
- if (!server) {
- DPRINTF(" could not find provider %sn",provider);
- break;
- }
- s_name = json_string_value(json_object_get(server,"name"));
- DPRINTF(" checking %sn",s_name);
- if (s_name && !strcmp(s_name,provider)) {
- json_object_set_new(server,"key",json_string(username));
- json_object_set_new(server,"secret",
- json_string(password));
- break;
- }
- }
-}
-
-/* Part of our API to the query module. */
-char *
-follow_link (char *object, char *key)
-{
- (void)object;
- (void)key;
-
- return "no_such_object";
-}
-
-int
-get_rep_count (void)
-{
- return g_atomic_int_get(&rep_count);
-}
diff --git a/proxy.h b/proxy.h
deleted file mode 100644
index e09cfa4..0000000
--- a/proxy.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* Copyright (C) 2010 Red Hat, Inc.
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>. */
-
-#if !defined(_PROXY_H)
-#define _PROXY_H
-
-#include "backend.h"
-
-typedef struct _provider {
- int index;
- const char *name;
- const char *type;
- const char *host;
- int port;
- const char *username;
- const char *password;
- const char *path;
- backend_func_tbl *func_tbl;
-} provider_t;
-
-char *parse_config (void);
-char *auto_config (void);
-void repl_init (void);
-void replicate (const char *url, size_t size,
- const char *policy);
-void replicate_delete (const char *url);
-void replicate_bcreate (const char *bucket);
-int get_provider (int i, provider_t *out);
-void update_provider (const char *provider,
- const char *username, const char *password);
-char *get_provider_value (int i, const char *fname);
-int get_rep_count (void);
-
-int auto_start (int dbport);
-void auto_stop (void);
-
-#endif
diff --git a/query.h b/query.h
index d541dd7..8189633 100644
--- a/query.h
+++ b/query.h
@@ -41,12 +41,12 @@ typedef struct _value {
struct _value *right;
} as_tree;
};
- char *resolved; /* saved result for T_OFIELD/T_SFIELD/T_LINK */
+ const char *resolved; /* saved result for T_OFIELD/T_SFIELD/T_LINK */
} value_t;
typedef struct {
- char *(*func) (void *, const char *);
- void *ctx;
+ const char *(*func) (void *, const char *);
+ void *ctx;
} getter_t;
#define CALL_GETTER(g,x) g->func(g->ctx,x)
diff --git a/query.leg b/query.leg
index 1c85e8b..2b9cf59 100644
--- a/query.leg
+++ b/query.leg
@@ -154,7 +154,7 @@ struct { char *name; char *value; } hacked_obj_fields[] = {
};
/* Fake out the eval code for unit testing. */
-char *
+const char *
unit_oget_func (void * notused, const char *text)
{
int i;
@@ -173,7 +173,7 @@ getter_t unit_oget = { unit_oget_func };
* Same as above, but the site-field stuff is so similar to the object-field
* stuff that it's not worth exercising too much separately.
*/
-char *
+const char *
unit_sget_func (void * notused, const char *text)
{
return "never";
@@ -213,17 +213,18 @@ extern char *follow_link (const char *object, const char *key);
* all component parts are string-valued themselves. That excludes numbers
* and booleans.
*/
-char *
+const char *
string_value (value_t *v, getter_t *oget, getter_t *sget)
{
- char *left;
+ const char *left;
switch (v->type) {
case T_STRING:
return v->as_str;
case T_OFIELD:
if (!v->resolved) {
- v->resolved = oget ? CALL_GETTER(oget,v->as_str) : NULL;
+ v->resolved = oget
+ ? CALL_GETTER(oget,v->as_str) : NULL;
}
return v->resolved;
case T_SFIELD:
@@ -274,11 +275,11 @@ int
compare (value_t *left, comp_t op, value_t *right,
getter_t *oget, getter_t *sget)
{
- char *lstr;
- char *rstr;
- int lval = 0; // solely to placate gcc
- int rval;
- int num_ok = 1;
+ const char *lstr;
+ const char *rstr;
+ int lval = 0; // solely to placate gcc
+ int rval;
+ int num_ok = 1;
lstr = string_value(left,oget,sget);
rstr = string_value(right,oget,sget);
@@ -420,11 +421,7 @@ free_value (value_t *v)
return;
}
- if (v->resolved) {
- printf("freeing resolved string "%s" (%p)n",
- v->resolved, v->resolved);
- }
- free(v->resolved);
+ free((char *)v->resolved);
switch (v->type) {
case T_STRING:
@@ -466,8 +463,8 @@ free_value (value_t *v)
int
eval (const value_t *v, getter_t *oget, getter_t *sget)
{
- int res;
- char *str;
+ int res;
+ const char *str;
switch (v->type) {
case T_NUMBER:
@@ -590,10 +587,10 @@ parse (const char *text)
int
main (int argc, char **argv)
{
- int i;
- value_t *expr;
- int res;
- char *str;
+ int i;
+ value_t *expr;
+ int res;
+ const char *str;
for (i = 1; i < argc; ++i) {
expr = parse(argv[i]);
diff --git a/rest.c b/rest.c
index 070c4eb..91b3552 100644
--- a/rest.c
+++ b/rest.c
@@ -38,7 +38,8 @@
#include "iwh.h"
#include "meta.h"
#include "backend.h"
-#include "proxy.h"
+#include "setup.h"
+#include "replica.h"
#include "template.h"
#include "mpipe.h"
#include "state_defs.h"
@@ -69,10 +70,10 @@ typedef struct {
MHD_AccessHandlerCallback handler;
} rule;
-static int fs_mode = 0;
static unsigned short my_port = MY_PORT;
static int autostart = 0;
-const char *program_name;
+const char *program_name;
+char *cfg_file = NULL;
static char *(reserved_name[]) = { "_default", "_query", "_new", NULL };
static char *(reserved_attr[]) = { "bucket", "key", "date", "etag", "loc", NULL };
@@ -247,8 +248,8 @@ proxy_get_cons (void *ctx, uint64_t pos, char *buf, int max)
child_res = NULL;
pthread_join(ms->backend_th,&child_res);
if (child_res == THREAD_FAILED) {
- DPRINTF("GET producer failedn");
- /* Nothing we can do; already sent status. */
+ DPRINTF("GET producer failedn");
+ /* Nothing we can do; already sent status. */
}
if (ms->from_master) {
pthread_join(ms->cache_th,NULL);
@@ -271,7 +272,7 @@ proxy_get_data (void *cctx, struct MHD_Connection *conn, const char *url,
pipe_private *pp2;
char *my_etag;
const char *user_etag;
- int rc;
+ int rc;
(void)cctx;
(void)method;
@@ -288,10 +289,10 @@ proxy_get_data (void *cctx, struct MHD_Connection *conn, const char *url,
ms->cleanup |= CLEANUP_URL;
my_etag = meta_has_copy(ms->bucket,ms->key,me);
- if (!my_etag) {
- DPRINTF("falling back to local for %s/%sn",ms->bucket,ms->key);
- ms->from_master = 0;
- }
+ if (!my_etag) {
+ DPRINTF("falling back to local for %s/%sn",ms->bucket,ms->key);
+ ms->from_master = 0;
+ }
else if (*my_etag) {
user_etag = MHD_lookup_connection_value(
conn, MHD_HEADER_KIND, "If-None-Match");
@@ -351,8 +352,8 @@ proxy_get_data (void *cctx, struct MHD_Connection *conn, const char *url,
pp2 = NULL;
}
- rc = pipe_cons_wait_init(&ms->pipe);
- ms->rc = (rc == 0) ? MHD_HTTP_OK : MHD_HTTP_INTERNAL_SERVER_ERROR;
+ rc = pipe_cons_wait_init(&ms->pipe);
+ ms->rc = (rc == 0) ? MHD_HTTP_OK : MHD_HTTP_INTERNAL_SERVER_ERROR;
resp = MHD_create_response_from_callback(
MHD_SIZE_UNKNOWN, 65536, proxy_get_cons, pp, child_closer);
@@ -1303,10 +1304,9 @@ check_location (my_state *ms)
static int
register_image (my_state *ms)
{
- char *site;
- provider_t prov;
- int i;
- char *next;
+ const char *site;
+ const provider_t *prov;
+ char *next;
site = g_hash_table_lookup(ms->dict,"site");
if (!site) {
@@ -1319,15 +1319,14 @@ register_image (my_state *ms)
*(next++) = '0';
}
- for (i = 0; get_provider(i,&prov); ++i) {
- if (strcmp(prov.name,site)) {
- continue;
- }
- return prov.func_tbl->register_func(ms,&prov,next,ms->dict);
+ prov = get_provider(site);
+ if (!prov) {
+ DPRINTF("site %s not foundn",site);
+ return MHD_HTTP_BAD_REQUEST;
}
- DPRINTF("site %s not foundn",site);
- return MHD_HTTP_BAD_REQUEST;
+ return prov->func_tbl->register_func(ms,prov,next,ms->dict);
+
}
static int
@@ -1406,9 +1405,6 @@ static int
show_parts (struct MHD_Connection *conn, my_state *ms)
{
struct MHD_Response *resp;
- void *ctx;
- const char *name;
- const char *value;
ms->aquery = meta_get_attrs(ms->bucket,ms->key);
if (!ms->aquery) {
@@ -1512,10 +1508,11 @@ proxy_object_post (void *cctx, struct MHD_Connection *conn, const char *url,
static int
prov_list_generator (void *ctx, uint64_t pos, char *buf, int max)
{
- my_state *ms = ctx;
- int len;
- provider_t prov;
- const char *accept_hdr;
+ my_state *ms = ctx;
+ int len;
+ gpointer key;
+ const provider_t *prov;
+ const char *accept_hdr;
(void)pos;
@@ -1528,6 +1525,7 @@ prov_list_generator (void *ctx, uint64_t pos, char *buf, int max)
return -1;
}
ms->cleanup |= CLEANUP_TMPL;
+ init_prov_iter(&ms->prov_iter);
len = tmpl_prov_header(ms->gen_ctx);
if (!len) {
return -1;
@@ -1543,9 +1541,9 @@ prov_list_generator (void *ctx, uint64_t pos, char *buf, int max)
return -1;
}
- if (get_provider(ms->gen_ctx->index,&prov)) {
- len = tmpl_prov_entry(ms->gen_ctx,prov.name,prov.type,
- prov.host, prov.port, prov.username, prov.password);
+ if (g_hash_table_iter_next(&ms->prov_iter,&key,(gpointer *)&prov)) {
+ len = tmpl_prov_entry(ms->gen_ctx,prov->name,prov->type,
+ prov->host, prov->port, prov->username, prov->password);
if (!len) {
return -1;
}
@@ -1875,13 +1873,13 @@ usage (int status)
{
if (status != EXIT_SUCCESS)
fprintf (stderr, _("Try `%s --help' for more information.n"),
- program_name);
+ program_name);
else
{
printf (_(" Usage: %s [OPTION]n "),
- program_name);
+ program_name);
fputs (_(" Deltacloud image-warehouse daemon.n A configuration file must be specified.n@@ -1900,7 +1898,7 @@ A configuration file must be specified.n n Report %s bugs to %s.n "),
- program_name, PACKAGE_BUGREPORT);
+ program_name, PACKAGE_BUGREPORT);
}
exit (status);
}
@@ -1977,7 +1975,7 @@ args_done:
}
}
else if (!autostart && cfg_file) {
- me = parse_config();
+ me = parse_config(cfg_file);
if (!me) {
error(0,0,"could not parse %s",cfg_file);
return !0;
diff --git a/state_defs.h b/state_defs.h
index f13c043..0460a00 100644
--- a/state_defs.h
+++ b/state_defs.h
@@ -58,6 +58,7 @@ typedef struct {
pthread_t cache_th;
/* for bucket/object/provider list generators */
tmpl_ctx_t *gen_ctx;
+ GHashTableIter prov_iter;
} my_state;
#define CLEANUP_CURL 0x01
diff --git a/t/basic b/t/basic
index 0f409a4..4dac5e9 100644
--- a/t/basic
+++ b/t/basic
@@ -48,9 +48,10 @@ EOF
emit_provider()
{
local xml_or_json=$1
- case $# in 7);; *) echo "emit_provider: wrong # args" 1>&2; exit 1;; esac
+ shift 1
case $xml_or_json in xml|json);;
*) echo "invalid xml_or_json $xml_or_json" 1>&2; exit 1;; esac
+ case $# in 6);; *) echo "emit_provider: wrong # args" 1>&2; exit 1;; esac
if test $xml_or_json = xml; then
printf @@ -63,7 +64,7 @@ emit_provider()
tt<password>%s</password>
t</provider>
</providers>
-' $2 $3 $4 $5 $6 $7
+' "$@"
else
printf '[
@@ -76,7 +77,7 @@ emit_provider()
tt"password": "%s"
t}
]
-' $2 $3 $4 $5 $6 $7
+' "$@"
fi
}
@@ -178,11 +179,10 @@ grep ' 404$' del_nosuch.err || fail=1
# ################## Providers
p_name=primary
-host='(null)'
# Verify that default providers information is as expected.
curl http://localhost:$port/_providers > p || fail=1
-emit_provider xml "$p_name" fs "$host" 0 '' '' > p.exp || fail=1
+emit_provider xml "$p_name" fs '' 0 '' '' > p.exp || fail=1
compare p.exp p || fail=1
for i in xml json; do
@@ -193,7 +193,7 @@ for i in xml json; do
curl_H -d provider="$p_name" -d username=u -d password=p http://localhost:$port/_providers || fail=1
curl_H http://localhost:$port/_providers > p || fail=1
- emit_provider $i "$p_name" fs "$host" 0 u p > p.xml || fail=1
+ emit_provider $i "$p_name" fs '' 0 u p > p.xml || fail=1
compare p.xml p || fail=1
# Attempt to modify some other attribute; should fail (but curl won't)
@@ -209,20 +209,20 @@ for i in xml json; do
curl_H -d provider="$p_name" -d username=u http://localhost:$port/_providers || fail=1
curl_H http://localhost:$port/_providers > p || fail=1
- emit_provider $i "$p_name" fs "$host" 0 u p > p.exp || fail=1
+ emit_provider $i "$p_name" fs '' 0 u p > p.exp || fail=1
compare p.exp p || fail=1
curl_H -d provider="$p_name" -d password=p http://localhost:$port/_providers || fail=1
curl_H http://localhost:$port/_providers > p || fail=1
- emit_provider $i "$p_name" fs "$host" 0 u p > p.exp || fail=1
+ emit_provider $i "$p_name" fs '' 0 u p > p.exp || fail=1
compare p.exp p || fail=1
# Try to change both username and password. Now, it must succeed.
curl_H -d provider="$p_name" -d username=v -d password=q http://localhost:$port/_providers || fail=1
curl_H http://localhost:$port/_providers > p || fail=1
- emit_provider $i "$p_name" fs "$host" 0 v q > p.exp || fail=1
+ emit_provider $i "$p_name" fs '' 0 v q > p.exp || fail=1
compare p.exp p || fail=1
# List an empty bucket.
diff --git a/template.c b/template.c
index a7f1cca..b66bc1d 100644
--- a/template.c
+++ b/template.c
@@ -273,6 +273,12 @@ tmpl_prov_entry (tmpl_ctx_t *ctx,
int size;
const tmpl_format_t *fmt = ctx->format;
+ if (!name) name = "";
+ if (!type) type = "";
+ if (!host) host = "";
+ if (!user) user = "";
+ if (!pass) pass = "";
+
size = snprintf(ctx->raw_buf,TMPL_BUF_SIZE,fmt->prov_entry,
name, type, host, port, user, pass);
if (size >= TMPL_BUF_SIZE) {
-----------------------------------------------------------------------
Summary of changes:
Makefile.am | 6 +-
auto.c | 14 +-
backend.c | 38 +-
iwh.h | 3 +-
meta.cpp | 2 +-
proxy.c | 1122 ----------------------------------------------------------
proxy.h | 49 ---
query.h | 6 +-
query.leg | 39 +--
rest.c | 72 ++--
state_defs.h | 1 +
t/basic | 18 +-
template.c | 6 +
13 files changed, 104 insertions(+), 1272 deletions(-)
delete mode 100644 proxy.c
delete mode 100644 proxy.h
repo.or.cz automatic notification. Contact project admin jim(a)meyering.net
if you want to unsubscribe, or site admin admin(a)repo.or.cz if you receive
no reply.
--
iwhd.git ("image warehouse daemon")
13 years, 6 months