This works, I think... I built a test RPM for Matt, but there was no word on compatibility with Ruby in Aeolus yet. Other than that, it looks like we're done here, unless I'm missing something fundamental.
-- Pete
diff --git a/Makefile.am b/Makefile.am index 7cf2e54..1a2027b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -47,7 +47,8 @@ iwhd_SOURCES = \ qparser.y \ rest.c \ setup.c \ - template.c + template.c \ + user.c
noinst_HEADERS = \ backend.h \ @@ -60,7 +61,8 @@ noinst_HEADERS = \ replica.h \ setup.h \ state_defs.h \ - template.h + template.h \ + user.h
EXTRA_iwhd_SOURCES = qlexer.l
diff --git a/NEWS b/NEWS index c7467a9..6c13654 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,11 @@ iwhd NEWS -*- outline -*-
* Noteworthy changes in release ?.? (????-??-??) [?]
+** New features + + A preliminary support is added for authorization with two-legged OAuth. + See the -o flag. + ** Bug fixes
iwhd now continues to work properly when mongo dies and is restarted. diff --git a/configure.ac b/configure.ac index e23d1fa..8b115cb 100644 --- a/configure.ac +++ b/configure.ac @@ -70,6 +70,9 @@ AC_CHECK_LIB([uuid], [uuid_generate_random], [AC_MSG_ERROR([Missing required uuid lib])]) AC_SUBST([UUID_LIB])
+dnl How to check properly? +LIBS="$LIBS -loauth" + PKG_CHECK_MODULES([HAIL],[libhail >= 0.8]) AC_SUBST([HAIL_LIBS]) AC_SUBST([HAIL_CFLAGS]) @@ -97,6 +100,9 @@ AC_CACHE_CHECK([whether json_load_file takes 3 arguments], AC_DEFINE_UNQUOTED([JANSSON_LOAD_FLAG], $iw_cv_func_jansson_flag, [Define to "0," if json_load_file and json_loads require a flags arguments.])
+AC_CHECK_HEADER([oauth.h], , + [AC_MSG_ERROR([Missing OAuth development library: liboauth-devel])]) + # from http://www.gnu.org/software/autoconf-archive/ AX_BOOST_BASE AX_BOOST_SYSTEM diff --git a/rest.c b/rest.c index 92f36cb..33135c6 100644 --- a/rest.c +++ b/rest.c @@ -15,12 +15,14 @@
#include <config.h>
+#include <ctype.h> #include <error.h> #include <fcntl.h> #include <getopt.h> #include <poll.h> #include <pthread.h> #include <semaphore.h> +#include <stdbool.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> @@ -33,6 +35,7 @@ #include <microhttpd.h> #include <hstor.h> /* only for ARRAY_SIZE at this point */ #include <curl/curl.h> +#include <oauth.h>
#include "configmake.h" /* for LOCALEDIR */ #include "dirname.h" @@ -50,6 +53,7 @@ #include "template.h" #include "mpipe.h" #include "state_defs.h" +#include "user.h" #include "version-etc.h" #include "xstrtol.h" #include "xvasprintf.h" @@ -113,6 +117,8 @@ static unsigned short my_port = MY_PORT; static LIST_HEAD(gc_list, _my_state) my_states; /* this keeps GC informed */ static pthread_mutex_t my_lock = PTHREAD_MUTEX_INITIALIZER; static const char *log_name; +static char my_hostname[HOST_NAME_MAX+1]; +static bool oauth = false;
static const char *const (reserved_name[]) = {"_default", "_new", "_policy", "_query", NULL}; static const char *const (reserved_attr[]) = {"_attrs", "_bucket", "_date", "_etag", "_key", "_loc", "_size", NULL}; @@ -2091,6 +2097,478 @@ parse_url (const char *url, my_state *ms) return eindex; }
+static char * +rebuild_url(bool is_ssl, struct MHD_Connection *conn, const char *path) +{ + const char *val; + char *ret; + int rc; + + /* + * Luckily for us, this header includes the colon-port. + * It really should be called "Netloc", not "Host". + */ + val = MHD_lookup_connection_value(conn,MHD_HEADER_KIND,"Host"); + if (!val) { + if (my_hostname[0] == 0) { + if (gethostname(my_hostname, HOST_NAME_MAX) != 0) { + /* TBD: This may flood - user-driven. */ + error (0, errno, _("gethostname")); + return NULL; + } + my_hostname[HOST_NAME_MAX] = 0; + } + /* TBD: And port? But this is a fall-back anyway. */ + val = my_hostname; + } + + rc = asprintf(&ret, "%s://%s%s", is_ssl? "https": "http", val, path); + if (rc < 0) { + DPRINTF("No core"); + return NULL; + } + return ret; +} + +/* Ensure that VAL consists of optional spaces, followed by + the string, TYPE, followed by more optional spaces. + If so, return a pointer to the first non-space following + TYPE in VAL. Otherwise, return NULL. */ +static const char * +auth_type_check (const char *val, const char *type) +{ + const char *p; + size_t tlen = strlen(type); + + p = val; + while (*p == ' ') { + if (*p == 0) { + return NULL; + } + p++; + } + if (strncmp(p, type, tlen) != 0) + return NULL; + p += tlen; + if (*p != ' ') + return NULL; + p++; + while (*p == ' ') { + if (*p == 0) { + return NULL; + } + p++; + } + return p; +} + +/* + * Split up an Authorization header. + * Note that this decodes the strings. + */ +static bool +auth_hdr_split (int *pargc, char ***pargv, const char *val) +{ + enum { ARGC_MAX = 20 }; /* better than 2-pass */ + enum { VAL_MAX = 512 }; /* a constant is not great, but... */ + enum { + ST_NONE, ST_ERR, ST_KEY, ST_EQ, ST_VAL, ST_Q2, ST_COM, + ST_PCENT1, ST_PCENT2 + } state; + int argc; + char **argv; + const char *p; + const char *key; + size_t klen = 0; + size_t vlen = 0; + char vbuf[VAL_MAX+1]; + unsigned int acc = 0, n; + char *arg; + + if (!(val = auth_type_check(val, "OAuth"))) + return false; + + argv = malloc((ARGC_MAX+1) * sizeof(char*)); + if (!argv) { + return false; + } + argc = 0; + + arg = NULL; + state = ST_NONE; + key = NULL; + for (p = val; *p != 0; p++) { + switch (state) { + case ST_NONE: + if (*p == ' ') { + ; + } + else if (!isascii(*p) || !isalpha(*p)) { + state = ST_ERR; + } + else { + state = ST_KEY; + key = p; + } + break; + case ST_KEY: + if (*p == '=') { + state = ST_EQ; + klen = p-key; + } + break; + case ST_EQ: + if (*p == '"') { + state = ST_VAL; + vlen = 0; + } + else { + /* + * A compiler writer would've had an ulcer + * from this, because obviously we are going + * to recover in a dirty way. But whatever, + * all we want is an error indication. + */ + state = ST_ERR; + } + break; + case ST_VAL: + if (*p == '"') { + vbuf[vlen] = 0; + state = ST_Q2; + } + else if (*p == '%') { + state = ST_PCENT1; + acc = 0; + } + else { + vbuf[vlen++] = *p; + } + break; + case ST_PCENT1: + if (isxdigit(*p)) { + if (*p <= '9') { + n = toupper(*p) - '0'; + } else { + n = toupper(*p) - 'A' + 10; + } + acc = (acc << 4) | (n & 0x0f); + state = ST_PCENT2; + } else { + state = ST_ERR; + } + break; + case ST_PCENT2: + if (isxdigit(*p)) { + if (*p <= '9') { + n = toupper(*p) - '0'; + } else { + n = toupper(*p) - 'A' + 10; + } + acc = (acc << 4) | (n & 0x0f); + if (iscntrl(acc)) + acc = '?'; + if (vlen >= VAL_MAX) { + state = ST_ERR; + break; + } + vbuf[vlen++] = acc; + state = ST_VAL; + } + else { + state = ST_ERR; + } + break; + case ST_Q2: + if (*p == ',') { + arg = malloc(klen+1+vlen+1); + if (!arg) { + state = ST_ERR; + break; + } + memcpy(arg, key, klen); + arg[klen] = '='; + memcpy(arg+klen+1, vbuf, vlen); + arg[klen+1+vlen] = 0; + + state = ST_COM; + key = NULL; + } + else { + state = ST_ERR; + } + break; + case ST_COM: + if (*p == ' ') { + if (!arg) { + state = ST_NONE; + key = NULL; + break; + } + if (argc >= ARGC_MAX) { + /* TBD: error out immediately */ + state = ST_ERR; + break; + } + argv[argc++] = arg; + arg = NULL; + state = ST_NONE; + key = NULL; + } + else { + state = ST_ERR; + } + default: // ST_ERR + if (*p == ',') { + state = ST_COM; + } + } + } + + if (arg) { + DPRINTF("auth_hdr_split ended with arg %s\n", arg); + free (arg); + } + + if (state == ST_Q2) { + if (key) { + arg = malloc(klen+1+vlen+1); + if (arg && argc < ARGC_MAX) { + memcpy(arg, key, klen); + arg[klen] = '='; + memcpy(arg+klen+1, vbuf, vlen); + arg[klen+1+vlen] = 0; + argv[argc++] = arg; + } + } + } + argv[argc] = NULL; + + *pargc = argc; + *pargv = argv; + return true; +} + +/* Extract the value from key="val". */ +static char * +oauth_param_val(const char *arg) +{ + char *val = strchr(arg, '='); + if (!val) + return NULL; + val++; + if (*val == 0) + return NULL; + return strdup(val); +} + +/* clone of oauth_param_exists with minimal changes */ +static int +oauth_param_find(char **argv, int argc, const char *key) +{ + int i; + size_t l = strlen(key); + for (i=0;i<argc;i++) + if (strlen(argv[i])>l && !strncmp(argv[i],key,l) && argv[i][l] == '=') + return i; + return -1; +} + +static int +do_oauth (struct MHD_Connection *conn, my_state *ms, + const char *url, const char *method) +{ + const char *val; + struct MHD_Response *resp; + int argc; + char **argv; + int oargc; + char **oargv; + OAuthMethod omethod; + int x; + struct user *u; + const char *user_name; + const char *sig_method; + const char *sig_supplied, *sig_calculated; + + val = MHD_lookup_connection_value(conn,MHD_HEADER_KIND,"Authorization"); + if (!val) { + resp = MHD_create_response_from_data(0,NULL,MHD_NO,MHD_NO); + if (!resp) { + return -1; + } + MHD_add_response_header(resp,"WWW-Authenticate","OAuth"); + MHD_queue_response(conn,MHD_HTTP_UNAUTHORIZED,resp); + MHD_destroy_response(resp); + return 1; + } + + if (!auth_hdr_split(&argc, &argv, val)) { + do_resp(conn,NULL,MHD_HTTP_BAD_REQUEST,NULL,"400\r\n"); + return 1; + } + if (verbose >= 2) { + int i; + printf("OAuth supplied [%d]\n", argc); + for (i = 0; i < argc; i++) { + printf(" %s\n", argv[i]); + } + } + + x = oauth_param_find(argv,argc,"oauth_consumer_key"); + if (x == -1) { + do_resp(conn,NULL,MHD_HTTP_BAD_REQUEST,NULL,"400 no user\r\n"); + return 1; + } + user_name = oauth_param_val(argv[x]); + if (!user_name) { + /* + * It should be impossible for oauth_param_val() to fail, + * because we only save parameters with correct syntax, + * but never underestimate the trickery of network input. + * Also, there's a strdup() inside it. + */ + do_resp(conn,NULL,MHD_HTTP_INTERNAL_SERVER_ERROR,NULL, + "500 null username\r\n"); + return 1; + } + u = user_lookup(user_name); + if (!u) { + do_resp(conn,NULL,MHD_HTTP_FORBIDDEN,NULL,"403 bad user\r\n"); + return 1; + } + + x = oauth_param_find(argv,argc,"oauth_signature_method"); + if (x == -1) { + do_resp(conn,NULL,MHD_HTTP_BAD_REQUEST,NULL, + "400 no signature method\r\n"); + return 1; + } + sig_method = oauth_param_val(argv[x]); + if (!sig_method) { + do_resp(conn,NULL,MHD_HTTP_INTERNAL_SERVER_ERROR,NULL, + "500 null signature method\r\n"); + return 1; + } + if (strcmp(sig_method, "HMAC-SHA1")==0) { + omethod = OA_HMAC; + } + else { + /* + * We don't support OA_RSA for now, it needs key management. + */ + do_resp(conn,NULL,MHD_HTTP_BAD_REQUEST,NULL, + "400 bad signature method (use HMAC-SHA1)\r\n"); + return 1; + } + + /* + * Putting marshalling arguments for and extracting results from + * oauth_sign_array2_process makes you dirty, but this way offloads + * some tricks to liboauth, so there. + */ + oargc = 1+argc; /* add a slot for URL */ + + /* + * If we just call malloc() here, we actually involve a function + * intercepted by GC. Then, glibc will abort inside liboauth + * oauth_sign_array2_process->xrealloc->free() => "invalid pointer". + * Solution: use a random liboauth function which allocates + * the memory we need. The result can be safely passed to liboauth. + */ + oargv = malloc((oargc+1)*sizeof(char*)); + + memset((void*)oargv, '0', (oargc+1)*sizeof(char*)); + ((char *)oargv)[(oargc+1)*sizeof(char*) - 1] = 0; + oargv = (char **) oauth_url_escape((char *)oargv); + + if (!oargv) { + do_resp(conn,NULL,MHD_HTTP_INTERNAL_SERVER_ERROR,NULL, + "500\r\n"); + return 1; + } + oargv[0] = rebuild_url(false, conn, url); /* TBD: SSL */ + if (!oargv[0]) { + do_resp(conn,NULL,MHD_HTTP_INTERNAL_SERVER_ERROR,NULL, + "500\r\n"); + return 1; + } + + // no good, must filter + // memcpy(&oargv[1], argv, (argc+1)*sizeof(char*)); + + oargc = 1; + + x = oauth_param_find(argv,argc,"oauth_nonce"); + if (x == -1) { + do_resp(conn,NULL,MHD_HTTP_BAD_REQUEST,NULL,"400 no nonce\r\n"); + return 1; + } + oargv[oargc++] = argv[x]; + + x = oauth_param_find(argv,argc,"oauth_timestamp"); + if (x == -1) { + do_resp(conn,NULL,MHD_HTTP_BAD_REQUEST,NULL, + "400 no timestamp\r\n"); + return 1; + } + oargv[oargc++] = argv[x]; + + oauth_sign_array2_process(&oargc, &oargv, NULL, omethod, method, + u->name, u->pass, NULL, NULL); + if (verbose >= 2) { + int i; + printf("OAuth calculated\n"); + for (i = 0; i < oargc; i++) { + printf(" %s\n", oargv[i]); + } + } + + /* + * We've got the signature, now verify. + */ + x = oauth_param_find(argv,argc,"oauth_signature"); + if (x == -1) { + do_resp(conn,NULL,MHD_HTTP_BAD_REQUEST,NULL, + "400 no signature\r\n"); + return 1; + } + sig_supplied = oauth_param_val(argv[x]); + if (!sig_supplied) { + do_resp(conn,NULL,MHD_HTTP_INTERNAL_SERVER_ERROR,NULL, + "500 null supplied\r\n"); + return 1; + } + + x = oauth_param_find(oargv,oargc,"oauth_signature"); + if (x == -1) { + do_resp(conn,NULL,MHD_HTTP_INTERNAL_SERVER_ERROR,NULL, + "500 no signature\r\n"); + return 1; + } + sig_calculated = oauth_param_val(oargv[x]); + if (!sig_calculated) { + do_resp(conn,NULL,MHD_HTTP_INTERNAL_SERVER_ERROR,NULL, + "500 null calculated\r\n"); + return 1; + } + + if (strcmp(sig_supplied, sig_calculated) != 0) { + DPRINTF("OAuth signature mismatch"); + return -1; + } + return 0; +} + +static int kv_iter_dump(void *cls, enum MHD_ValueKind kind, + const char *key, const char *value) +{ + /* ok to use printf when MHD_USE_THREAD_PER_CONNECTION maybe? */ + printf("%s: %s\n", key, value); + + return MHD_YES; +} + static int access_handler_0 (void *cctx, struct MHD_Connection *conn, const char *url, const char *method, const char *version, const char *data, @@ -2099,10 +2577,24 @@ access_handler_0 (void *cctx, struct MHD_Connection *conn, const char *url, unsigned int i; url_type utype; my_state *ms = *rctx; + int rc;
log_check(); + if (verbose >= 2) { + printf(">> HTTP\n"); + rc = MHD_get_connection_values(conn, MHD_HEADER_KIND, + kv_iter_dump, NULL); + printf(">> headers: %d\n", rc); + }
if (ms) { + if (oauth) { + rc = do_oauth(conn,ms,url,method); + if (rc < 0) + return MHD_NO; + if (rc > 0) + return MHD_YES; + } return ms->handler(cctx,conn,url,method,version, data,data_size,rctx); } @@ -2131,12 +2623,20 @@ access_handler_0 (void *cctx, struct MHD_Connection *conn, const char *url, ms->conn = conn; *rctx = ms; pthread_mutex_lock(&my_lock); - LIST_INSERT_HEAD(&my_states, ms, gc_link); + LIST_INSERT_HEAD(&my_states,ms,gc_link); pthread_mutex_unlock(&my_lock); + if (oauth) { + rc = do_oauth(conn,ms,url,method); + if (rc < 0) + return MHD_NO; + if (rc > 0) + return MHD_YES; + } return ms->handler(cctx,conn,url,method,version, data,data_size,rctx); }
+ /* This is *unauthenticated*. Remove altogether? TBD */ if (!strcmp(method,"QUIT")) { (void)sem_post((sem_t *)cctx); return MHD_NO; @@ -2194,7 +2694,9 @@ static const struct option my_options[] = { { "db", required_argument, NULL, 'd' }, { "logfile", required_argument, NULL, 'l' }, { "master", required_argument, NULL, 'm' }, + { "oauth", no_argument, NULL, 'o' }, { "port", required_argument, NULL, 'p' }, + { "usercred", required_argument, NULL, 'U' }, { "verbose", no_argument, NULL, 'v' }, { "version", no_argument, NULL, GETOPT_VERSION_CHAR }, { "help", no_argument, NULL, GETOPT_HELP_CHAR }, @@ -2222,7 +2724,9 @@ A configuration file must be specified.\n\ -d, --db=HOST_PORT database server as ip[:port]\n\ -l, --logfile=FILE logfile (default stdout/stderr)\n\ -m, --master=HOST_PORT master (upstream) server as ip[:port]\n\ + -o, --oauth enable OAuth\n\ -p, --port=PORT alternate listen port (default 9090)\n\ + -U, --usercred=U:P singleton pair of username:password\n\ -v, --verbose verbose/debug output\n\ \n\ --help display this help and exit\n\ @@ -2275,6 +2779,7 @@ main (int argc, char **argv) sem_t the_sem; bool autostart = false; char *cfg_file = NULL; + char *usercred = NULL;
set_program_name (argv[0]); setlocale (LC_ALL, ""); @@ -2287,7 +2792,8 @@ main (int argc, char **argv) db_host = xstrdup ("localhost");
for (;;) - switch (getopt_long(argc,argv,"ac:d:l:m:p:v",my_options,NULL)) { + switch (getopt_long(argc,argv,"ac:d:l:m:op:U:v",my_options,NULL)) + { case 'a': autostart = true; break; @@ -2308,10 +2814,16 @@ main (int argc, char **argv) free (master_host); master_host = NULL; extract_host_port (optarg, &master_host, &master_port); break; + case 'o': + oauth = true; + break; case 'p': assert (optarg); my_port = get_port (optarg); break; + case 'U': /* Uppercase because very temporary until -u appears. */ + usercred = optarg; + break; case 'v': ++verbose; break; @@ -2366,6 +2878,10 @@ args_done: usage (EXIT_FAILURE); }
+ if (usercred) { + user_add(usercred); + } + char *t; char const *tmpdir = ((t = getenv ("TMPDIR")) ? t : "/tmp"); tmpfile_template = xasprintf ("%s/iwhd.XXXXXX", tmpdir); @@ -2382,6 +2898,9 @@ args_done: printf("db is at %s:%u\n",db_host,db_port); printf("will listen on port %u\n",my_port); printf("my location is "%s"\n",me); + if (oauth) { + printf("autheticated with OAuth\n"); + } if (fflush(stdout) || ferror(stdout)) error(EXIT_FAILURE, 0, _("write failed")); } diff --git a/t/.gitignore b/t/.gitignore new file mode 100644 index 0000000..e5baa0c --- /dev/null +++ b/t/.gitignore @@ -0,0 +1,2 @@ +*.o +poke diff --git a/t/Makefile.am b/t/Makefile.am index a3b39d7..28f105c 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -24,11 +24,13 @@ TESTS = \ provider \ replication \ auto \ - registration + registration \ + oauth
lock_dir = $(abs_builddir)/lock-dir clean-local: $(AM_V_GEN)rm -rf "$(lock_dir)" + $(AM_V_GEN)rm -f poke poke.o hstor.o
.PHONY: prereq prereq: @@ -36,10 +38,24 @@ prereq:
$(TEST_LOGS): prereq
+hstor.o: hstor.c hoa.h + gcc -Wall $(LIBXML_CFLAGS) -I$(top_builddir) -O2 -c -o $@ $< + +poke.o: poke.c hoa.h + gcc -Wall -I$(top_builddir) -O2 -c -o $@ $< + +poke: poke.o hstor.o + gcc -o poke $^ $(LIBXML_LIBS) $(CURL_LIB) $(LIBS) + +oauth: poke + EXTRA_DIST = \ $(TESTS) \ init.cfg \ - init.sh + init.sh \ + hoa.h \ + hstor.c \ + poke.c
TESTS_ENVIRONMENT = \ tmp__=$$TMPDIR; test -d "$$tmp__" || tmp__=.; \ diff --git a/t/hoa.h b/t/hoa.h new file mode 100644 index 0000000..609bb7d --- /dev/null +++ b/t/hoa.h @@ -0,0 +1,144 @@ +#ifndef _HSTOR_H +#define _HSTOR_H + +/* + * Copyright 2008-2011 Red Hat, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + */ +/* + * This is very clearly hstor.h, so why not call it that? The reason is + * that we want to be very sure about what header is being included, + * regardless of how confusing -I chains are. + */ + +#include <sys/queue.h> +#include <sys/types.h> +#include <stdbool.h> +#include <curl/curl.h> + +struct hstor_client { + CURL *curl; + char *acc; + char *user; + char *key; + bool verbose; +}; + +struct hstor_bucket { + SLIST_ENTRY(hstor_bucket) blink; + char *name; +}; + +struct hstor_blist { + char *own_id; /* ID */ + char *own_name; /* DisplayName */ + SLIST_HEAD(_hstor_blist, hstor_bucket) list; + struct hstor_bucket *tail; +}; + +struct hstor_object { + SLIST_ENTRY(hstor_object) clink; + char *key; + char *time_mod; + char *etag; + off_t size; + char *storage; + char *own_id; + char *own_name; +}; + +struct hstor_keylist { + char *name; + char *prefix; + char *marker; + char *delim; + unsigned int max_keys; + bool trunc; + SLIST_HEAD(_hstor_object, hstor_object) contents; + struct hstor_object *contail; +}; + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +#endif + +// #define PATH_ESCAPE_MASK 0x02 +// #define QUERY_ESCAPE_MASK 0x04 + +enum { + HREQ_MAX_HDR = 128, /* max hdrs per req */ +}; + +struct http_hdr { + char *key; + char *val; +}; + +struct http_req { + char *method; /* GET, POST, etc. */ + char *orig_path; /* XXX not used in OAuth */ +}; + +enum ReqQ { + URIQ_ACL, + URIQ_LOCATION, + URIQ_LOGGING, + URIQ_TORRENT, + URIQNUM +}; + +enum ReqACLC { + ACLC_PRIV, + ACLC_PUB_R, + ACLC_PUB_RW, + ACLC_AUTH_R, + ACLCNUM +}; + +/* uri.c */ +extern int huri_field_unescape(char *s, int s_len); +extern char* huri_field_escape (char *signed_str, unsigned char mask); + +/* hstor.c */ +extern void hstor_free(struct hstor_client *hstor); +extern void hstor_free_blist(struct hstor_blist *blist); +extern void hstor_free_bucket(struct hstor_bucket *buck); +extern void hstor_free_object(struct hstor_object *obj); +extern void hstor_free_keylist(struct hstor_keylist *keylist); + +extern struct hstor_client *hstor_new(const char *service_acc, + const char *user, const char *secret_key); + +extern bool hstor_add_bucket(struct hstor_client *hstor, const char *name); +extern bool hstor_del_bucket(struct hstor_client *hstor, const char *name); + +extern struct hstor_blist *hstor_list_buckets(struct hstor_client *hstor); + +extern bool hstor_get(struct hstor_client *hstor, const char *bucket, const char *key, + size_t (*write_cb)(void *, size_t, size_t, void *), + void *user_data, bool want_headers); +extern void *hstor_get_inline(struct hstor_client *hstor, const char *bucket, + const char *key, bool want_headers, size_t *len); +extern bool hstor_put(struct hstor_client *hstor, const char *bucket, const char *key, + size_t (*read_cb)(void *, size_t, size_t, void *), + off_t len, void *user_data); +extern bool hstor_put_inline(struct hstor_client *hstor, const char *bucket, + const char *key, void *data, off_t len); +extern bool hstor_del(struct hstor_client *hstor, const char *bucket, const char *key); + +extern struct hstor_keylist *hstor_keys(struct hstor_client *hstor, + const char *bucket); + +#endif /* _HSTOR_H */ diff --git a/t/hstor.c b/t/hstor.c new file mode 100644 index 0000000..a82d58b --- /dev/null +++ b/t/hstor.c @@ -0,0 +1,894 @@ +/* + * Copyright 2008-2011 Red Hat, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + * + */ + +#define _GNU_SOURCE +#include <config.h> + +#include <stdlib.h> +#include <string.h> +#include <ctype.h> +#include <sys/queue.h> +#include <curl/curl.h> +#include <libxml/tree.h> +#include <oauth.h> + +#include "hoa.h" + +static int _strcasecmp(const unsigned char *a, const char *b) +{ + return xmlStrcasecmp(a, (const unsigned char *) b); +} + +static int _strcmp(const unsigned char *a, const char *b) +{ + return xmlStrcmp(a, (const unsigned char *) b); +} + +struct api_buf { + char *buf; + size_t alloc; + size_t used; +}; + +static size_t api_wcb(void *ptr, size_t bsz, size_t nmemb, void *arg) +{ + struct api_buf *bp = arg; + char *mem; + size_t len; + + if (bp->alloc - bp->used < nmemb) { + len = (((bp->used + nmemb) / 2000) + 1) * 2000; + mem = realloc(bp->buf, len); + if (!mem) + return 0; + bp->buf = mem; + bp->alloc = len; + } + memcpy(bp->buf + bp->used, ptr, nmemb); + bp->used += nmemb; + return nmemb; +} + +static size_t api_rcb(void *ptr, size_t bsz, size_t nmemb, void *arg) +{ + struct api_buf *bp = arg; + size_t count; + + count = bp->alloc - bp->used; + if (count > nmemb) + count = nmemb; + if (count) { + memcpy(ptr, bp->buf + bp->used, count); + bp->used += count; + } + return count; +} + +void hstor_free(struct hstor_client *hstor) +{ + if (hstor->curl) + curl_easy_cleanup(hstor->curl); + free(hstor->acc); + free(hstor->user); + free(hstor->key); + free(hstor); +} + +struct hstor_client *hstor_new(const char *service_acc, + const char *user, const char *secret_key) +{ + struct hstor_client *hstor; + + hstor = calloc(1, sizeof(struct hstor_client)); + if (!hstor) + return NULL; + + hstor->acc = strdup(service_acc); + hstor->user = strdup(user); + hstor->key = strdup(secret_key); + if (!hstor->acc || !hstor->user || !hstor->key) + goto err_out; + + if (curl_global_init(CURL_GLOBAL_ALL)) + goto err_out; + + hstor->curl = curl_easy_init(); + if (!hstor->curl) + goto err_out; + + return hstor; + +err_out: + hstor_free(hstor); + return NULL; +} + +void hstor_free_bucket(struct hstor_bucket *buck) +{ + if (!buck) + return; + + free(buck->name); + free(buck); +} + +void hstor_free_blist(struct hstor_blist *blist) +{ + if (!blist) + return; + + free(blist->own_id); + free(blist->own_name); + + while (!SLIST_EMPTY(&blist->list)) { + struct hstor_bucket *buck = SLIST_FIRST(&blist->list); + SLIST_REMOVE_HEAD(&blist->list, blink); + hstor_free_bucket(buck); + } + + free(blist); +} + +static void hstor_parse_link(xmlDocPtr doc, xmlNode *node, + struct hstor_blist *blist) +{ + struct hstor_bucket *buck; + xmlChar *rel, *href; + + buck = calloc(1, sizeof(*buck)); + if (!buck) + return; + + if (_strcmp(node->name, "link") != 0) + goto err_out; + + rel = xmlGetProp(node, (xmlChar *)"rel"); + if (!rel) + goto err_out; + if (_strcasecmp(rel, "bucket") != 0) + goto err_out; + + href = xmlGetProp(node, (xmlChar *)"href"); + if (!href) + goto err_out; + + /* XXX Trim the URL base, return bucket name. */ + buck->name = strdup((char *)href); + + xmlFree(rel); + xmlFree(href); + + if (buck->name) { + if (blist->tail) { + SLIST_INSERT_AFTER(blist->tail, buck, blink); + } else { + SLIST_INSERT_HEAD(&blist->list, buck, blink); + blist->tail = buck; + } + } else { + hstor_free_bucket(buck); + } + return; + +err_out: + hstor_free_bucket(buck); +} + +static bool hstor_resplit(const struct hstor_client *hstor, + const char *bucket, const char *key, + char **url, char **hosthdr, char **path) +{ + char *unesc_path; + int rc; + + rc = asprintf(&unesc_path, "/%s/%s", bucket, key); + if (rc < 0) + goto err_spath; +#if 0 /* hutil */ + *path = huri_field_escape(unesc_path, PATH_ESCAPE_MASK); + if (!*path) + goto err_epath; +#else + *path = unesc_path; /* XXX Oh bloody eff, link hutil.c too?! */ +#endif + + rc = asprintf(hosthdr, "Host: %s", hstor->acc); + if (rc < 0) + goto err_host; + + rc = asprintf(url, "http://%s%s", hstor->acc, *path); + if (rc < 0) + goto err_url; + +#if 0 /* hutil */ + free(unesc_path); +#endif + return true; + + /* free(*url); */ + err_url: + free(*hosthdr); + err_host: + free(*path); +#if 0 /* hutil */ + err_epath: + free(unesc_path); +#endif + err_spath: + return false; +} + +static char *hstor_auth(struct hstor_client *hstor, struct http_req *req, + const char *url) +{ + int argc; + char **argv = NULL; + char *ret; + char *ohdr; + + argc = oauth_split_url_parameters(url, &argv); +{ int i; + printf("OAuth args\n"); + for (i = 0; i < argc; i++) { + printf(" %s\n", argv[i]); + } +} /* P3 */ + + oauth_sign_array2_process(&argc, &argv, + NULL, OA_HMAC, req->method, hstor->user, hstor->key, NULL, NULL); + + ohdr = oauth_serialize_url_sep(argc, 1, argv, ", ", 0x6); + + if (asprintf(&ret, "Authorization: OAuth %s", ohdr) < 0) { + oauth_free_array(&argc, &argv); + return NULL; + } + printf("request URL=%s\n", url); /* P3 */ + printf("request header=%s\n", ret); /* P3 */ + + oauth_free_array(&argc, &argv); + return ret; +} + +struct hstor_blist *hstor_list_buckets(struct hstor_client *hstor) +{ + struct http_req req; + char *host, *url, *auth; + struct curl_slist *headers = NULL; + struct api_buf apib; + struct hstor_blist *blist; + xmlDocPtr doc; + xmlNode *node; + int rc; + + memset(&apib, 0, sizeof(struct api_buf)); + apib.buf = malloc(4000); + if (!apib.buf) + goto err_data; + apib.alloc = 4000; + + memset(&req, 0, sizeof(req)); + req.method = "GET"; + req.orig_path = "/"; + + if (asprintf(&host, "Host: %s", hstor->acc) < 0) + goto err_host; + if (asprintf(&url, "http://%s/", hstor->acc) < 0) + goto err_url; + + auth = hstor_auth(hstor, &req, url); + if (!auth) + goto err_auth; + + headers = curl_slist_append(headers, host); + headers = curl_slist_append(headers, auth); + + curl_easy_reset(hstor->curl); + if (hstor->verbose) + curl_easy_setopt(hstor->curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(hstor->curl, CURLOPT_URL, url); + curl_easy_setopt(hstor->curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(hstor->curl, CURLOPT_ENCODING, ""); + curl_easy_setopt(hstor->curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(hstor->curl, CURLOPT_WRITEFUNCTION, api_wcb); + curl_easy_setopt(hstor->curl, CURLOPT_WRITEDATA, &apib); + curl_easy_setopt(hstor->curl, CURLOPT_TCP_NODELAY, 1); + + rc = curl_easy_perform(hstor->curl); + + curl_slist_free_all(headers); + + if (rc) + goto err_out; + + doc = xmlReadMemory(apib.buf, apib.used, "foo.xml", NULL, 0); + if (!doc) + goto err_out; + + node = xmlDocGetRootElement(doc); + if (!node) + goto err_out_doc; + + if (_strcmp(node->name, "api")) + goto err_out_doc; + + blist = calloc(1, sizeof(*blist)); + if (!blist) + goto err_out_doc; + + node = node->children; + while (node) { + if (node->type != XML_ELEMENT_NODE) { + node = node->next; + continue; + } + + if (!_strcmp(node->name, "link")) + hstor_parse_link(doc, node, blist); + + node = node->next; + } + + xmlFreeDoc(doc); + free(apib.buf); + free(auth); + free(url); + free(host); + + return blist; + +err_out_doc: + xmlFreeDoc(doc); +err_out: + free(auth); +err_auth: + free(url); +err_url: + free(host); +err_host: + free(apib.buf); +err_data: + return NULL; +} + +bool hstor_del_bucket(struct hstor_client *hstor, const char *name) +{ + struct http_req req; + char *host, *url, *orig_path, *auth; + struct curl_slist *headers = NULL; + int rc; + + if (!hstor_resplit(hstor, name, "", &url, &host, &orig_path)) + goto err_split; + + memset(&req, 0, sizeof(req)); + req.method = "DELETE"; + req.orig_path = orig_path; + + auth = hstor_auth(hstor, &req, url); + if (!auth) + goto err_auth; + + headers = curl_slist_append(headers, host); + headers = curl_slist_append(headers, auth); + + curl_easy_reset(hstor->curl); + if (hstor->verbose) + curl_easy_setopt(hstor->curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(hstor->curl, CURLOPT_URL, url); + curl_easy_setopt(hstor->curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(hstor->curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(hstor->curl, CURLOPT_CUSTOMREQUEST, req.method); + curl_easy_setopt(hstor->curl, CURLOPT_TCP_NODELAY, 1); + + rc = curl_easy_perform(hstor->curl); + + curl_slist_free_all(headers); + + free(auth); + free(url); + free(host); + free(orig_path); + return (rc == 0); + +err_auth: + free(url); + free(host); + free(orig_path); +err_split: + return false; +} + +bool hstor_add_bucket(struct hstor_client *hstor, const char *name) +{ + struct http_req req; + char *host, *url, *auth; + struct curl_slist *headers = NULL; + char *postbuf; + struct api_buf apib, apob; + int rc; + + if (asprintf(&postbuf, "name=%s", name) < 0) + goto err_postbuf; + + memset(&apob, 0, sizeof(struct api_buf)); + apob.buf = postbuf; + apob.alloc = strlen(postbuf); + + memset(&apib, 0, sizeof(struct api_buf)); + apib.buf = malloc(4000); + if (!apib.buf) + goto err_data; + apib.alloc = 4000; + + memset(&req, 0, sizeof(req)); + req.method = "POST"; + req.orig_path = "/_new"; + + if (asprintf(&host, "Host: %s", hstor->acc) < 0) + goto err_host; + if (asprintf(&url, "http://%s/_new", hstor->acc) < 0) + goto err_url; + + auth = hstor_auth(hstor, &req, url); + if (!auth) + goto err_auth; + + headers = curl_slist_append(headers, host); + headers = curl_slist_append(headers, auth); + + curl_easy_reset(hstor->curl); + if (hstor->verbose) + curl_easy_setopt(hstor->curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(hstor->curl, CURLOPT_URL, url); + curl_easy_setopt(hstor->curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(hstor->curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(hstor->curl, CURLOPT_WRITEFUNCTION, api_wcb); + curl_easy_setopt(hstor->curl, CURLOPT_WRITEDATA, &apib); + curl_easy_setopt(hstor->curl, CURLOPT_POST, 1); + curl_easy_setopt(hstor->curl, CURLOPT_POSTFIELDSIZE, apob.alloc); + curl_easy_setopt(hstor->curl, CURLOPT_READFUNCTION, api_rcb); + curl_easy_setopt(hstor->curl, CURLOPT_READDATA, &apob); + // curl_easy_setopt(hstor->curl, CURLOPT_TCP_NODELAY, 1); + + rc = curl_easy_perform(hstor->curl); + + curl_slist_free_all(headers); + + free(auth); + free(url); + free(host); + free(apib.buf); + free(postbuf); + return (rc == 0); + +err_auth: + free(url); +err_url: + free(host); +err_host: + free(apib.buf); +err_data: + free(postbuf); +err_postbuf: + return false; +} + +bool hstor_get(struct hstor_client *hstor, const char *bucket, const char *key, + size_t (*write_cb)(void *, size_t, size_t, void *), + void *user_data, bool want_headers) +{ + struct http_req req; + char *url, *host, *orig_path, *auth; + struct curl_slist *headers = NULL; + int rc; + + if (!hstor_resplit(hstor, bucket, key, &url, &host, &orig_path)) + goto err_split; + + memset(&req, 0, sizeof(req)); + req.method = "GET"; + req.orig_path = orig_path; + + auth = hstor_auth(hstor, &req, url); + if (!auth) + goto err_auth; + + headers = curl_slist_append(headers, host); + headers = curl_slist_append(headers, auth); + + curl_easy_reset(hstor->curl); + if (hstor->verbose) + curl_easy_setopt(hstor->curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(hstor->curl, CURLOPT_URL, url); + curl_easy_setopt(hstor->curl, CURLOPT_HEADER, want_headers ? 1 : 0); + curl_easy_setopt(hstor->curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(hstor->curl, CURLOPT_ENCODING, ""); + curl_easy_setopt(hstor->curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(hstor->curl, CURLOPT_WRITEFUNCTION, write_cb); + curl_easy_setopt(hstor->curl, CURLOPT_WRITEDATA, user_data); + curl_easy_setopt(hstor->curl, CURLOPT_TCP_NODELAY, 1); + + rc = curl_easy_perform(hstor->curl); + + curl_slist_free_all(headers); + free(auth); + free(url); + free(host); + free(orig_path); + return (rc == 0); + +err_auth: + free(url); + free(host); + free(orig_path); +err_split: + return false; +} + +void *hstor_get_inline(struct hstor_client *hstor, const char *bucket, + const char *key, bool want_headers, size_t *len) +{ + bool rcb; + void *mem; + struct api_buf apib; + + memset(&apib, 0, sizeof(struct api_buf)); + apib.buf = malloc(4000); + if (!apib.buf) + return NULL; + apib.alloc = 4000; + + rcb = hstor_get(hstor, bucket, key, api_wcb, &apib, want_headers); + if (!rcb) { + free(apib.buf); + return NULL; + } + + if (len) + *len = apib.used; + + mem = apib.buf; + return mem; +} + +bool hstor_put(struct hstor_client *hstor, const char *bucket, const char *key, + size_t (*read_cb)(void *, size_t, size_t, void *), + off_t len, void *user_data) +{ + struct http_req req; + char *host, *url, *orig_path, *auth; + char *uhdr_buf = NULL; + struct curl_slist *headers = NULL; + int rc = -1; + + if (!hstor_resplit(hstor, bucket, key, &url, &host, &orig_path)) + goto err_split; + + memset(&req, 0, sizeof(req)); + req.method = "PUT"; + req.orig_path = orig_path; + + auth = hstor_auth(hstor, &req, url); + if (!auth) + goto err_auth; + + headers = curl_slist_append(headers, host); + headers = curl_slist_append(headers, auth); + + curl_easy_reset(hstor->curl); + if (hstor->verbose) + curl_easy_setopt(hstor->curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(hstor->curl, CURLOPT_URL, url); + curl_easy_setopt(hstor->curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(hstor->curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(hstor->curl, CURLOPT_READFUNCTION, read_cb); + curl_easy_setopt(hstor->curl, CURLOPT_READDATA, user_data); + curl_easy_setopt(hstor->curl, CURLOPT_CUSTOMREQUEST, req.method); + curl_easy_setopt(hstor->curl, CURLOPT_UPLOAD, 1); + curl_easy_setopt(hstor->curl, CURLOPT_INFILESIZE_LARGE, + (curl_off_t)len); + curl_easy_setopt(hstor->curl, CURLOPT_TCP_NODELAY, 1); + + rc = curl_easy_perform(hstor->curl); + + curl_slist_free_all(headers); + free(url); + free(host); + free(orig_path); + free(uhdr_buf); + return (rc == 0); + + /* free(uhdr_buf); */ +err_auth: + free(url); + free(host); + free(orig_path); +err_split: + return false; +} + +struct hstor_put_info { + void *data; + off_t len; +}; + +bool hstor_put_inline(struct hstor_client *hstor, const char *bucket, const char *key, + void *data, off_t len) +{ + struct api_buf apob; + + memset(&apob, 0, sizeof(struct api_buf)); + apob.buf = data; + apob.alloc = len; + + return hstor_put(hstor, bucket, key, api_rcb, len, &apob); +} + +bool hstor_del(struct hstor_client *hstor, const char *bucket, const char *key) +{ + struct http_req req; + char *host, *url, *orig_path, *auth; + struct curl_slist *headers = NULL; + int rc; + + if (!hstor_resplit(hstor, bucket, key, &url, &host, &orig_path)) + goto err_split; + + memset(&req, 0, sizeof(req)); + req.method = "DELETE"; + req.orig_path = orig_path; + + auth = hstor_auth(hstor, &req, url); + if (!auth) + goto err_auth; + + headers = curl_slist_append(headers, host); + headers = curl_slist_append(headers, auth); + + curl_easy_reset(hstor->curl); + if (hstor->verbose) + curl_easy_setopt(hstor->curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(hstor->curl, CURLOPT_URL, url); + curl_easy_setopt(hstor->curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(hstor->curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(hstor->curl, CURLOPT_CUSTOMREQUEST, req.method); + curl_easy_setopt(hstor->curl, CURLOPT_TCP_NODELAY, 1); + + rc = curl_easy_perform(hstor->curl); + + curl_slist_free_all(headers); + free(auth); + free(url); + free(host); + free(orig_path); + return (rc == 0); + +err_auth: + free(url); + free(host); + free(orig_path); +err_split: + return false; +} + +void hstor_free_object(struct hstor_object *obj) +{ + if (!obj) + return; + + free(obj->key); + free(obj->time_mod); + free(obj->etag); + free(obj->storage); + free(obj->own_id); + free(obj->own_name); + free(obj); +} + +void hstor_free_keylist(struct hstor_keylist *keylist) +{ + if (!keylist) + return; + + free(keylist->name); + free(keylist->prefix); + free(keylist->marker); + free(keylist->delim); + + while (!SLIST_EMPTY(&keylist->contents)) { + struct hstor_object *obj = SLIST_FIRST(&keylist->contents); + SLIST_REMOVE_HEAD(&keylist->contents, clink); + hstor_free_object(obj); + } + + free(keylist); +} + +static void hstor_parse_key(xmlDocPtr doc, xmlNode *node, + struct hstor_keylist *keylist) +{ + struct hstor_object *obj; + xmlNode *txt; + + obj = calloc(1, sizeof(*obj)); + if (!obj) + return; + + node = node->children; + while (node) { + if (node->type != XML_ELEMENT_NODE) { + node = node->next; + continue; + } + + if (!_strcmp(node->name, "key")) { + txt = node->children; + if (txt && txt->type==XML_TEXT_NODE && txt->content) { + free(obj->key); + obj->key = strdup((char *)txt->content); + } else { + /* P3 */ fprintf(stderr, "no text for key\n"); + } + } else { + /* P3 */ fprintf(stderr, "not key %s\n", node->name); + } + node = node->next; + } + + if (obj->key) + if (keylist->contail) { + SLIST_INSERT_AFTER(keylist->contail, obj, clink); + } else { + SLIST_INSERT_HEAD(&keylist->contents, obj, clink); + keylist->contail = obj; + } + else + hstor_free_object(obj); +} + +struct hstor_keylist *hstor_keys(struct hstor_client *hstor, const char *bucket) +{ + struct http_req req; + char *host, *orig_path, *auth; + struct curl_slist *headers = NULL; + struct api_buf apib; + struct hstor_keylist *keylist; + xmlDocPtr doc; + xmlNode *node; + char *url; + int rc; + + if (!bucket) /* see hstor_list_buckets() for this */ + goto err_param; + + memset(&apib, 0, sizeof(struct api_buf)); + apib.buf = malloc(4000); + if (!apib.buf) + goto err_data; + apib.alloc = 4000; + + if (asprintf(&orig_path, "/%s/", bucket) < 0) + goto err_spath; + + memset(&req, 0, sizeof(req)); + req.method = "GET"; + req.orig_path = orig_path; + + if (asprintf(&host, "Host: %s", hstor->acc) < 0) + goto err_host; + + if (asprintf(&url, "http://%s%s", hstor->acc, orig_path) < 0) + goto err_url; + + auth = hstor_auth(hstor, &req, url); + if (!auth) +{ +/* P3 */ fprintf(stderr, "no auth\n"); + goto err_auth; +} + + headers = curl_slist_append(headers, host); + headers = curl_slist_append(headers, auth); + + curl_easy_reset(hstor->curl); + if (hstor->verbose) + curl_easy_setopt(hstor->curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(hstor->curl, CURLOPT_URL, url); + curl_easy_setopt(hstor->curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(hstor->curl, CURLOPT_ENCODING, ""); + curl_easy_setopt(hstor->curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(hstor->curl, CURLOPT_WRITEFUNCTION, api_wcb); + curl_easy_setopt(hstor->curl, CURLOPT_WRITEDATA, &apib); + curl_easy_setopt(hstor->curl, CURLOPT_TCP_NODELAY, 1); + + rc = curl_easy_perform(hstor->curl); + + curl_slist_free_all(headers); + + if (rc) +{ +/* P3 */ fprintf(stderr, "curl reported error\n"); + goto err_out; +} + + doc = xmlReadMemory(apib.buf, apib.used, "foo.xml", NULL, 0); + if (!doc) +{ +/* P3 */ fprintf(stderr, "no XML\n"); + goto err_out; +} + + node = xmlDocGetRootElement(doc); + if (!node) +{ +/* P3 */ fprintf(stderr, "no root\n"); + goto err_out_doc; +} + + if (_strcmp(node->name, "objects")) +{ +/* P3 */ fprintf(stderr, "no API %s\n", node->name); + goto err_out_doc; +} + + keylist = calloc(1, sizeof(*keylist)); + if (!keylist) + goto err_out_doc; + + node = node->children; + while (node) { + if (node->type != XML_ELEMENT_NODE) { + node = node->next; + continue; + } + + if (!_strcmp(node->name, "object")) + hstor_parse_key(doc, node, keylist); + else + /* P3 */ fprintf(stderr, "not object %s\n", node->name); + + node = node->next; + } + + xmlFreeDoc(doc); + free(auth); + free(url); + free(host); + free(orig_path); + free(apib.buf); + + return keylist; + +err_out_doc: + xmlFreeDoc(doc); +err_out: + free(auth); +err_auth: + free(url); +err_url: + free(host); +err_host: + free(orig_path); +err_spath: + free(apib.buf); +err_data: +err_param: + return NULL; +} + diff --git a/t/oauth b/t/oauth new file mode 100644 index 0000000..c014591 --- /dev/null +++ b/t/oauth @@ -0,0 +1,64 @@ +#!/bin/sh +# Test authentication functionality. + +. "${srcdir=.}/init.sh"; path_prepend_ .. + +fail=0 + +mkdir FS mongod iwhd || framework_failure_ mkdir failed + +# This actually mostly helps against weirdness with global variables. +# The makefile ensures that we only ever get here with the poke built. +# We have no idea why it's essential to use abs_top_builddir instead +# of top_builddir. It's a peculiarity of GNU auto-makefiles. +test -f ${abs_top_builddir}/t/poke \ + || fail_ "no $abs_top_builddir/t/poke" + +m_port=$(get_port $mongo_base_port $lock_dir/m-) \ + || fail_ "failed to get mongodb port" + +mongod --port $m_port --pidfilepath mongod/pid --dbpath mongod > mongod.log 2>&1 & +mongo_pid=$! +cleanup_() { kill -9 $mongo_pid; } + +# Wait for up to 5 seconds for mongod to begin listening. +wait_for .1 50 'mongo localhost:$m_port < /dev/null' \ + || framework_failure_ mongod failed to start + +port=$(get_port 9095 $lock_dir/i-) || fail_ "failed to get iwhd port" + +ulimit -c unlimited + +printf '[{"path": "FS", "type": "fs", "name": "primary"}]\n' \ + > iwhd.cfg || fail=1 + +iwhd -v -o -U chiaki:nodame \ + -p $port -c iwhd.cfg -d localhost:$m_port & +iwhd_pid=$! +cleanup_() { kill -9 $mongo_pid; kill $iwhd_pid; } + +## Wait for up to 5 seconds for iwhd to begin listening on $port. +#wait_for .1 50 "curl -s http://localhost:$port" \ +# || { echo iwhd failed to listen; Exit 1; } +sleep 5 + +cat <<EOF >object.data +Test +EOF + +${abs_top_builddir}/t/poke -h localhost:$port -u chiaki -p nodame -b buk -o || fail=1 + +${abs_top_builddir}/t/poke -h localhost:$port -u chiaki -p nodame -b buk -k p1 -o -f object.data || fail=1 + +${abs_top_builddir}/t/poke -h localhost:$port -u chiaki -p nodame -b buk -l || fail=1 + +${abs_top_builddir}/t/poke -h localhost:$port -u chiaki -p nodame -b buk -k p1 -i -f p1.data || fail=1 + +test "$(cat p1.data)" = "$(cat object.data)" || fail=1 + +# Now let's test a bad password. The iwhd also logs "OAuth signature mismatch", +# but parsing its log is perilous and is not really necessary. Just repeat +# the previous poke command exactly. +${abs_top_builddir}/t/poke -h localhost:$port -u chiaki -p badpass -b buk -k p1 -i -f p1.data && fail=1 + +Exit $fail diff --git a/t/poke.c b/t/poke.c new file mode 100644 index 0000000..1ce3106 --- /dev/null +++ b/t/poke.c @@ -0,0 +1,625 @@ +/* + * Copyright 2011 Red Hat, Inc. + * + * Test for authorized iwhd. + * If curl(1) supported OAuth, we would not need this. + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <stdbool.h> +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <assert.h> +#include <dirent.h> +#include <sys/queue.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include "hoa.h" + +enum poke_mode { + MODE_NUL, MODE_IN, MODE_OUT, MODE_DEL, MODE_LIST +}; + +struct params { + int mode; + bool verbose; + bool do_csum; + char *file; + char *host; /* netloc actually */ + char *user; + char *pass; + char *bucket; + char *key; +}; + +struct input_ctx { + struct params *par; + struct hstor_client *httpstor; + char *errbuf; + char *pfx; +}; + +struct put_ctx { + unsigned int csum; + bool do_csum; + int fd; + off_t off; + off_t total; +}; + +struct get_ctx { + unsigned int csum; + bool do_csum; + FILE *f; + off_t off; +}; + +#define BLKSZ 65536 + +#define CSUM_INIT 0xFFFFFFFF + +static void do_input(struct params *par, struct hstor_client *httpstor, char *errbuf); +static void input_one(struct params *par, struct hstor_client *httpstor, + char *errbuf, char *key, FILE *f, char *filename); +static void do_output(struct params *par, struct hstor_client *httpstor, char *errbuf); +static void output_dir(struct params *par, struct hstor_client *httpstor, + char *dirname, char *suffix); +static void output_name(struct params *par, struct hstor_client *httpstor, + char *basekey, char *basedir, char *suffix); +static void output_one(struct params *par, struct hstor_client *httpstor, + char *key, int fd, char *filename); +static void do_delete(struct params *par, struct hstor_client *httpstor, char *errbuf); +static void do_list(struct params *par, struct hstor_client *httpstor, char *errbuf); +static size_t put_cb(void *ptr, size_t membsize, size_t nmemb, void *user_data); +static size_t get_cb(void *ptr, size_t membsize, size_t nmemb, void *user_data); +static char *cat_fs(char *dir, char *file); +static char *cat(int n, char *v[]); +static int isdir(char *filename); +static void incrsum(unsigned int *psum, const unsigned char *data, size_t len); +static void parse_args(struct params *par, int argc, char **argv); +static void Usage(void); + +struct params pars; + +int main(int argc, char **argv) +{ + struct hstor_client *httpstor; + char *acc; + char *errbuf; + // bool rcb; + int rc; + + parse_args(&pars, argc, argv); + + acc = pars.host; + + httpstor = hstor_new(acc, pars.user, pars.pass); + if (httpstor == NULL) { + fprintf(stderr, "Unable to initialize a channel to %s\n", acc); + exit(1); + } + + if (pars.verbose) + httpstor->verbose = true; + + if ((errbuf = malloc(CURL_ERROR_SIZE)) == NULL) { + fprintf(stderr, "No core\n"); + exit(1); + } + strcpy(errbuf, "HAXX SCREWUP, USE -v"); + rc = curl_easy_setopt(httpstor->curl, CURLOPT_ERRORBUFFER, errbuf); + if (rc) { + fprintf(stderr, "curl_easy_setopt(CURLOPT_ERRORBUFFER): %d\n", rc); + } + + switch (pars.mode) { + case MODE_IN: + do_input(&pars, httpstor, errbuf); + break; + case MODE_OUT: + do_output(&pars, httpstor, errbuf); + break; + case MODE_DEL: + do_delete(&pars, httpstor, errbuf); + break; + case MODE_LIST: + do_list(&pars, httpstor, errbuf); + break; + default: + abort(); + } + + hstor_free(httpstor); + free(errbuf); + return 0; +} + +static void do_input(struct params *par, struct hstor_client *httpstor, char *errbuf) +{ + FILE *f; + + if (par->file) { + /* XXX open before opening connection to the server */ + if ((f = fopen(par->file, "wb")) == NULL) { + fprintf(stderr, "Unable to open %s: %s\n", + par->file, strerror(errno)); + exit(1); + } + input_one(par, httpstor, errbuf, par->key, f, par->file); + fclose(f); + } else { + input_one(par, httpstor, errbuf, par->key, stdout, "-"); + } +} + +static void input_one(struct params *par, struct hstor_client *httpstor, + char *errbuf, char *key, FILE *f, char *filename) +{ + struct get_ctx getctx; + + memset(&getctx, 0, sizeof(getctx)); + getctx.f = f; + getctx.do_csum = par->do_csum; + getctx.csum = CSUM_INIT; + + if (!hstor_get(httpstor, par->bucket, key, get_cb, &getctx, false)) { + fprintf(stderr, "Failed get, bucket %s key %s file %s\n", + par->bucket, key, filename); + /* delete the file? */ + exit(1); + } + + // if (getctx.off != total) foo + + if (par->do_csum) + printf("csum 0x%x\n", getctx.csum); +} + +static void do_output(struct params *par, struct hstor_client *httpstor, char *errbuf) +{ + int fd; + + if (par->key == NULL) { + if (!hstor_add_bucket(httpstor, par->bucket)) { + fprintf(stderr, "Failed to add bucket %s: %s\n", + par->bucket, errbuf); + exit(1); + } + return; + } + + if (isdir(par->file)) { + output_dir(par, httpstor, par->file, NULL); + return; + } + + if (par->file) { + if ((fd = open(par->file, O_RDONLY)) == -1) { + fprintf(stderr, "Unable to open %s: %s\n", + par->file, strerror(errno)); + exit(1); + } + output_one(par, httpstor, par->key, fd, par->file); + close(fd); + } else { + output_one(par, httpstor, par->key, 0, "-"); + } +} + +static void output_dir(struct params *par, struct hstor_client *httpstor, + char *dirname, char *suffix) +{ + DIR *dir; + struct dirent *ent; + char *fullname; + char *newdir; + char *newsuffix; + + dir = opendir(dirname); + if (!dir) { + fprintf(stderr, "Failed to open directory %s: %s\n", + dirname, strerror(errno)); + exit(1); + } + + while ((ent = readdir(dir)) != NULL) { + if (strcmp(ent->d_name, ".") == 0 || + strcmp(ent->d_name, "..") == 0) + continue; + + fullname = cat_fs(dirname, ent->d_name); + if (suffix && suffix[0] != 0) { + newsuffix = cat_fs(suffix, ent->d_name); + } else { + newsuffix = strdup(ent->d_name); + if (!newsuffix) { + fprintf(stderr, "No core\n"); + exit(1); + } + } + + if (ent->d_type == DT_UNKNOWN) { + if (isdir(fullname)) { + newdir = cat_fs(par->file, newsuffix); + output_dir(par, httpstor, newdir, newsuffix); + free(newdir); + } else { + output_name(par, httpstor, par->key, par->file, + newsuffix); + } + } else if (ent->d_type == DT_DIR) { + newdir = cat_fs(par->file, newsuffix); + output_dir(par, httpstor, newdir, newsuffix); + free(newdir); + } else { + output_name(par, httpstor, par->key, par->file, + newsuffix); + } + + free(fullname); + free(newsuffix); + } + + closedir(dir); +} + +static void output_name(struct params *par, struct hstor_client *httpstor, + char *basekey, char *basedir, char *suffix) +{ + int fd; + char *key; + char *file; + + key = cat_fs(basekey, suffix); + file = cat_fs(basedir, suffix); + if ((fd = open(file, O_RDONLY)) == -1) { + fprintf(stderr, "Unable to open %s: %s\n", + file, strerror(errno)); + exit(1); + } + output_one(par, httpstor, key, fd, file); + close(fd); + free(key); + free(file); +} + +static void output_one(struct params *par, struct hstor_client *httpstor, + char *key, int fd, char *filename) +{ + struct stat statb; + off_t total; + struct put_ctx putctx; + + if (fstat(fd, &statb) == -1) { + fprintf(stderr, "Unable to stat %s: %s\n", + filename, strerror(errno)); + exit(1); + } + if (!S_ISREG(statb.st_mode)) { + fprintf(stderr, "Not a regular file '%s'\n", filename); + exit(1); + } + total = statb.st_size; + + memset(&putctx, 0, sizeof(putctx)); + putctx.fd = fd; + putctx.do_csum = par->do_csum; + putctx.csum = CSUM_INIT; + putctx.total = total; + + if (!hstor_put(httpstor, par->bucket, key, put_cb, total, &putctx)) { + fprintf(stderr, + "Failed to put, bucket %s key %s file %s\n", + par->bucket, key, filename); + exit(1); + } + + if (putctx.off != total) { + fprintf(stderr, + "Short put, bucket %s key %s file %s off %ld\n", + par->bucket, key, filename, (long)putctx.off); + exit(1); + } + + if (par->do_csum) + printf("key %s csum 0x%08x\n", key, putctx.csum); +} + +static void do_delete(struct params *par, struct hstor_client *httpstor, char *errbuf) +{ + + if (par->key) { + if (!hstor_del(httpstor, par->bucket, par->key)) { + fprintf(stderr, + "Failed to delete, bucket %s key %s: %s\n", + par->bucket, par->key, errbuf); + exit(1); + } + } else { + if (!hstor_del_bucket(httpstor, par->bucket)) { + fprintf(stderr, "Failed to delete bucket %s: %s\n", + par->bucket, errbuf); + exit(1); + } + } +} + +/* + */ +static void do_list(struct params *par, struct hstor_client *httpstor, char *errbuf) +{ + struct hstor_keylist *list; + struct hstor_object *obj; + + list = hstor_keys(httpstor, par->bucket); + if (!list) { + fprintf(stderr, "Failed to list, prefix `%s'\n", + par->key? par->key: "(none)"); + exit(1); + } + + SLIST_FOREACH(obj, &list->contents, clink) { + /* XXX replace with etag when available. */ + printf("key `%s' size %ld\n", obj->key, (long)obj->size); + } + + hstor_free_keylist(list); +} + +static size_t put_cb(void *ptr, size_t membsize, size_t nmemb, void *user_data) +{ + struct put_ctx *ctx = user_data; + size_t len; + ssize_t rc; + + assert(membsize == 1); + + if (ctx->off >= ctx->total) + return 0; + + len = nmemb; + if (len > ctx->total - ctx->off) + len = ctx->total - ctx->off; + rc = read(ctx->fd, ptr, len); + if (rc < 0) { + fprintf(stderr, "read error: %s", strerror(errno)); + return -1; + } + if (rc < len) { + fprintf(stderr, "short read: %ld at %ld", rc, (long)ctx->off); + return -1; + } + if (ctx->do_csum) + incrsum(&ctx->csum, ptr, len); + + ctx->off += len; + return len; +} + +static size_t get_cb(void *ptr, size_t membsize, size_t nmemb, void *user_data) +{ + struct get_ctx *ctx = user_data; + size_t rc; + + assert(membsize == 1); + + if (ctx->do_csum) + incrsum(&ctx->csum, ptr, nmemb); + rc = fwrite(ptr, 1, nmemb, ctx->f); + if (rc < nmemb) { + if (ferror(ctx->f)) { + fprintf(stderr, "write error: %s", strerror(errno)); + return -1; + } + fprintf(stderr, "short write: %ld at %ld", + (long)rc, (long)ctx->off); + return -1; + } + + ctx->off += rc; + return rc; +} + +static char *cat_fs(char *dir, char *file) +{ + char *v[3]; + v[0] = dir; v[1] = "/"; v[2] = file; + return cat(3, v); +} + +static char *cat(int n, char *v[]) +{ + char *ret; + size_t len; + char *p; + int i; + + if (n == 0) + return NULL; + + len = 1; + for (i = 0; i < n; i++) + len += strlen(v[i]); + + ret = malloc(len); + if (!ret) { + fprintf(stderr, "No core\n"); + exit(1); + } + + p = ret; + for (i = 0; i < n; i++) { + len = strlen(v[i]); + if (len) { + memcpy(p, v[i], len); + p += len; + } + } + *p = 0; + return ret; +} + +static int isdir(char *filename) +{ + struct stat statb; + + if (filename == NULL) + return 0; + + if (stat(filename, &statb) == -1) { + if (errno == ENOENT) + return 0; + fprintf(stderr, "Unable to stat %s: %s\n", + filename, strerror(errno)); + exit(1); + } + + if (!S_ISDIR(statb.st_mode)) + return 0; + return 1; +} + +static void incrsum(unsigned int *psum, const unsigned char *data, size_t len) +{ + unsigned int sum; + + sum = *psum; + while (len) { + sum ^= *data; + sum = sum << 1 | sum >> 31; + data++; + --len; + } + *psum = sum; +} + +static void parse_args(struct params *par, int argc, char **argv) +{ + char *arg; + + memset(par, 0, sizeof(struct params)); + + ++argv; + while ((arg = *argv++) != NULL) { + if (arg[0] == '-') { + switch (arg[1]) { + case 'b': + if (*argv == NULL) + Usage(); + par->bucket = *argv++; + break; + case 'c': + par->do_csum = true; + break; + case 'd': + par->mode = MODE_DEL; + break; + case 'f': + if (*argv == NULL) + Usage(); + par->file = *argv++; + break; + case 'h': + if (*argv == NULL) + Usage(); + par->host = *argv++; + break; + case 'i': + par->mode = MODE_IN; + break; + case 'k': + if (*argv == NULL) + Usage(); + par->key = *argv++; + break; + case 'l': + par->mode = MODE_LIST; + break; + case 'o': + par->mode = MODE_OUT; + break; + case 'p': + /* + * Of course this is a bad idea since other + * users can snoop it with ps, but whatever. + */ + if (*argv == NULL) + Usage(); + par->pass = *argv++; + break; + case 'u': + if (*argv == NULL) + Usage(); + par->user = *argv++; + break; + case 'v': + par->verbose = true; + break; + default: + Usage(); + } + } else { + Usage(); + } + } + + if (par->mode != MODE_IN && par->mode != MODE_OUT && + par->mode != MODE_DEL && par->mode != MODE_LIST) { + fprintf(stderr, "Mode (-i/-o/-d/-l) is missing\n"); + Usage(); + } + if (par->host == NULL) { + fprintf(stderr, "Host (-h) is missing\n"); + Usage(); + } + if (par->bucket == NULL) { + fprintf(stderr, "Bucket (-b) is missing\n"); + Usage(); + } + if (par->mode == MODE_IN && par->key == NULL) { + fprintf(stderr, "Key (-k) is missing\n"); + Usage(); + } + if (par->mode == MODE_OUT && par->key == NULL && par->file != NULL) { + fprintf(stderr, "Bucket creation needs no file argument\n"); + Usage(); + } + if (par->mode == MODE_DEL && par->file != NULL) { + fprintf(stderr, "Deleting needs no file argument\n"); + Usage(); + } + if (par->mode == MODE_LIST && par->file != NULL) { + fprintf(stderr, "Listing needs no file argument\n"); + Usage(); + } + if (par->user == NULL) { + fprintf(stderr, "User (-u) is missing\n"); + Usage(); + } + if (par->pass == NULL) { + fprintf(stderr, "Password (-p) is missing\n"); + Usage(); + } +} + +static void Usage(void) +{ + fprintf(stderr, "Usage: poke [-i|-o|-d|-l] {options}\n" + "Modes:\n" + " -i Input (from server)\n" + " -o Output (to server)\n" + " -d Delete\n" + " -l List\n" + "Options:\n" + " -h host host[:port]\n" + " -u user, -p password\n" + " -b bucket\n" + " -k key use filename-like keys, no slashes;" + " empty and none are different\n" + " -f file filename for I/O data\n"); + exit(2); +} diff --git a/user.c b/user.c new file mode 100644 index 0000000..33f92d9 --- /dev/null +++ b/user.c @@ -0,0 +1,69 @@ +/* Copyright (C) 2010-2011 Red Hat, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see http://www.gnu.org/licenses/. */ + +#include <config.h> + +#include <error.h> +#include <stdlib.h> +#include <string.h> + +#include "iwh.h" /* _() */ +#include "user.h" + +static const char *username, *userpass; + +struct user * +user_lookup (const char *userid) +{ + /* + * We do not have a real user table yet, so just compare with -U. + */ + if (username == NULL || strcmp(userid, username) != 0) + return NULL; + + struct user *ret = malloc(sizeof *ret); + if (!ret) + return NULL; + memset(ret,0,sizeof *ret); + strcpy(ret->name, username); + strcpy(ret->pass, userpass); + return ret; +} + +void +user_add(const char *usercred) +{ + const char *p = strchr(usercred,':'); + if (!p) { + error (EXIT_FAILURE, 0, + _("The -U argument must contain a colon")); + } + if (p == usercred) { + error (EXIT_FAILURE, 0, + _("The user part of -U argument is empty")); + } + username = strndup(usercred,p-usercred); + userpass = strdup(p+1); + if (!username || !userpass) { + error (EXIT_FAILURE, 0, _("No core")); + } + if (strlen(username) > USERNAME_MAX) { + error (EXIT_FAILURE, 0, _("User name too long")); + } + if (strlen(userpass) > USERPASS_MAX) { + error (EXIT_FAILURE, 0, _("User password too long")); + } +} + diff --git a/user.h b/user.h new file mode 100644 index 0000000..b4e8bab --- /dev/null +++ b/user.h @@ -0,0 +1,29 @@ +/* Copyright (C) 2011 Red Hat, Inc. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see http://www.gnu.org/licenses/. */ + +#ifndef _USER_H +#define _USER_H + +#define USERNAME_MAX 64 +#define USERPASS_MAX 64 +struct user { + char name[USERNAME_MAX+1]; + char pass[USERPASS_MAX+1]; +}; + +extern struct user *user_lookup (const char *userid); +extern void user_add (const char *usercred); + +#endif /* _USER_H */