From 20c38fca579b329be00550aad3e7a38d5082f42c Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 25 Jan 2021 23:33:05 +0100 Subject: [PATCH 001/121] Add link to new release --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index de96ce3..3dd87d8 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Kore only supports x64, arm and aarch64 architectures. Building Kore ------------- -Clone this repository or get the latest release at [https://kore.io/releases/4.0.0](https://kore.io/releases/4.0.0). +Clone this repository or get the latest release at [https://kore.io/releases/4.1.0](https://kore.io/releases/4.1.0). Requirements * openssl (1.0.2, 1.1.0 or 1.1.1) From c36a7759f11782f6e1d7cb9b84d5d6d5f1460c9e Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Thu, 28 Jan 2021 10:17:02 +0100 Subject: [PATCH 002/121] Get Kore to build with Python 3.10.0a3. --- src/python.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/python.c b/src/python.c index c8cd755..577f158 100644 --- a/src/python.c +++ b/src/python.c @@ -1037,7 +1037,13 @@ python_coro_run(struct python_coro *coro) python_coro_trace("running", coro); PyErr_Clear(); +#if PY_VERSION_HEX < 0x030a00a1 item = _PyGen_Send((PyGenObject *)coro->obj, NULL); +#else + /* Depend on the result in item only. */ + (void)PyIter_Send(coro->obj, NULL, &item); +#endif + if (item == NULL) { if (coro->gatherop == NULL && PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) { From 0549295f30008897a95e1d2e034459dc39b0491a Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Thu, 28 Jan 2021 13:34:43 +0100 Subject: [PATCH 003/121] Cleanup integer types for # Python arguments. Define PY_SSIZE_T_CLEAN before pulling in Python.h --- include/kore/python_api.h | 2 ++ src/python.c | 20 ++++++-------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/include/kore/python_api.h b/include/kore/python_api.h index 4d680a7..21ce053 100644 --- a/include/kore/python_api.h +++ b/include/kore/python_api.h @@ -21,6 +21,8 @@ #undef _POSIX_C_SOURCE #undef _XOPEN_SOURCE +#define PY_SSIZE_T_CLEAN 1 + #include #include diff --git a/src/python.c b/src/python.c index 577f158..8cafd27 100644 --- a/src/python.c +++ b/src/python.c @@ -4552,12 +4552,7 @@ pyhttp_body_read(struct pyhttp_request *pyreq, PyObject *args) return (NULL); } - if (ret > INT_MAX) { - PyErr_SetString(PyExc_RuntimeError, "ret > INT_MAX"); - return (NULL); - } - - result = Py_BuildValue("ny#", ret, buf, (int)ret); + result = Py_BuildValue("ny#", ret, buf, ret); if (result == NULL) return (PyErr_NoMemory()); @@ -4676,12 +4671,7 @@ pyhttp_file_read(struct pyhttp_file *pyfile, PyObject *args) return (NULL); } - if (ret > INT_MAX) { - PyErr_SetString(PyExc_RuntimeError, "ret > INT_MAX"); - return (NULL); - } - - result = Py_BuildValue("ny#", ret, buf, (int)ret); + result = Py_BuildValue("ny#", ret, buf, ret); if (result == NULL) return (PyErr_NoMemory()); @@ -4725,8 +4715,9 @@ pyhttp_websocket_handshake(struct pyhttp_request *pyreq, PyObject *args) static PyObject * pyconnection_websocket_send(struct pyconnection *pyc, PyObject *args) { + int op; + ssize_t len; const char *data; - int op, len; if (pyc->c->proto != CONN_PROTO_WEBSOCKET) { PyErr_SetString(PyExc_TypeError, "not a websocket connection"); @@ -4761,10 +4752,11 @@ static PyObject * python_websocket_broadcast(PyObject *self, PyObject *args) { struct connection *c; + ssize_t len; struct pyconnection *pyc; const char *data; PyObject *pysrc; - int op, broadcast, len; + int op, broadcast; len = -1; From 26fdbc40304f8bf6445d2dd34bd70d6940d0592a Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 31 Jan 2021 14:54:26 +0100 Subject: [PATCH 004/121] Add -fno-common to CFLAGS. Newer compilers have this enabled by default, instead of -fcommon so enable it here to catch any problems early. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 22fbb48..12430f0 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ FEATURES_INC= CFLAGS+=-Wall -Werror -Wstrict-prototypes -Wmissing-prototypes CFLAGS+=-Wmissing-declarations -Wshadow -Wpointer-arith -Wcast-qual CFLAGS+=-Wsign-compare -Iinclude/kore -I$(OBJDIR) -std=c99 -pedantic -CFLAGS+=-Wtype-limits +CFLAGS+=-Wtype-limits -fno-common CFLAGS+=-DPREFIX='"$(PREFIX)"' -fstack-protector-all ifneq ("$(OPENSSL_PATH)", "") From 25d47db9e7da1a1abfbaa3da55aae5c4e62bc65e Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 9 Mar 2021 15:13:45 +0100 Subject: [PATCH 005/121] Let VERSION depend on OBJDIR. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 12430f0..a53bdba 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,7 @@ $(PLATFORM): $(OBJDIR) force $(PYTHON_CURLOPT): $(OBJDIR) force @cp $(PYTHON_CURLOPT) $(OBJDIR) -$(VERSION): force +$(VERSION): $(OBJDIR) force @if [ -d .git ]; then \ GIT_REVISION=`git rev-parse --short=8 HEAD`; \ GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`; \ From 754eab5b4b50f8b0561f5b7a1c9dd4734b51c009 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 10 Mar 2021 17:30:01 +0100 Subject: [PATCH 006/121] sync config with reality --- conf/kore.conf.example | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/conf/kore.conf.example b/conf/kore.conf.example index d535717..501a1fe 100644 --- a/conf/kore.conf.example +++ b/conf/kore.conf.example @@ -189,12 +189,13 @@ validator v_regex regex ^/test/[a-z]*$ validator v_number regex ^[0-9]*$ validator v_session function v_session_validate -# Specify what TLS version to be used. Default is TLSv1.2 +# Specify what TLS version to be used. Default is TLSv1.3 if available. +# Otherwise it will use TLS 1.2. # Allowed values: -# 1.2 for TLSv1.2 (default) -# 1.0 for TLSv1.0 -# both for TLSv1.0 and TLSv1.2 -#tls_version 1.2 +# 1.3 for TLSv1.3 (default, if available) +# 1.2 for TLSv1.2 +# both for TLSv1.2 and TLSv1.3 +#tls_version 1.3 # Specify the TLS ciphers that will be used. #tls_cipher ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK:!kRSA:!kDSA From 85b93f7957f224f4d4e605f7e9a76f0724e08794 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 23 Mar 2021 15:03:47 +0100 Subject: [PATCH 007/121] KORE_SECCOMP_FILTER() does not require a semicolumn. It's a wrapper around a function. --- include/kore/seccomp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/kore/seccomp.h b/include/kore/seccomp.h index 5f3b790..7d95ced 100644 --- a/include/kore/seccomp.h +++ b/include/kore/seccomp.h @@ -145,7 +145,7 @@ * KORE_SYSCALL_DENY_ERRNO(socket, EACCESS), * KORE_SYSCALL_DENY_ERRNO(ioctl, EACCESS), * KORE_SYSCALL_ALLOW(poll), - * ); + * ) */ #define KORE_SECCOMP_FILTER(name, ...) \ struct sock_filter _scfilt[] = { \ From 0031f0271eb23f1fb29de7cee97ca63c9e6584e3 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 26 Mar 2021 10:54:47 +0100 Subject: [PATCH 008/121] update counts to reflect reality much beer is owed. --- BEERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BEERS b/BEERS index f5e7da7..cb62a73 100644 --- a/BEERS +++ b/BEERS @@ -17,7 +17,7 @@ I will note down the beer of your choice between the brackets. [] Daniel Fahlgren x 2 [] Dmitrii Golub [] Elliot Schlegelmilch -[] Erik Karlsson +[] Erik Karlsson x 2 [] Frederic Cambus [] Guy Nankivell [] James Turner @@ -30,7 +30,7 @@ I will note down the beer of your choice between the brackets. [] Raphaël Monrouzeau [] Raymond Pasco [] Remy Noulin -[] Rickard Lind +[] Rickard Lind x 2 [] Shih-Yuan Lee [] Stanislav Yudin [] Stig Telfer From 9cfcd9a4be6e17c24f606440fbd2c224aac892e8 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 30 Mar 2021 14:19:48 +0200 Subject: [PATCH 009/121] JSON API improvements. - Try harder to mark integers as KORE_JSON_TYPE_INTEGER, especially if they fit in the internal representation of one (int64_t). - Move error codes into the JSON code itself, rather then requiring a kore_json data structure. This allows the JSON API to relay errors such as "item not found" or "type mismatch" properly when looking at items. - When asking for a KORE_JSON_TYPE_INTEGER_U64 and a KORE_JSON_TYPE_INTEGER was found with the same name, check if it could be returned properly and do so if possible. --- examples/json/src/json.c | 14 +++++- include/kore/kore.h | 6 +-- src/json.c | 98 ++++++++++++++++++++++++++++------------ 3 files changed, 84 insertions(+), 34 deletions(-) diff --git a/examples/json/src/json.c b/examples/json/src/json.c index f5d3bcd..9370e53 100755 --- a/examples/json/src/json.c +++ b/examples/json/src/json.c @@ -30,14 +30,24 @@ page(struct http_request *req) kore_json_init(&json, req->http_body->data, req->http_body->length); if (!kore_json_parse(&json)) { - kore_buf_appendf(&buf, "%s\n", kore_json_strerror(&json)); + kore_buf_appendf(&buf, "%s\n", kore_json_strerror()); } else { item = kore_json_find_string(json.root, "foo/bar"); if (item != NULL) { kore_buf_appendf(&buf, "foo.bar = '%s'\n", item->data.string); } else { - kore_buf_appendf(&buf, "string foo.bar not found\n"); + kore_buf_appendf(&buf, "foo.bar %s\n", + kore_json_strerror()); + } + + item = kore_json_find_integer_u64(json.root, "foo/integer"); + if (item != NULL) { + kore_buf_appendf(&buf, + "foo.integer = '%" PRIu64 "'\n", item->data.u64); + } else { + kore_buf_appendf(&buf, "foo.integer %s\n", + kore_json_strerror()); } } diff --git a/include/kore/kore.h b/include/kore/kore.h index b38c0c7..8f66ad3 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -570,7 +570,6 @@ struct kore_buf { struct kore_json { const u_int8_t *data; int depth; - int error; size_t length; size_t offset; @@ -588,7 +587,7 @@ struct kore_json_item { char *string; double number; int literal; - int64_t s64; + int64_t integer; u_int64_t u64; } data; @@ -1038,13 +1037,14 @@ void kore_buf_appendv(struct kore_buf *, const char *, va_list); void kore_buf_replace_string(struct kore_buf *, const char *, const void *, size_t); +int kore_json_errno(void); int kore_json_parse(struct kore_json *); void kore_json_cleanup(struct kore_json *); void kore_json_item_free(struct kore_json_item *); void kore_json_init(struct kore_json *, const void *, size_t); void kore_json_item_tobuf(struct kore_json_item *, struct kore_buf *); -const char *kore_json_strerror(struct kore_json *); +const char *kore_json_strerror(void); struct kore_json_item *kore_json_find(struct kore_json_item *, const char *, u_int32_t); struct kore_json_item *kore_json_create_item(struct kore_json_item *, diff --git a/src/json.c b/src/json.c index fc3238d..94625d6 100644 --- a/src/json.c +++ b/src/json.c @@ -49,6 +49,8 @@ static u_int8_t json_null_literal[] = { 'n', 'u', 'l', 'l' }; static u_int8_t json_true_literal[] = { 't', 'r', 'u', 'e' }; static u_int8_t json_false_literal[] = { 'f', 'a', 'l', 's', 'e' }; +static int json_errno = 0; + static const char *json_errtab[] = { "no error", "invalid JSON object", @@ -84,8 +86,10 @@ kore_json_parse(struct kore_json *json) if (json->root) return (KORE_RESULT_OK); + json_errno = 0; + if (json_consume_whitespace(json) == -1) { - json->error = KORE_JSON_ERR_INVALID_JSON; + json_errno = KORE_JSON_ERR_INVALID_JSON; return (KORE_RESULT_ERROR); } @@ -93,22 +97,22 @@ kore_json_parse(struct kore_json *json) return (KORE_RESULT_ERROR); if (!json_guess_type(ch, &type)) { - json->error = KORE_JSON_ERR_INVALID_JSON; + json_errno = KORE_JSON_ERR_INVALID_JSON; return (KORE_RESULT_ERROR); } json->root = json_item_alloc(type, NULL, NULL); if (!json->root->parse(json, json->root)) { - if (json->error == 0) - json->error = KORE_JSON_ERR_INVALID_JSON; + if (json_errno == 0) + json_errno = KORE_JSON_ERR_INVALID_JSON; return (KORE_RESULT_ERROR); } /* Don't allow garbage at the end. */ (void)json_consume_whitespace(json); if (json->offset != json->length) { - json->error = KORE_JSON_ERR_INVALID_JSON; + json_errno = KORE_JSON_ERR_INVALID_JSON; return (KORE_RESULT_ERROR); } @@ -122,16 +126,21 @@ kore_json_find(struct kore_json_item *root, const char *path, u_int32_t type) char *copy; char *tokens[KORE_JSON_DEPTH_MAX + 1]; + json_errno = 0; copy = kore_strdup(path); if (!kore_split_string(copy, "/", tokens, KORE_JSON_DEPTH_MAX)) { kore_free(copy); + json_errno = KORE_JSON_ERR_INVALID_SEARCH; return (NULL); } item = json_find_item(root, tokens, type, 0); kore_free(copy); + if (item == NULL && json_errno == 0) + json_errno = KORE_JSON_ERR_INVALID_SEARCH; + return (item); } @@ -145,11 +154,17 @@ kore_json_cleanup(struct kore_json *json) kore_json_item_free(json->root); } -const char * -kore_json_strerror(struct kore_json *json) +int +kore_json_errno(void) { - if (json->error >= 0 && json->error <= KORE_JSON_ERR_LAST) - return (json_errtab[json->error]); + return (json_errno); +} + +const char * +kore_json_strerror(void) +{ + if (json_errno >= 0 && json_errno <= KORE_JSON_ERR_LAST) + return (json_errtab[json_errno]); return ("unknown JSON error"); } @@ -182,7 +197,7 @@ kore_json_create_item(struct kore_json_item *parent, const char *name, item->data.number = va_arg(args, double); break; case KORE_JSON_TYPE_INTEGER: - item->data.s64 = va_arg(args, int64_t); + item->data.integer = va_arg(args, int64_t); break; case KORE_JSON_TYPE_INTEGER_U64: item->data.u64 = va_arg(args, u_int64_t); @@ -248,7 +263,7 @@ kore_json_item_tobuf(struct kore_json_item *item, struct kore_buf *buf) kore_buf_appendf(buf, "%f", item->data.number); break; case KORE_JSON_TYPE_INTEGER: - kore_buf_appendf(buf, "%" PRId64, item->data.s64); + kore_buf_appendf(buf, "%" PRId64, item->data.integer); break; case KORE_JSON_TYPE_INTEGER_U64: kore_buf_appendf(buf, "%" PRIu64, item->data.u64); @@ -321,15 +336,33 @@ json_find_item(struct kore_json_item *object, char **tokens, break; } - if (nitem == NULL) + if (nitem == NULL) { + json_errno = KORE_JSON_ERR_NOT_FOUND; return (NULL); + } item = nitem; } if (tokens[pos + 1] == NULL) { + /* + * If an uint64 was required and we find an item + * with the same name but marked as an integer check + * if it can be represented as a uint64. + * + * If it can, reduce the type to integer so we match + * on it as well. + */ + if (type == KORE_JSON_TYPE_INTEGER_U64 && + item->type == KORE_JSON_TYPE_INTEGER) { + if (item->data.integer >= 0) + type = KORE_JSON_TYPE_INTEGER; + } + if (item->type == type) return (item); + + json_errno = KORE_JSON_ERR_TYPE_MISMATCH; return (NULL); } @@ -343,6 +376,9 @@ json_find_item(struct kore_json_item *object, char **tokens, break; } + if (item == NULL && json_errno == 0) + json_errno = KORE_JSON_ERR_NOT_FOUND; + return (item); } @@ -443,7 +479,7 @@ static int json_next_byte(struct kore_json *json, u_int8_t *ch, int peek) { if (json->offset >= json->length) { - json->error = KORE_JSON_ERR_EOF; + json_errno = KORE_JSON_ERR_EOF; return (KORE_RESULT_ERROR); } @@ -513,7 +549,7 @@ json_parse_object(struct kore_json *json, struct kore_json_item *object) int ret, hasnext; if (json->depth++ >= KORE_JSON_DEPTH_MAX) { - json->error = KORE_JSON_ERR_DEPTH; + json_errno = KORE_JSON_ERR_DEPTH; return (KORE_RESULT_ERROR); } @@ -537,7 +573,7 @@ json_parse_object(struct kore_json *json, struct kore_json_item *object) switch (ch) { case '}': if (hasnext) { - json->error = KORE_JSON_ERR_INVALID_JSON; + json_errno = KORE_JSON_ERR_INVALID_JSON; goto cleanup; } json->offset++; @@ -596,8 +632,8 @@ json_parse_object(struct kore_json *json, struct kore_json_item *object) } cleanup: - if (ret == KORE_RESULT_ERROR && json->error == 0) - json->error = KORE_JSON_ERR_INVALID_OBJECT; + if (ret == KORE_RESULT_ERROR && json_errno == 0) + json_errno = KORE_JSON_ERR_INVALID_OBJECT; json->depth--; @@ -614,7 +650,7 @@ json_parse_array(struct kore_json *json, struct kore_json_item *array) int ret, hasnext; if (json->depth++ >= KORE_JSON_DEPTH_MAX) { - json->error = KORE_JSON_ERR_DEPTH; + json_errno = KORE_JSON_ERR_DEPTH; return (KORE_RESULT_ERROR); } @@ -637,7 +673,7 @@ json_parse_array(struct kore_json *json, struct kore_json_item *array) if (ch == ']') { if (hasnext) { - json->error = KORE_JSON_ERR_INVALID_JSON; + json_errno = KORE_JSON_ERR_INVALID_JSON; goto cleanup; } json->offset++; @@ -675,8 +711,8 @@ json_parse_array(struct kore_json *json, struct kore_json_item *array) } cleanup: - if (ret == KORE_RESULT_ERROR && json->error == 0) - json->error = KORE_JSON_ERR_INVALID_ARRAY; + if (ret == KORE_RESULT_ERROR && json_errno == 0) + json_errno = KORE_JSON_ERR_INVALID_ARRAY; json->depth--; @@ -762,10 +798,14 @@ json_parse_number(struct kore_json *json, struct kore_json_item *number) kore_strtodouble(str, -DBL_MAX, DBL_MAX, &ret); break; case KORE_JSON_TYPE_INTEGER: - number->data.s64 = (int64_t)kore_strtonum64(str, 1, &ret); + number->data.integer = (int64_t)kore_strtonum64(str, 1, &ret); break; case KORE_JSON_TYPE_INTEGER_U64: - number->data.s64 = kore_strtonum64(str, 0, &ret); + number->data.u64 = kore_strtonum64(str, 0, &ret); + if (number->data.u64 <= INT64_MAX) { + type = KORE_JSON_TYPE_INTEGER; + number->data.integer = number->data.u64; + } break; default: goto cleanup; @@ -774,8 +814,8 @@ json_parse_number(struct kore_json *json, struct kore_json_item *number) number->type = type; cleanup: - if (ret == KORE_RESULT_ERROR && json->error == 0) - json->error = KORE_JSON_ERR_INVALID_NUMBER; + if (ret == KORE_RESULT_ERROR && json_errno == 0) + json_errno = KORE_JSON_ERR_INVALID_NUMBER; return (ret); } @@ -826,8 +866,8 @@ json_parse_literal(struct kore_json *json, struct kore_json_item *literal) ret = KORE_RESULT_OK; cleanup: - if (ret == KORE_RESULT_ERROR && json->error == 0) - json->error = KORE_JSON_ERR_INVALID_LITERAL; + if (ret == KORE_RESULT_ERROR && json_errno == 0) + json_errno = KORE_JSON_ERR_INVALID_LITERAL; return (ret); } @@ -895,8 +935,8 @@ json_get_string(struct kore_json *json) res = kore_buf_stringify(&json->tmpbuf, NULL); cleanup: - if (res == NULL && json->error == 0) - json->error = KORE_JSON_ERR_INVALID_STRING; + if (res == NULL && json_errno == 0) + json_errno = KORE_JSON_ERR_INVALID_STRING; return (res); } From a160a9e7ff324b767ea54793d0a56212362a4ae8 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 7 Apr 2021 14:51:11 +0200 Subject: [PATCH 010/121] Use the correct directive for unix binds. --- conf/kore.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/kore.conf.example b/conf/kore.conf.example index 501a1fe..41cd3ec 100644 --- a/conf/kore.conf.example +++ b/conf/kore.conf.example @@ -12,7 +12,7 @@ # Server configuration. server tls { bind 127.0.0.1 443 - #bind_unix /var/run/kore.sock + #unix /var/run/kore.sock } #server notls { From 81a09a04d650b8274745a56bb71ffe3d1fb81f62 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 7 Apr 2021 14:52:41 +0200 Subject: [PATCH 011/121] Use route instead of old static/dynamic. --- conf/kore.conf.example | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/conf/kore.conf.example b/conf/kore.conf.example index 41cd3ec..971cb45 100644 --- a/conf/kore.conf.example +++ b/conf/kore.conf.example @@ -299,24 +299,24 @@ domain localhost { accesslog /var/log/kore_access.log # Page handlers with no authentication required. - static /css/style.css serve_style_css - static / serve_index - static /intro.jpg serve_intro - static /b64test serve_b64test - static /upload serve_file_upload - static /lock-test serve_lock_test - static /validator serve_validator - static /params-test serve_params_test - static /private serve_private + route /css/style.css serve_style_css + route / serve_index + route /intro.jpg serve_intro + route /b64test serve_b64test + route /upload serve_file_upload + route /lock-test serve_lock_test + route /validator serve_validator + route /params-test serve_params_test + route /private serve_private # Restrict some URIs to certain methods restrict /private post restrict /validator post get head # Page handlers with authentication. - static /private/test serve_private_test auth_example + route /private/test serve_private_test auth_example - # Allow access to files from the directory static_files via + # Allow access to files from the directory route_files via # the /files/ URI. # # Note the directory given must be relative to the root configuration @@ -370,7 +370,7 @@ domain localhost { # client_verify /other/ca.crt # client_verify_depth 1 -# static /css/style.css serve_style_css -# static / serve_index -# dynamic ^/[a-z0-9_]*$ serve_profile +# route /css/style.css serve_style_css +# route / serve_index +# route ^/[a-z0-9_]*$ serve_profile #} From cc276e247179fb59c0dcd530250dd72fb8dd2b80 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Thu, 8 Apr 2021 09:10:58 +0200 Subject: [PATCH 012/121] Add kore_json_item_attach(). Allows a JSON subtree to be engrafted after creation. from Joel Arbring via patches@ --- include/kore/kore.h | 1 + src/json.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/include/kore/kore.h b/include/kore/kore.h index 8f66ad3..493d593 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -1043,6 +1043,7 @@ void kore_json_cleanup(struct kore_json *); void kore_json_item_free(struct kore_json_item *); void kore_json_init(struct kore_json *, const void *, size_t); void kore_json_item_tobuf(struct kore_json_item *, struct kore_buf *); +void kore_json_item_attach(struct kore_json_item *, struct kore_json_item *); const char *kore_json_strerror(void); struct kore_json_item *kore_json_find(struct kore_json_item *, diff --git a/src/json.c b/src/json.c index 94625d6..034e2e6 100644 --- a/src/json.c +++ b/src/json.c @@ -292,6 +292,24 @@ kore_json_item_tobuf(struct kore_json_item *item, struct kore_buf *buf) } } +void +kore_json_item_attach(struct kore_json_item *parent, + struct kore_json_item *item) +{ + if (item->parent != NULL) + fatal("%s: item already has parent", __func__); + + item->parent = parent; + + if (parent->type != KORE_JSON_TYPE_OBJECT && + parent->type != KORE_JSON_TYPE_ARRAY) { + fatal("%s: invalid parent type (%d)", + __func__, parent->type); + } + + TAILQ_INSERT_TAIL(&parent->data.items, item, list); +} + static struct kore_json_item * json_find_item(struct kore_json_item *object, char **tokens, u_int32_t type, int pos) From b6570e10a9694cbc8f46f23a715425cd20ad06e7 Mon Sep 17 00:00:00 2001 From: Frederic Cambus Date: Wed, 7 Apr 2021 11:57:45 +0200 Subject: [PATCH 013/121] Do not send the HSTS header if tls is not enabled in the server context. --- src/http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http.c b/src/http.c index e2871f2..15aa9ef 100644 --- a/src/http.c +++ b/src/http.c @@ -2138,7 +2138,7 @@ http_response_normal(struct http_request *req, struct connection *c, } } - if (http_hsts_enable) { + if (c->ssl && http_hsts_enable) { kore_buf_appendf(header_buf, "strict-transport-security: "); kore_buf_appendf(header_buf, "max-age=%" PRIu64 "; includeSubDomains\r\n", From 44bffcb1c93e16129120d768d95bd0af96c5e35f Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Thu, 8 Apr 2021 09:25:19 +0200 Subject: [PATCH 014/121] Unix socket path improvements. - Unlink the socket path if possible before attempting to bind to it. - Unlink the socket path if possible when Kore is shutting down. inspired by a diff from Joel Arbring via patches@ --- src/kore.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/kore.c b/src/kore.c index 238303c..9963140 100644 --- a/src/kore.c +++ b/src/kore.c @@ -497,6 +497,15 @@ kore_server_bind_unix(struct kore_server *srv, const char *path, if (!kore_listener_init(l, AF_UNIX, ccb)) return (KORE_RESULT_ERROR); + if (sun.sun_path[0] != '\0') { + if (unlink(sun.sun_path) == -1 && errno != ENOENT) { + kore_log(LOG_ERR, "unlink: %s: %s", + sun.sun_path, errno_s); + kore_listener_free(l); + return (KORE_RESULT_ERROR); + } + } + if (bind(l->fd, (struct sockaddr *)&sun, socklen) == -1) { kore_log(LOG_ERR, "bind: %s", errno_s); kore_listener_free(l); @@ -657,11 +666,30 @@ kore_server_free(struct kore_server *srv) void kore_listener_free(struct listener *l) { + int rm; + LIST_REMOVE(l, list); if (l->fd != -1) close(l->fd); + rm = 0; + +#if defined(__linux__) + if (worker == NULL && l->family == AF_UNIX && l->host[0] != '@') + rm++; +#else + if (worker == NULL && l->family == AF_UNIX) + rm++; +#endif + if (rm) { + if (unlink(l->host) == -1) { + kore_log(LOG_NOTICE, + "failed to remove unix socket %s (%s)", l->host, + errno_s); + } + } + kore_free(l->host); kore_free(l->port); From a92f6d17ccc89d588d9adde592e41d2ee488fb56 Mon Sep 17 00:00:00 2001 From: Frederic Cambus Date: Thu, 8 Apr 2021 10:11:48 +0200 Subject: [PATCH 015/121] Stop hardcoding HTTP/1.1 in access logs, Kore also supports HTTP/1.0. --- src/accesslog.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/accesslog.c b/src/accesslog.c index c0cf79a..287fa5d 100644 --- a/src/accesslog.c +++ b/src/accesslog.c @@ -66,7 +66,7 @@ kore_accesslog(struct http_request *req) size_t avail; time_t curtime; int len, attempts; - const char *ptr, *method, *cn, *referer; + const char *ptr, *method, *http_version, *cn, *referer; char addr[INET6_ADDRSTRLEN], *cn_value; switch (req->method) { @@ -93,6 +93,12 @@ kore_accesslog(struct http_request *req) break; } + if (req->flags & HTTP_VERSION_1_0) + http_version = "HTTP/1.0"; + + if (req->flags & HTTP_VERSION_1_1) + http_version = "HTTP/1.1"; + if (req->referer != NULL) referer = req->referer; else @@ -168,9 +174,9 @@ kore_accesslog(struct http_request *req) worker->lb.offset += sizeof(*hdr); len = snprintf(worker->lb.buf + worker->lb.offset, avail, - "%s - %s [%s] \"%s %s HTTP/1.1\" %d %zu \"%s\" \"%s\"\n", - addr, cn, tbuf, method, req->path, req->status, - req->content_length, referer, req->agent); + "%s - %s [%s] \"%s %s %s\" %d %zu \"%s\" \"%s\"\n", + addr, cn, tbuf, method, req->path, http_version, + req->status, req->content_length, referer, req->agent); if (len == -1) fatal("failed to create log entry"); From eea41776577918b8fe57e588b78599147b1843c2 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 9 Apr 2021 20:16:57 +0200 Subject: [PATCH 016/121] new BEERs available for grabs --- BEERS | 1 + 1 file changed, 1 insertion(+) diff --git a/BEERS b/BEERS index cb62a73..f348a72 100644 --- a/BEERS +++ b/BEERS @@ -21,6 +21,7 @@ I will note down the beer of your choice between the brackets. [] Frederic Cambus [] Guy Nankivell [] James Turner +[] Joel Arbring x 2 [] Manuel Kniep [] Marcin Szczepaniak [] Matt Thompson From 087da688dd21497550f62c95ded45faffccc1fc9 Mon Sep 17 00:00:00 2001 From: Frederic Cambus Date: Wed, 14 Apr 2021 15:14:26 +0200 Subject: [PATCH 017/121] Stop hardcoding HTTP error codes in http_error_response() calls. Use predefined HTTP_STATUS_* macros instead. --- src/http.c | 61 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/src/http.c b/src/http.c index 15aa9ef..9f9b214 100644 --- a/src/http.c +++ b/src/http.c @@ -271,7 +271,7 @@ http_check_timeout(struct connection *c, u_int64_t now) d = 0; if (d >= c->http_timeout) { - http_error_response(c, 408); + http_error_response(c, HTTP_STATUS_REQUEST_TIMEOUT); kore_connection_disconnect(c); return (KORE_RESULT_ERROR); } @@ -746,13 +746,13 @@ http_header_recv(struct netbuf *nb) h = kore_split_string(hbuf, "\r\n", headers, HTTP_REQ_HEADER_MAX); if (h < 2) { - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (KORE_RESULT_OK); } v = kore_split_string(headers[0], " ", request, 4); if (v != 3) { - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (KORE_RESULT_OK); } @@ -763,12 +763,12 @@ http_header_recv(struct netbuf *nb) continue; if ((host = http_validate_header(headers[i])) == NULL) { - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (KORE_RESULT_OK); } if (*host == '\0') { - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (KORE_RESULT_OK); } @@ -777,7 +777,7 @@ http_header_recv(struct netbuf *nb) } if (host == NULL) { - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (KORE_RESULT_OK); } @@ -796,13 +796,13 @@ http_header_recv(struct netbuf *nb) if ((value = http_validate_header(headers[i])) == NULL) { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (KORE_RESULT_OK); } if (*value == '\0') { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (KORE_RESULT_OK); } @@ -823,14 +823,16 @@ http_header_recv(struct netbuf *nb) if (req->flags & HTTP_REQUEST_EXPECT_BODY) { if (http_body_max == 0) { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 405); + http_error_response(req->owner, + HTTP_STATUS_METHOD_NOT_ALLOWED); return (KORE_RESULT_OK); } if (!http_request_header(req, "content-length", &clp)) { kore_debug("expected body but no content-length"); req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 411); + http_error_response(req->owner, + HTTP_STATUS_LENGTH_REQUIRED); return (KORE_RESULT_OK); } @@ -838,7 +840,8 @@ http_header_recv(struct netbuf *nb) if (v == KORE_RESULT_ERROR) { kore_debug("content-length invalid: %s", clp); req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 411); + http_error_response(req->owner, + HTTP_STATUS_LENGTH_REQUIRED); return (KORE_RESULT_OK); } @@ -852,7 +855,8 @@ http_header_recv(struct netbuf *nb) kore_log(LOG_NOTICE, "body too large (%zu > %zu)", req->content_length, http_body_max); req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 413); + http_error_response(req->owner, + HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE); return (KORE_RESULT_OK); } @@ -865,7 +869,8 @@ http_header_recv(struct netbuf *nb) "%s/http_body.XXXXXX", http_body_disk_path); if (l == -1 || (size_t)l >= HTTP_BODY_PATH_MAX) { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 500); + http_error_response(req->owner, + HTTP_STATUS_INTERNAL_ERROR); return (KORE_RESULT_ERROR); } @@ -873,7 +878,8 @@ http_header_recv(struct netbuf *nb) req->http_body_fd = mkstemp(req->http_body_path); if (req->http_body_fd == -1) { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 500); + http_error_response(req->owner, + HTTP_STATUS_INTERNAL_ERROR); return (KORE_RESULT_OK); } @@ -881,7 +887,8 @@ http_header_recv(struct netbuf *nb) end_headers, (nb->s_off - len)); if (ret == -1 || (size_t)ret != (nb->s_off - len)) { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 500); + http_error_response(req->owner, + HTTP_STATUS_INTERNAL_ERROR); return (KORE_RESULT_OK); } } else { @@ -912,7 +919,8 @@ http_header_recv(struct netbuf *nb) SHA256_Final(req->http_body_digest, &req->hashctx); if (!http_body_rewind(req)) { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 500); + http_error_response(req->owner, + HTTP_STATUS_INTERNAL_ERROR); return (KORE_RESULT_OK); } } @@ -1590,7 +1598,7 @@ http_request_new(struct connection *c, const char *host, size_t hostlen, pathlen, qsoff; if (http_request_count >= http_request_limit) { - http_error_response(c, 503); + http_error_response(c, HTTP_STATUS_SERVICE_UNAVAILABLE); return (NULL); } @@ -1598,18 +1606,18 @@ http_request_new(struct connection *c, const char *host, method, path, version); if ((hostlen = strlen(host)) >= KORE_DOMAINNAME_LEN - 1) { - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (NULL); } if ((pathlen = strlen(path)) >= HTTP_URI_LEN - 1) { - http_error_response(c, 414); + http_error_response(c, HTTP_STATUS_REQUEST_URI_TOO_LARGE); return (NULL); } if (strcasecmp(version, "http/1.1")) { if (strcasecmp(version, "http/1.0")) { - http_error_response(c, 505); + http_error_response(c, HTTP_STATUS_BAD_VERSION); return (NULL); } @@ -1630,7 +1638,7 @@ http_request_new(struct connection *c, const char *host, case AF_INET6: if (*host == '[') { if ((hp = strrchr(host, ']')) == NULL) { - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (NULL); } hp++; @@ -1688,7 +1696,7 @@ http_request_new(struct connection *c, const char *host, m = HTTP_METHOD_PATCH; flags |= HTTP_REQUEST_EXPECT_BODY; } else { - http_error_response(c, 400); + http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (NULL); } @@ -2024,14 +2032,16 @@ http_body_recv(struct netbuf *nb) ret = write(req->http_body_fd, nb->buf, nb->s_off); if (ret == -1 || (size_t)ret != nb->s_off) { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 500); + http_error_response(req->owner, + HTTP_STATUS_INTERNAL_ERROR); return (KORE_RESULT_ERROR); } } else if (req->http_body != NULL) { kore_buf_append(req->http_body, nb->buf, nb->s_off); } else { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 500); + http_error_response(req->owner, + HTTP_STATUS_INTERNAL_ERROR); return (KORE_RESULT_ERROR); } @@ -2045,7 +2055,8 @@ http_body_recv(struct netbuf *nb) req->content_length = req->http_body_length; if (!http_body_rewind(req)) { req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, 500); + http_error_response(req->owner, + HTTP_STATUS_INTERNAL_ERROR); return (KORE_RESULT_ERROR); } SHA256_Final(req->http_body_digest, &req->hashctx); From b6cb6c14f2e0f933fb0bff644bc70e2e57c8cb37 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 18 Apr 2021 11:57:53 +0200 Subject: [PATCH 018/121] kore_json_strerror() no longer takes a param. --- src/acme.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/acme.c b/src/acme.c index 239ecbc..3cca0c2 100644 --- a/src/acme.c +++ b/src/acme.c @@ -483,7 +483,7 @@ acme_parse_directory(void) if (!kore_json_parse(&json)) { kore_log(LOG_NOTICE, "failed to parse directory payload from ACME server (%s)", - kore_json_strerror(&json)); + kore_json_strerror()); goto cleanup; } @@ -688,7 +688,7 @@ acme_order_create_submit(struct acme_sign_op *op, struct kore_buf *payload) if (!kore_json_parse(&json)) { kore_log(LOG_NOTICE, "[%s] failed to parse order payload from ACME server (%s)", - domain, kore_json_strerror(&json)); + domain, kore_json_strerror()); goto cleanup; } @@ -806,7 +806,7 @@ acme_order_update_submit(struct acme_sign_op *op, struct kore_buf *payload) if (!kore_json_parse(&json)) { kore_log(LOG_NOTICE, "[%s] failed to parse order payload from ACME server (%s)", - order->domain, kore_json_strerror(&json)); + order->domain, kore_json_strerror()); goto cleanup; } @@ -1224,7 +1224,7 @@ acme_order_auth_update_submit(struct acme_sign_op *op, struct kore_buf *payload) if (!kore_json_parse(&json)) { kore_log(LOG_NOTICE, "[%s:auth] failed to parse payload from ACME server (%s)", - order->domain, kore_json_strerror(&json)); + order->domain, kore_json_strerror()); goto cleanup; } From a27227d37f447b1a2a2f948ef4aaa89ffac030be Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 19 Apr 2021 09:47:18 +0200 Subject: [PATCH 019/121] Rework how kodev create does python apps. Drop the kore.conf for python apps, all configuration can be done from inside the python code since kore4. Adds all the basic goo in the app.py file to get up and running. --- src/cli.c | 53 ++++++++++++++--------------------------------------- 1 file changed, 14 insertions(+), 39 deletions(-) diff --git a/src/cli.c b/src/cli.c index f446a5c..33d1224 100644 --- a/src/cli.c +++ b/src/cli.c @@ -209,7 +209,6 @@ static void file_create_src(void); static void file_create_config(void); static void file_create_gitignore(void); static void file_create_python_src(void); -static void file_create_python_config(void); static void cli_generate_certs(void); static void cli_file_create(const char *, const char *, size_t); @@ -253,7 +252,6 @@ static const char *python_gen_dirs[] = { static struct filegen python_gen_files[] = { { file_create_python_src }, - { file_create_python_config }, { file_create_gitignore }, { NULL } }; @@ -336,41 +334,33 @@ static const char *build_data = "# included if you build with the \"prod\" flavor.\n" "#}\n"; -static const char *python_config_data = - "# %s configuration\n" - "\n" - "server tls {\n" - "\tbind 127.0.0.1 8888\n" - "}\n" - "tls_dhparam\tdh2048.pem\n" - "\n" - "domain * {\n" - "\tattach\t\ttls\n" - "\n" - "\tcertfile\tcert/server.pem\n" - "\tcertkey\t\tcert/key.pem\n" - "\n" - "\troute\t/\tkoreapp.index\n" - "}\n"; - static const char *python_init_data = "from .app import koreapp\n"; static const char *python_app_data = "import kore\n" "\n" - "class App:\n" + "class KoreApp:\n" " def __init__(self):\n" " pass\n" "\n" " def configure(self, args):\n" - " kore.config.file = \"kore.conf\"\n" + " kore.config.tls_dhparam = \"dh2048.pem\"\n" " kore.config.deployment = \"development\"\n" + " kore.server(\"default\", ip=\"127.0.0.1\", port=\"8888\")\n" + "\n" + " d = kore.domain(\"*\",\n" + " attach=\"default\",\n" + " key=\"cert/key.pem\",\n" + " cert=\"cert/server.pem\",\n" + " )\n" + "\n" + " d.route(\"/\", self.index, methods=[\"get\"])\n" "\n" " async def index(self, req):\n" " req.response(200, b'')\n" "\n" - "koreapp = App()"; + "koreapp = KoreApp()"; static const char *dh2048_data = "-----BEGIN DH PARAMETERS-----\n" @@ -875,19 +865,6 @@ file_create_python_src(void) free(name); } -static void -file_create_python_config(void) -{ - int l; - char *name, *data; - - (void)cli_vasprintf(&name, "%s/kore.conf", appl); - l = cli_vasprintf(&data, python_config_data, appl); - cli_file_create(name, data, l); - free(name); - free(data); -} - static void file_create_src(void) { @@ -1616,10 +1593,8 @@ cli_run_kore_python(void) fatal("could not get cwd: %s", errno_s); args[0] = cmd; - args[1] = "-frnc"; - args[2] = "kore.conf"; - args[3] = pwd; - args[4] = NULL; + args[1] = pwd; + args[2] = NULL; execvp(args[0], args); fatal("failed to start '%s': %s", args[0], errno_s); From 960fe5afd35a15083df2e0097a782e76d3f6ccc1 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 20 Apr 2021 10:00:46 +0200 Subject: [PATCH 020/121] drop unused __init__ in cli generation --- src/cli.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cli.c b/src/cli.c index 33d1224..eb701df 100644 --- a/src/cli.c +++ b/src/cli.c @@ -341,9 +341,6 @@ static const char *python_app_data = "import kore\n" "\n" "class KoreApp:\n" - " def __init__(self):\n" - " pass\n" - "\n" " def configure(self, args):\n" " kore.config.tls_dhparam = \"dh2048.pem\"\n" " kore.config.deployment = \"development\"\n" From cf9e97f087b98130a761270b049a6c605a637d61 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 21 Apr 2021 10:48:00 +0200 Subject: [PATCH 021/121] Improve TLS settings and dependencies. - Kore now only supports OpenSSL 1.1.1 and LibreSSL 3.x. - Revise the default TLS ciphersuites. - Kore now carries ffdhe4096.pem and installs it under PREFIX/share/kore. - Kore its tls_dhparam config setting defaults to the path mentioned above so you no longer have to set it. --- Makefile | 1 + README.md | 5 +- conf/kore.conf.example | 7 ++- include/kore/kore.h | 3 +- misc/ffdhe4096.pem | 13 +++++ src/cli.c | 27 +++++---- src/domain.c | 129 ++++------------------------------------- 7 files changed, 49 insertions(+), 136 deletions(-) create mode 100644 misc/ffdhe4096.pem diff --git a/Makefile b/Makefile index a53bdba..6201d4c 100644 --- a/Makefile +++ b/Makefile @@ -198,6 +198,7 @@ install: install -m 555 $(KORE) $(DESTDIR)$(INSTALL_DIR)/$(KORE) install -m 644 kore.features $(DESTDIR)$(SHARE_DIR)/features install -m 644 include/kore/*.h $(DESTDIR)$(INCLUDE_DIR) + install -m 644 misc/ffdhe4096.pem $(DESTDIR)$(SHARE_DIR)/ffdhe4096.pem $(MAKE) -C kodev install $(MAKE) install-sources diff --git a/README.md b/README.md index 3dd87d8..ed18dc6 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ License Documentation -------------- -[Read the documentation](https://docs.kore.io/4.0.0/) +[Read the documentation](https://docs.kore.io/4.1.0/) Performance ----------- @@ -55,8 +55,7 @@ Building Kore Clone this repository or get the latest release at [https://kore.io/releases/4.1.0](https://kore.io/releases/4.1.0). Requirements -* openssl (1.0.2, 1.1.0 or 1.1.1) - (note: libressl 3.0.0+ works as a replacement) +* openssl 1.1.1 or libressl 3.x Requirement for asynchronous curl (optional) * libcurl (7.64.0 or higher) diff --git a/conf/kore.conf.example b/conf/kore.conf.example index 971cb45..6f01053 100644 --- a/conf/kore.conf.example +++ b/conf/kore.conf.example @@ -198,10 +198,11 @@ validator v_session function v_session_validate #tls_version 1.3 # Specify the TLS ciphers that will be used. -#tls_cipher ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK:!kRSA:!kDSA +#tls_cipher AEAD-AES256-GCM-SHA384:AEAD-CHACHA20-POLY1305-SHA256:AEAD-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256 -# Required DH parameters for TLS. -#tls_dhparam dh2048.pem +# Required DH parameters for TLS if DHE ciphersuites are in-use. +# Defaults to SHARE_DIR/ffdhe4096.pem, can be overwritten. +#tls_dhparam /usr/local/share/kore/ffdhe4096.pem # OpenBSD specific settings. # Add more pledges if your application requires more privileges. diff --git a/include/kore/kore.h b/include/kore/kore.h index 493d593..c26553f 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -103,7 +103,8 @@ extern int daemon(int, int); #define KORE_DOMAINNAME_LEN 255 #define KORE_PIDFILE_DEFAULT "kore.pid" -#define KORE_DEFAULT_CIPHER_LIST "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK:!kRSA:!kDSA" +#define KORE_DHPARAM_PATH PREFIX "/share/kore/ffdhe4096.pem" +#define KORE_DEFAULT_CIPHER_LIST "AEAD-AES256-GCM-SHA384:AEAD-CHACHA20-POLY1305-SHA256:AEAD-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256" #if defined(KORE_DEBUG) #define kore_debug(...) \ diff --git a/misc/ffdhe4096.pem b/misc/ffdhe4096.pem new file mode 100644 index 0000000..3cf0fcb --- /dev/null +++ b/misc/ffdhe4096.pem @@ -0,0 +1,13 @@ +-----BEGIN DH PARAMETERS----- +MIICCAKCAgEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEfz9zeNVs7ZRkDW7w09N75nAI4YbRvydbmyQd62R0mkff3 +7lmMsPrBhtkcrv4TCYUTknC0EwyTvEN5RPT9RFLi103TZPLiHnH1S/9croKrnJ32 +nuhtK8UiNjoNq8Uhl5sN6todv5pC1cRITgq80Gv6U93vPBsg7j/VnXwl5B0rZp4e +8W5vUsMWTfT7eTDp5OWIV7asfV9C1p9tGHdjzx1VA0AEh/VbpX4xzHpxNciG77Qx +iu1qHgEtnmgyqQdgCpGBMMRtx3j5ca0AOAkpmaMzy4t6Gh25PXFAADwqTs6p+Y0K +zAqCkc3OyX3Pjsm1Wn+IpGtNtahR9EGC4caKAH5eZV9q//////////8CAQI= +-----END DH PARAMETERS----- diff --git a/src/cli.c b/src/cli.c index eb701df..842e91f 100644 --- a/src/cli.c +++ b/src/cli.c @@ -287,7 +287,7 @@ static const char *config_data = "\n" "load\t\t./%s.so\n" "\n" - "tls_dhparam\tdh2048.pem\n" + "tls_dhparam\tdh4096.pem\n" "\n" "domain * {\n" "\tattach\t\ttls\n" @@ -342,7 +342,7 @@ static const char *python_app_data = "\n" "class KoreApp:\n" " def configure(self, args):\n" - " kore.config.tls_dhparam = \"dh2048.pem\"\n" + " kore.config.tls_dhparam = \"dh4096.pem\"\n" " kore.config.deployment = \"development\"\n" " kore.server(\"default\", ip=\"127.0.0.1\", port=\"8888\")\n" "\n" @@ -359,15 +359,20 @@ static const char *python_app_data = "\n" "koreapp = KoreApp()"; -static const char *dh2048_data = +static const char *dh4096_data = "-----BEGIN DH PARAMETERS-----\n" - "MIIBCAKCAQEAn4f4Qn5SudFjEYPWTbUaOTLUH85YWmmPFW1+b5bRa9ygr+1wfamv\n" - "VKVT7jO8c4msSNikUf6eEfoH0H4VTCaj+Habwu+Sj+I416r3mliMD4SjNsUJrBrY\n" - "Y0QV3ZUgZz4A8ARk/WwQcRl8+ZXJz34IaLwAcpyNhoV46iHVxW0ty8ND0U4DIku/\n" - "PNayKimu4BXWXk4RfwNVP59t8DQKqjshZ4fDnbotskmSZ+e+FHrd+Kvrq/WButvV\n" - "Bzy9fYgnUlJ82g/bziCI83R2xAdtH014fR63MpElkqdNeChb94pPbEdFlNUvYIBN\n" - "xx2vTUQMqRbB4UdG2zuzzr5j98HDdblQ+wIBAg==\n" - "-----END DH PARAMETERS-----"; + "MIICCAKCAgEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz\n" + "+8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a\n" + "87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7\n" + "YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi\n" + "7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD\n" + "ssbzSibBsu/6iGtCOGEfz9zeNVs7ZRkDW7w09N75nAI4YbRvydbmyQd62R0mkff3\n" + "7lmMsPrBhtkcrv4TCYUTknC0EwyTvEN5RPT9RFLi103TZPLiHnH1S/9croKrnJ32\n" + "nuhtK8UiNjoNq8Uhl5sN6todv5pC1cRITgq80Gv6U93vPBsg7j/VnXwl5B0rZp4e\n" + "8W5vUsMWTfT7eTDp5OWIV7asfV9C1p9tGHdjzx1VA0AEh/VbpX4xzHpxNciG77Qx\n" + "iu1qHgEtnmgyqQdgCpGBMMRtx3j5ca0AOAkpmaMzy4t6Gh25PXFAADwqTs6p+Y0K\n" + "zAqCkc3OyX3Pjsm1Wn+IpGtNtahR9EGC4caKAH5eZV9q//////////8CAQI=\n" + "-----END DH PARAMETERS-----\n"; static const char *gitignore = "*.o\n.flavor\n.objs\n%s.so\nassets.h\ncert\n"; @@ -1371,7 +1376,7 @@ cli_generate_certs(void) char issuer[64]; /* Write out DH parameters. */ - cli_file_create("dh2048.pem", dh2048_data, strlen(dh2048_data)); + cli_file_create("dh4096.pem", dh4096_data, strlen(dh4096_data)); /* Create new certificate. */ if ((x509 = X509_new()) == NULL) diff --git a/src/domain.c b/src/domain.c index 15b85a9..1663b96 100644 --- a/src/domain.c +++ b/src/domain.c @@ -14,12 +14,6 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -/* - * XXX - Lots of OPENSSL ifdefs here for 1.0.2 and 1.1.0 release lines. - * The idea is to only support 1.1.1 down the line and remove the rest. - * (although we have to remain compat with 1.0.2 due to LibreSSL). - */ - #include #include @@ -68,54 +62,8 @@ static int keymgr_rsa_privenc(int, const unsigned char *, static ECDSA_SIG *keymgr_ecdsa_sign(const unsigned char *, int, const BIGNUM *, const BIGNUM *, EC_KEY *); -#if defined(KORE_OPENSSL_NEWER_API) static RSA_METHOD *keymgr_rsa_meth = NULL; static EC_KEY_METHOD *keymgr_ec_meth = NULL; -#else -/* - * Run own ecdsa_method data structure as OpenSSL has this in ecs_locl.h - * and does not export this on systems. - */ -struct ecdsa_method { - const char *name; - ECDSA_SIG *(*ecdsa_do_sign)(const unsigned char *, - int, const BIGNUM *, const BIGNUM *, EC_KEY *); - int (*ecdsa_sign_setup)(EC_KEY *, BN_CTX *, BIGNUM **, - BIGNUM **); - int (*ecdsa_do_verify)(const unsigned char *, int, - const ECDSA_SIG *, EC_KEY *); - int flags; - char *app_data; -}; -#endif - -#if !defined(KORE_OPENSSL_NEWER_API) -static ECDSA_METHOD keymgr_ecdsa = { - "kore ECDSA keymgr method", - keymgr_ecdsa_sign, - NULL, - NULL, - 0, - NULL -}; - -static RSA_METHOD keymgr_rsa = { - "kore RSA keymgr method", - NULL, - NULL, - keymgr_rsa_privenc, - NULL, - NULL, - NULL, - keymgr_rsa_init, - keymgr_rsa_finish, - RSA_METHOD_FLAG_NO_CHECK, - NULL, - NULL, - NULL, - NULL -}; -#endif static u_int16_t domain_id = 0; static struct kore_domain *cached[KORE_DOMAIN_CACHE]; @@ -128,7 +76,6 @@ kore_domain_init(void) for (i = 0; i < KORE_DOMAIN_CACHE; i++) cached[i] = NULL; -#if defined(KORE_OPENSSL_NEWER_API) if (keymgr_rsa_meth == NULL) { if ((keymgr_rsa_meth = RSA_meth_new("kore RSA keymgr method", RSA_METHOD_FLAG_NO_CHECK)) == NULL) @@ -145,7 +92,6 @@ kore_domain_init(void) } EC_KEY_METHOD_set_sign(keymgr_ec_meth, NULL, NULL, keymgr_ecdsa_sign); -#endif #if !defined(TLS1_3_VERSION) if (!kore_quiet) { @@ -159,7 +105,6 @@ kore_domain_init(void) void kore_domain_cleanup(void) { -#if defined(KORE_OPENSSL_NEWER_API) if (keymgr_rsa_meth != NULL) { RSA_meth_free(keymgr_rsa_meth); keymgr_rsa_meth = NULL; @@ -169,7 +114,6 @@ kore_domain_cleanup(void) EC_KEY_METHOD_free(keymgr_ec_meth); keymgr_ec_meth = NULL; } -#endif } struct kore_domain * @@ -278,40 +222,24 @@ kore_domain_tlsinit(struct kore_domain *dom, int type, { const u_int8_t *ptr; RSA *rsa; + BIO *bio; X509 *x509; EVP_PKEY *pkey; STACK_OF(X509_NAME) *certs; EC_KEY *eckey; const SSL_METHOD *method; -#if !defined(KORE_OPENSSL_NEWER_API) - EC_KEY *ecdh; -#endif kore_debug("kore_domain_tlsinit(%s)", dom->domain); if (dom->ssl_ctx != NULL) SSL_CTX_free(dom->ssl_ctx); -#if defined(KORE_OPENSSL_NEWER_API) if ((method = TLS_method()) == NULL) fatalx("TLS_method(): %s", ssl_errno_s); -#else - switch (tls_version) { - case KORE_TLS_VERSION_1_3: - case KORE_TLS_VERSION_1_2: - case KORE_TLS_VERSION_BOTH: - method = TLSv1_2_server_method(); - break; - default: - fatalx("unknown tls_version: %d", tls_version); - return; - } -#endif if ((dom->ssl_ctx = SSL_CTX_new(method)) == NULL) fatalx("SSL_ctx_new(): %s", ssl_errno_s); -#if defined(KORE_OPENSSL_NEWER_API) if (!SSL_CTX_set_min_proto_version(dom->ssl_ctx, TLS1_2_VERSION)) fatalx("SSL_CTX_set_min_proto_version: %s", ssl_errno_s); @@ -346,7 +274,6 @@ kore_domain_tlsinit(struct kore_domain *dom, int type, fatalx("unknown tls_version: %d", tls_version); return; } -#endif switch (type) { case KORE_PEM_CERT_CHAIN: @@ -380,22 +307,13 @@ kore_domain_tlsinit(struct kore_domain *dom, int type, if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) fatalx("no RSA public key present"); RSA_set_app_data(rsa, dom); -#if defined(KORE_OPENSSL_NEWER_API) RSA_set_method(rsa, keymgr_rsa_meth); -#else - RSA_set_method(rsa, &keymgr_rsa); -#endif break; case EVP_PKEY_EC: if ((eckey = EVP_PKEY_get1_EC_KEY(pkey)) == NULL) fatalx("no EC public key present"); -#if defined(KORE_OPENSSL_NEWER_API) EC_KEY_set_ex_data(eckey, 0, dom); EC_KEY_set_method(eckey, keymgr_ec_meth); -#else - ECDSA_set_ex_data(eckey, 0, dom); - ECDSA_set_method(eckey, &keymgr_ecdsa); -#endif break; default: fatalx("unknown public key in certificate"); @@ -409,22 +327,22 @@ kore_domain_tlsinit(struct kore_domain *dom, int type, dom->domain, ssl_errno_s); } - if (tls_dhparam == NULL) - fatalx("No DH parameters given"); + if (tls_dhparam == NULL) { + if ((bio = BIO_new_file(KORE_DHPARAM_PATH, "r")) == NULL) + fatal("failed to open %s", KORE_DHPARAM_PATH); + + tls_dhparam = PEM_read_bio_DHparams(bio, NULL, NULL, NULL); + BIO_free(bio); + + if (tls_dhparam == NULL) + fatal("PEM_read_bio_DHparams(): %s", ssl_errno_s); + } SSL_CTX_set_tmp_dh(dom->ssl_ctx, tls_dhparam); SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_SINGLE_DH_USE); -#if defined(KORE_OPENSSL_NEWER_API) if (!SSL_CTX_set_ecdh_auto(dom->ssl_ctx, 1)) fatalx("SSL_CTX_set_ecdh_auto: %s", ssl_errno_s); -#else - if ((ecdh = EC_KEY_new_by_curve_name(NID_secp384r1)) == NULL) - fatalx("EC_KEY_new_by_curve_name: %s", ssl_errno_s); - - SSL_CTX_set_tmp_ecdh(dom->ssl_ctx, ecdh); - EC_KEY_free(ecdh); -#endif SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_SINGLE_ECDH_USE); SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_NO_COMPRESSION); @@ -605,27 +523,17 @@ keymgr_init(void) if ((meth = RSA_get_default_method()) == NULL) fatal("failed to obtain RSA method"); -#if defined(KORE_OPENSSL_NEWER_API) RSA_meth_set_pub_enc(keymgr_rsa_meth, RSA_meth_get_pub_enc(meth)); RSA_meth_set_pub_dec(keymgr_rsa_meth, RSA_meth_get_pub_dec(meth)); RSA_meth_set_bn_mod_exp(keymgr_rsa_meth, RSA_meth_get_bn_mod_exp(meth)); -#else - keymgr_rsa.rsa_pub_enc = meth->rsa_pub_enc; - keymgr_rsa.rsa_pub_dec = meth->rsa_pub_dec; - keymgr_rsa.bn_mod_exp = meth->bn_mod_exp; -#endif } static int keymgr_rsa_init(RSA *rsa) { if (rsa != NULL) { -#if defined(KORE_OPENSSL_NEWER_API) RSA_set_flags(rsa, RSA_flags(rsa) | RSA_FLAG_EXT_PKEY | RSA_METHOD_FLAG_NO_CHECK); -#else - rsa->flags |= RSA_FLAG_EXT_PKEY | RSA_METHOD_FLAG_NO_CHECK; -#endif return (1); } @@ -702,13 +610,8 @@ keymgr_ecdsa_sign(const unsigned char *dgst, int dgst_len, if (len > sizeof(keymgr_buf)) fatal("keymgr_buf too small"); -#if defined(KORE_OPENSSL_NEWER_API) if ((dom = EC_KEY_get_ex_data(eckey, 0)) == NULL) fatal("EC_KEY has no domain"); -#else - if ((dom = ECDSA_get_ex_data(eckey, 0)) == NULL) - fatal("EC_KEY has no domain"); -#endif memset(keymgr_buf, 0, sizeof(keymgr_buf)); req = (struct kore_keyreq *)keymgr_buf; @@ -890,23 +793,13 @@ domain_load_certificate_chain(SSL_CTX *ctx, const void *data, size_t len) if (SSL_CTX_use_certificate(ctx, x) == 0) return (NULL); -#if defined(KORE_OPENSSL_NEWER_API) SSL_CTX_clear_chain_certs(ctx); -#else - sk_X509_pop_free(ctx->extra_certs, X509_free); - ctx->extra_certs = NULL; -#endif ERR_clear_error(); while ((ca = PEM_read_bio_X509(in, NULL, NULL, NULL)) != NULL) { /* ca its reference count won't be increased. */ -#if defined(KORE_OPENSSL_NEWER_API) if (SSL_CTX_add0_chain_cert(ctx, ca) == 0) return (NULL); -#else - if (SSL_CTX_add_extra_chain_cert(ctx, ca) == 0) - return (NULL); -#endif } err = ERR_peek_last_error(); From c77ec598e7eb75bef4cb972b4d053f30bea9f6be Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 21 Apr 2021 10:52:02 +0200 Subject: [PATCH 022/121] Remove dh parameters from kodev. --- src/cli.c | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/cli.c b/src/cli.c index 842e91f..fbd50ac 100644 --- a/src/cli.c +++ b/src/cli.c @@ -287,8 +287,6 @@ static const char *config_data = "\n" "load\t\t./%s.so\n" "\n" - "tls_dhparam\tdh4096.pem\n" - "\n" "domain * {\n" "\tattach\t\ttls\n" "\n" @@ -342,7 +340,6 @@ static const char *python_app_data = "\n" "class KoreApp:\n" " def configure(self, args):\n" - " kore.config.tls_dhparam = \"dh4096.pem\"\n" " kore.config.deployment = \"development\"\n" " kore.server(\"default\", ip=\"127.0.0.1\", port=\"8888\")\n" "\n" @@ -359,21 +356,6 @@ static const char *python_app_data = "\n" "koreapp = KoreApp()"; -static const char *dh4096_data = - "-----BEGIN DH PARAMETERS-----\n" - "MIICCAKCAgEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz\n" - "+8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a\n" - "87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7\n" - "YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi\n" - "7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD\n" - "ssbzSibBsu/6iGtCOGEfz9zeNVs7ZRkDW7w09N75nAI4YbRvydbmyQd62R0mkff3\n" - "7lmMsPrBhtkcrv4TCYUTknC0EwyTvEN5RPT9RFLi103TZPLiHnH1S/9croKrnJ32\n" - "nuhtK8UiNjoNq8Uhl5sN6todv5pC1cRITgq80Gv6U93vPBsg7j/VnXwl5B0rZp4e\n" - "8W5vUsMWTfT7eTDp5OWIV7asfV9C1p9tGHdjzx1VA0AEh/VbpX4xzHpxNciG77Qx\n" - "iu1qHgEtnmgyqQdgCpGBMMRtx3j5ca0AOAkpmaMzy4t6Gh25PXFAADwqTs6p+Y0K\n" - "zAqCkc3OyX3Pjsm1Wn+IpGtNtahR9EGC4caKAH5eZV9q//////////8CAQI=\n" - "-----END DH PARAMETERS-----\n"; - static const char *gitignore = "*.o\n.flavor\n.objs\n%s.so\nassets.h\ncert\n"; #endif /* !KODEV_MINIMAL */ @@ -538,8 +520,7 @@ cli_create(int argc, char **argv) cli_generate_certs(); printf("%s created successfully!\n", appl); - printf("WARNING: DO NOT USE THE GENERATED DH PARAMETERS " - "AND CERTIFICATES IN PRODUCTION\n"); + printf("WARNING: DO NOT USE THE GENERATED CERTIFICATE IN PRODUCTION\n"); } #endif @@ -1375,9 +1356,6 @@ cli_generate_certs(void) RSA *kpair; char issuer[64]; - /* Write out DH parameters. */ - cli_file_create("dh4096.pem", dh4096_data, strlen(dh4096_data)); - /* Create new certificate. */ if ((x509 = X509_new()) == NULL) fatal("X509_new(): %s", ssl_errno_s); From c4a60c54bb7a67680138419b72548a565c40984f Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 21 Apr 2021 22:39:35 +0200 Subject: [PATCH 023/121] resolve tls_dhparam after configure. --- src/config.c | 21 +++++++++++++++------ src/domain.c | 13 ++----------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/config.c b/src/config.c index e4382ef..f7dc090 100644 --- a/src/config.c +++ b/src/config.c @@ -304,6 +304,7 @@ void kore_parse_config(void) { FILE *fp; + BIO *bio; char path[PATH_MAX]; if (finalized) @@ -327,6 +328,17 @@ kore_parse_config(void) (void)fclose(fp); } + if (tls_dhparam == NULL) { + if ((bio = BIO_new_file(KORE_DHPARAM_PATH, "r")) == NULL) + fatal("failed to open %s", KORE_DHPARAM_PATH); + + tls_dhparam = PEM_read_bio_DHparams(bio, NULL, NULL, NULL); + BIO_free(bio); + + if (tls_dhparam == NULL) + fatal("PEM_read_bio_DHparams(): %s", ssl_errno_s); + } + if (!kore_module_loaded()) fatal("no application module was loaded"); @@ -341,17 +353,14 @@ kore_parse_config(void) } } - if (getuid() != 0 && skip_chroot == 0) { + if (getuid() != 0 && skip_chroot == 0) fatal("cannot chroot, use -n to skip it"); - } - if (skip_runas != 1 && kore_runas_user == NULL) { + if (skip_runas != 1 && kore_runas_user == NULL) fatal("missing runas user, use -r to skip it"); - } - if (getuid() != 0 && skip_runas == 0) { + if (getuid() != 0 && skip_runas == 0) fatal("cannot drop privileges, use -r to skip it"); - } if (skip_runas) { if (!kore_quiet) diff --git a/src/domain.c b/src/domain.c index 1663b96..96caf63 100644 --- a/src/domain.c +++ b/src/domain.c @@ -222,7 +222,6 @@ kore_domain_tlsinit(struct kore_domain *dom, int type, { const u_int8_t *ptr; RSA *rsa; - BIO *bio; X509 *x509; EVP_PKEY *pkey; STACK_OF(X509_NAME) *certs; @@ -327,16 +326,8 @@ kore_domain_tlsinit(struct kore_domain *dom, int type, dom->domain, ssl_errno_s); } - if (tls_dhparam == NULL) { - if ((bio = BIO_new_file(KORE_DHPARAM_PATH, "r")) == NULL) - fatal("failed to open %s", KORE_DHPARAM_PATH); - - tls_dhparam = PEM_read_bio_DHparams(bio, NULL, NULL, NULL); - BIO_free(bio); - - if (tls_dhparam == NULL) - fatal("PEM_read_bio_DHparams(): %s", ssl_errno_s); - } + if (tls_dhparam == NULL) + fatal("no DH parameters specified"); SSL_CTX_set_tmp_dh(dom->ssl_ctx, tls_dhparam); SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_SINGLE_DH_USE); From fb335e1e0c3f0dff311dc9bbdbfbe992cb2b67f4 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 2 May 2021 00:23:11 +0200 Subject: [PATCH 024/121] Major Python API improvements. 1) Add @kore.route as a decorator for Python. This decorator can be used on non-class methods to automatically declare their route and parameters. Takes the same arguments as the kore.domain.route function that exists today. Provides a nice clean way of setting up Kore if you dont want a whole class based approach. 2) Remove the requirement for the name for kore.server() and the kore.domain(attach=) keywords. Instead of no name was given, the name "default" is used in both places resulting in less boilerplating. 3) Allow multiple routes to be defined for the same URI as long as the methods are different. So you can have one method for GET / and another for POST /. All changes combined condense the initial experience of getting a Kore Python app up and running: eg: import kore kore.server(ip="127.0.0.1", port="8888", tls=False) kore.domain("*") @kore.route("/", methods=["get"]) async def index(req): req.response(200, b'get method') @kore.route("/", methods=["post"]) async def index_post(req) req.response(200, b'post method') --- Makefile | 2 + include/kore/kore.h | 4 +- include/kore/python_api.h | 1 + include/kore/python_methods.h | 36 +++- kodev/Makefile | 2 + src/cli.c | 1 + src/config.c | 5 +- src/http.c | 8 +- src/kore.c | 4 + src/module.c | 29 ++- src/python.c | 378 ++++++++++++++++++++++++---------- 11 files changed, 341 insertions(+), 129 deletions(-) diff --git a/Makefile b/Makefile index 6201d4c..d5ebd8c 100644 --- a/Makefile +++ b/Makefile @@ -134,6 +134,8 @@ endif ifeq ("$(OSNAME)", "darwin") CFLAGS+=-I/opt/local/include/ -I/usr/local/opt/openssl/include LDFLAGS+=-L/opt/local/lib -L/usr/local/opt/openssl/lib + CFLAGS+=-I/opt/homebrew/opt/openssl/include + LDFLAGS+=-L/opt/homebrew/opt/openssl/lib S_SRC+=src/bsd.c else ifeq ("$(OSNAME)", "linux") CFLAGS+=-D_GNU_SOURCE=1 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 diff --git a/include/kore/kore.h b/include/kore/kore.h index c26553f..5554d10 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -952,8 +952,8 @@ void kore_domain_crl_add(struct kore_domain *, const void *, size_t); int kore_module_handler_new(struct kore_domain *, const char *, const char *, const char *, int); void kore_module_handler_free(struct kore_module_handle *); -struct kore_module_handle *kore_module_handler_find(struct http_request *, - struct kore_domain *); +int kore_module_handler_find(struct http_request *, + struct kore_domain *, int, struct kore_module_handle **); #endif struct kore_runtime_call *kore_runtime_getcall(const char *); diff --git a/include/kore/python_api.h b/include/kore/python_api.h index 21ce053..d8052aa 100644 --- a/include/kore/python_api.h +++ b/include/kore/python_api.h @@ -34,6 +34,7 @@ void kore_python_proc_reap(void); int kore_python_coro_pending(void); void kore_python_path(const char *); void kore_python_coro_delete(void *); +void kore_python_routes_resolve(void); void kore_python_log_error(const char *); PyObject *kore_python_callable(PyObject *, const char *); diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index e4ad14d..65895f7 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -54,6 +54,7 @@ static PyObject *python_kore_task_kill(PyObject *, PyObject *); static PyObject *python_kore_prerequest(PyObject *, PyObject *); static PyObject *python_kore_task_create(PyObject *, PyObject *); static PyObject *python_kore_socket_wrap(PyObject *, PyObject *); +static PyObject *python_kore_route(PyObject *, PyObject *, PyObject *); static PyObject *python_kore_timer(PyObject *, PyObject *, PyObject *); static PyObject *python_kore_domain(PyObject *, PyObject *, PyObject *); static PyObject *python_kore_gather(PyObject *, PyObject *, PyObject *); @@ -62,6 +63,7 @@ static PyObject *python_kore_sendobj(PyObject *, PyObject *, static PyObject *python_kore_server(PyObject *, PyObject *, PyObject *); + #if defined(KORE_USE_PGSQL) static PyObject *python_kore_pgsql_query(PyObject *, PyObject *, PyObject *); @@ -101,10 +103,11 @@ static struct PyMethodDef pykore_methods[] = { METHOD("prerequest", python_kore_prerequest, METH_VARARGS), METHOD("task_create", python_kore_task_create, METH_VARARGS), METHOD("socket_wrap", python_kore_socket_wrap, METH_VARARGS), + METHOD("route", python_kore_route, METH_VARARGS | METH_KEYWORDS), METHOD("timer", python_kore_timer, METH_VARARGS | METH_KEYWORDS), + METHOD("domain", python_kore_domain, METH_VARARGS | METH_KEYWORDS), METHOD("server", python_kore_server, METH_VARARGS | METH_KEYWORDS), METHOD("gather", python_kore_gather, METH_VARARGS | METH_KEYWORDS), - METHOD("domain", python_kore_domain, METH_VARARGS | METH_KEYWORDS), METHOD("sendobj", python_kore_sendobj, METH_VARARGS | METH_KEYWORDS), METHOD("websocket_broadcast", python_websocket_broadcast, METH_VARARGS), #if defined(KORE_USE_PGSQL) @@ -186,9 +189,38 @@ static PyTypeObject pyseccomp_type = { }; #endif +struct pyroute { + PyObject_HEAD + char *path; + PyObject *func; + PyObject *kwargs; + struct kore_domain *domain; + TAILQ_ENTRY(pyroute) list; +}; + +static PyObject *pyroute_inner(struct pyroute *, PyObject *); +static void pyroute_dealloc(struct pyroute *); + +static PyMethodDef pyroute_methods[] = { + METHOD("inner", pyroute_inner, METH_VARARGS), + METHOD(NULL, NULL, -1) +}; + +static PyTypeObject pyroute_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.route", + .tp_doc = "kore route function", + .tp_methods = pyroute_methods, + .tp_basicsize = sizeof(struct pyroute), + .tp_dealloc = (destructor)pyroute_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + struct pydomain { PyObject_HEAD - struct kore_domain *config; + struct kore_domain *config; + struct kore_module_handle *next; + PyObject *kwargs; }; static PyObject *pydomain_filemaps(struct pydomain *, PyObject *); diff --git a/kodev/Makefile b/kodev/Makefile index c204d05..b21a6d6 100644 --- a/kodev/Makefile +++ b/kodev/Makefile @@ -30,6 +30,8 @@ OSNAME=$(shell uname -s | sed -e 's/[-_].*//g' | tr A-Z a-z) ifeq ("$(OSNAME)", "darwin") CFLAGS+=-I/opt/local/include/ -I/usr/local/opt/openssl/include LDFLAGS+=-L/opt/local/lib -L/usr/local/opt/openssl/lib + CFLAGS+=-I/opt/homebrew/opt/openssl/include + LDFLAGS+=-L/opt/homebrew/opt/openssl/lib else ifeq ("$(OSNAME)", "linux") CFLAGS+=-D_GNU_SOURCE=1 endif diff --git a/src/cli.c b/src/cli.c index fbd50ac..f102146 100644 --- a/src/cli.c +++ b/src/cli.c @@ -1901,6 +1901,7 @@ cli_build_flags_common(struct buildopt *bopt, struct cli_buf *buf) /* Add default openssl include path from homebrew / ports under OSX. */ cli_buf_appendf(buf, "-I/opt/local/include "); cli_buf_appendf(buf, "-I/usr/local/opt/openssl/include "); + cli_buf_appendf(buf, "-I/opt/homebrew/opt/openssl/include "); #endif if (bopt->single_binary == 0) { cli_kore_features(bopt, &data, &len); diff --git a/src/config.c b/src/config.c index f7dc090..7f3466e 100644 --- a/src/config.c +++ b/src/config.c @@ -1876,13 +1876,10 @@ configure_deployment(char *value) skip_chroot = 0; } else { kore_log(LOG_NOTICE, - "pyko.config.deployment: bad value '%s'", value); + "kore.config.deployment: bad value '%s'", value); return (KORE_RESULT_ERROR); } - if (!kore_quiet) - kore_log(LOG_NOTICE, "deployment set to %s", value); - return (KORE_RESULT_OK); } diff --git a/src/http.c b/src/http.c index 9f9b214..fd46591 100644 --- a/src/http.c +++ b/src/http.c @@ -1594,7 +1594,7 @@ http_request_new(struct connection *c, const char *host, struct kore_domain *dom; struct http_request *req; char *p, *hp; - int m, flags; + int m, flags, exists; size_t hostlen, pathlen, qsoff; if (http_request_count >= http_request_limit) { @@ -1748,7 +1748,7 @@ http_request_new(struct connection *c, const char *host, } /* Checked further down below if we need to 404. */ - req->hdlr = kore_module_handler_find(req, dom); + exists = kore_module_handler_find(req, dom, m, &req->hdlr); TAILQ_INIT(&(req->resp_headers)); TAILQ_INIT(&(req->req_headers)); @@ -1774,13 +1774,13 @@ http_request_new(struct connection *c, const char *host, return (NULL); } - if (req->hdlr == NULL) { + if (exists == 0) { http_request_free(req); http_error_response(c, HTTP_STATUS_NOT_FOUND); return (NULL); } - if (!(req->hdlr->methods & m)) { + if (req->hdlr == NULL) { http_request_free(req); http_error_response(c, HTTP_STATUS_METHOD_NOT_ALLOWED); return (NULL); diff --git a/src/kore.c b/src/kore.c index 9963140..a500fbc 100644 --- a/src/kore.c +++ b/src/kore.c @@ -909,6 +909,10 @@ kore_server_start(int argc, char *argv[]) kore_call_parent_configure(argc, argv); #endif +#if defined(KORE_USE_PYTHON) + kore_python_routes_resolve(); +#endif + /* Check if keymgr will be active. */ LIST_FOREACH(srv, &kore_servers, list) { if (srv->tls) { diff --git a/src/module.c b/src/module.c index 6da2ad4..ff57a35 100644 --- a/src/module.c +++ b/src/module.c @@ -283,23 +283,38 @@ kore_module_handler_free(struct kore_module_handle *hdlr) kore_free(hdlr); } -struct kore_module_handle * -kore_module_handler_find(struct http_request *req, struct kore_domain *dom) +int +kore_module_handler_find(struct http_request *req, struct kore_domain *dom, + int method, struct kore_module_handle **out) { struct kore_module_handle *hdlr; + int exists; + + exists = 0; + *out = NULL; TAILQ_FOREACH(hdlr, &(dom->handlers), list) { if (hdlr->type == HANDLER_TYPE_STATIC) { - if (!strcmp(hdlr->path, req->path)) - return (hdlr); + if (!strcmp(hdlr->path, req->path)) { + if (hdlr->methods & method) { + *out = hdlr; + return (1); + } + exists++; + } } else { if (!regexec(&(hdlr->rctx), req->path, - HTTP_CAPTURE_GROUPS, req->cgroups, 0)) - return (hdlr); + HTTP_CAPTURE_GROUPS, req->cgroups, 0)) { + if (hdlr->methods & method) { + *out = hdlr; + return (1); + } + exists++; + } } } - return (NULL); + return (exists); } #endif /* !KORE_NO_HTTP */ diff --git a/src/python.c b/src/python.c index 8cafd27..8ae91f0 100644 --- a/src/python.c +++ b/src/python.c @@ -77,6 +77,15 @@ static PyObject *pyhttp_request_alloc(const struct http_request *); static struct python_coro *python_coro_create(PyObject *, struct http_request *); +static struct kore_domain *python_route_domain_resolve(struct pyroute *); + +static int python_route_install(struct pyroute *); +static int python_route_params(PyObject *, + struct kore_module_handle *, const char *, int); +static int python_route_methods(PyObject *, PyObject *, + struct kore_module_handle *); +static int python_route_auth(PyObject *, + struct kore_module_handle *); static int python_coro_run(struct python_coro *); static void python_coro_wakeup(struct python_coro *); @@ -108,10 +117,6 @@ static int pyhttp_iterobj_chunk_sent(struct netbuf *); static int pyhttp_iterobj_next(struct pyhttp_iterobj *); static void pyhttp_iterobj_disconnect(struct connection *); -static int pydomain_params(PyObject *, - struct kore_module_handle *, const char *, int); -static int pydomain_auth(PyObject *, struct kore_module_handle *); - #if defined(KORE_USE_PGSQL) static int pykore_pgsql_result(struct pykore_pgsql *); static void pykore_pgsql_callback(struct kore_pgsql *, void *); @@ -259,6 +264,7 @@ static struct pyseccomp *py_seccomp = NULL; #endif static TAILQ_HEAD(, pyproc) procs; +static TAILQ_HEAD(, pyroute) routes; static struct reqcall_list prereq; static struct kore_pool coro_pool; @@ -301,6 +307,7 @@ kore_python_init(void) TAILQ_INIT(&prereq); TAILQ_INIT(&procs); + TAILQ_INIT(&routes); TAILQ_INIT(&coro_runnable); TAILQ_INIT(&coro_suspended); @@ -351,6 +358,9 @@ kore_python_init(void) kore_seccomp_filter("python", filter_python, KORE_FILTER_LEN(filter_python)); #endif + + if (!kore_configure_setting("deployment", "dev")) + fatal("failed to set initial deployment"); } void @@ -449,6 +459,19 @@ kore_python_coro_pending(void) return (!TAILQ_EMPTY(&coro_runnable)); } +void +kore_python_routes_resolve(void) +{ + struct pyroute *route; + + while ((route = TAILQ_FIRST(&routes)) != NULL) { + TAILQ_REMOVE(&routes, route, list); + if (!python_route_install(route)) + fatalx("failed to install route for %s", route->path); + Py_DECREF((PyObject *)route); + } +} + void kore_python_log_error(const char *function) { @@ -1570,6 +1593,7 @@ python_module_init(void) python_push_type("pylock", pykore, &pylock_type); python_push_type("pytimer", pykore, &pytimer_type); python_push_type("pyqueue", pykore, &pyqueue_type); + python_push_type("pyroute", pykore, &pyroute_type); python_push_type("pysocket", pykore, &pysocket_type); python_push_type("pydomain", pykore, &pydomain_type); python_push_type("pyconnection", pykore, &pyconnection_type); @@ -1756,9 +1780,6 @@ python_kore_server(PyObject *self, PyObject *args, PyObject *kwargs) struct kore_server *srv; const char *name, *ip, *port, *path; - if (!PyArg_ParseTuple(args, "s", &name)) - return (NULL); - if (kwargs == NULL) { PyErr_SetString(PyExc_RuntimeError, "missing keyword args"); return (NULL); @@ -1778,6 +1799,16 @@ python_kore_server(PyObject *self, PyObject *args, PyObject *kwargs) return (NULL); } + name = python_string_from_dict(kwargs, "name"); + if (name == NULL) + name = "default"; + + if ((srv = kore_server_lookup(name)) != NULL) { + PyErr_Format(PyExc_RuntimeError, + "server '%s' already exist", name); + return (NULL); + } + srv = kore_server_create(name); python_bool_from_dict(kwargs, "tls", &srv->tls); @@ -2011,16 +2042,11 @@ python_kore_domain(PyObject *self, PyObject *args, PyObject *kwargs) if (!PyArg_ParseTuple(args, "s", &name)) return (NULL); - if (kwargs == NULL) { - PyErr_SetString(PyExc_RuntimeError, "missing keyword args"); - return (NULL); - } + if (kwargs != NULL) + attach = python_string_from_dict(kwargs, "attach"); - if ((attach = python_string_from_dict(kwargs, "attach")) == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "missing or invalid 'attach' keyword"); - return (NULL); - } + if (attach == NULL) + attach = "default"; if ((srv = kore_server_lookup(attach)) == NULL) { PyErr_Format(PyExc_RuntimeError, @@ -2029,6 +2055,11 @@ python_kore_domain(PyObject *self, PyObject *args, PyObject *kwargs) } if (srv->tls) { + if (kwargs == NULL) { + PyErr_Format(PyExc_RuntimeError, + "no keywords for TLS enabled domain %s", name); + return (NULL); + } key = python_string_from_dict(kwargs, "key"); cert = python_string_from_dict(kwargs, "cert"); @@ -2069,6 +2100,9 @@ python_kore_domain(PyObject *self, PyObject *args, PyObject *kwargs) if ((domain = PyObject_New(struct pydomain, &pydomain_type)) == NULL) return (NULL); + domain->next = NULL; + domain->kwargs = NULL; + if ((domain->config = kore_domain_new(name)) == NULL) fatal("failed to create new domain configuration"); @@ -2096,6 +2130,35 @@ python_kore_domain(PyObject *self, PyObject *args, PyObject *kwargs) return ((PyObject *)domain); } +static PyObject * +python_kore_route(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *path; + PyObject *inner; + struct pyroute *route; + + if ((route = PyObject_New(struct pyroute, &pyroute_type)) == NULL) + return (NULL); + + if (!PyArg_ParseTuple(args, "s", &path)) + return (NULL); + + route->domain = NULL; + route->kwargs = kwargs; + route->path = kore_strdup(path); + + Py_XINCREF(route->kwargs); + + inner = PyObject_GetAttrString((PyObject *)route, "inner"); + if (inner == NULL) { + Py_DECREF((PyObject *)route); + PyErr_SetString(PyExc_RuntimeError, "failed to find inner"); + return (NULL); + } + + return (inner); +} + static PyObject * python_kore_gather(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -4931,6 +4994,36 @@ pyhttp_file_get_filename(struct pyhttp_file *pyfile, void *closure) return (name); } +void +pyroute_dealloc(struct pyroute *route) +{ + kore_free(route->path); + + Py_XDECREF(route->func); + Py_XDECREF(route->kwargs); + + PyObject_Del((PyObject *)route); +} + +static PyObject * +pyroute_inner(struct pyroute *route, PyObject *args) +{ + PyObject *obj; + + if (!PyArg_ParseTuple(args, "O", &obj)) + return (NULL); + + if (!PyCallable_Check(obj)) + return (NULL); + + route->func = obj; + Py_INCREF(route->func); + + TAILQ_INSERT_TAIL(&routes, route, list); + + return (route->func); +} + void pydomain_dealloc(struct pydomain *domain) { @@ -5004,114 +5097,179 @@ pydomain_filemaps(struct pydomain *domain, PyObject *args) static PyObject * pydomain_route(struct pydomain *domain, PyObject *args, PyObject *kwargs) { - struct kore_module_handle *hdlr; - int method; - const char *path, *val; - Py_ssize_t list_len, idx; - PyObject *callable, *repr, *obj, *item; + PyObject *obj; + const char *path; + struct pyroute *route; - if (!PyArg_ParseTuple(args, "sO", &path, &callable)) + if (!PyArg_ParseTuple(args, "sO", &path, &obj)) return (NULL); - if (!PyCallable_Check(callable)) + if (!PyCallable_Check(obj)) return (NULL); - TAILQ_FOREACH(hdlr, &domain->config->handlers, list) { - if (!strcmp(hdlr->path, path)) { - PyErr_Format(PyExc_RuntimeError, - "route '%s' exists", path); - return (NULL); - } - } - - if ((repr = PyObject_Repr(callable)) == NULL) + if ((route = PyObject_New(struct pyroute, &pyroute_type)) == NULL) return (NULL); - val = PyUnicode_AsUTF8(repr); + route->kwargs = kwargs; + route->domain = domain->config; + route->path = kore_strdup(path); - hdlr = kore_calloc(1, sizeof(*hdlr)); - hdlr->dom = domain->config; - hdlr->func = kore_strdup(val); - hdlr->path = kore_strdup(path); - hdlr->methods = HTTP_METHOD_ALL; - TAILQ_INIT(&hdlr->params); + Py_XINCREF(route->kwargs); - Py_DECREF(repr); + route->func = obj; + Py_INCREF(route->func); - hdlr->rcall = kore_calloc(1, sizeof(struct kore_runtime_call)); - hdlr->rcall->addr = callable; - hdlr->rcall->runtime = &kore_python_runtime; - - if (kwargs != NULL) { - if ((obj = PyDict_GetItemString(kwargs, "methods")) != NULL) { - if (!PyList_CheckExact(obj)) { - kore_module_handler_free(hdlr); - return (NULL); - } - - hdlr->methods = 0; - list_len = PyList_Size(obj); - - for (idx = 0; idx < list_len; idx++) { - if ((item = PyList_GetItem(obj, idx)) == NULL) { - kore_module_handler_free(hdlr); - return (NULL); - } - - if ((val = PyUnicode_AsUTF8(item)) == NULL) { - kore_module_handler_free(hdlr); - return (NULL); - } - - method = http_method_value(val); - if (method == 0) { - PyErr_Format(PyExc_RuntimeError, - "unknown method '%s'", val); - kore_module_handler_free(hdlr); - return (NULL); - } - - hdlr->methods |= method; - if (method == HTTP_METHOD_GET) - hdlr->methods |= HTTP_METHOD_HEAD; - - if (!pydomain_params(kwargs, - hdlr, val, method)) { - kore_module_handler_free(hdlr); - return (NULL); - } - } - } - - if ((obj = PyDict_GetItemString(kwargs, "auth")) != NULL) { - if (!pydomain_auth(obj, hdlr)) { - kore_module_handler_free(hdlr); - return (NULL); - } - } - } - - if (path[0] == '/') { - hdlr->type = HANDLER_TYPE_STATIC; - } else { - hdlr->type = HANDLER_TYPE_DYNAMIC; - - if (regcomp(&hdlr->rctx, hdlr->path, REG_EXTENDED)) { - PyErr_SetString(PyExc_RuntimeError, - "failed to compile regex for path"); - kore_module_handler_free(hdlr); - return (NULL); - } - } - - Py_INCREF(callable); - TAILQ_INSERT_TAIL(&domain->config->handlers, hdlr, list); + TAILQ_INSERT_TAIL(&routes, route, list); Py_RETURN_NONE; } static int -pydomain_params(PyObject *kwargs, struct kore_module_handle *hdlr, +python_route_install(struct pyroute *route) +{ + const char *val; + struct kore_domain *domain; + struct kore_module_handle *hdlr, *entry; + PyObject *kwargs, *repr, *obj; + + if ((repr = PyObject_Repr(route->func)) == NULL) { + kore_python_log_error("python_route_install"); + return (KORE_RESULT_ERROR); + } + + domain = python_route_domain_resolve(route); + + hdlr = kore_calloc(1, sizeof(*hdlr)); + hdlr->dom = domain; + hdlr->methods = HTTP_METHOD_ALL; + hdlr->path = kore_strdup(route->path); + + TAILQ_INIT(&hdlr->params); + + val = PyUnicode_AsUTF8(repr); + hdlr->func = kore_strdup(val); + + kwargs = route->kwargs; + + hdlr->rcall = kore_calloc(1, sizeof(struct kore_runtime_call)); + hdlr->rcall->addr = route->func; + hdlr->rcall->runtime = &kore_python_runtime; + Py_INCREF(hdlr->rcall->addr); + + if (kwargs != NULL) { + if ((obj = PyDict_GetItemString(kwargs, "methods")) != NULL) { + if (!python_route_methods(obj, kwargs, hdlr)) { + kore_python_log_error("python_route_install"); + kore_module_handler_free(hdlr); + return (KORE_RESULT_ERROR); + } + } + + if ((obj = PyDict_GetItemString(kwargs, "auth")) != NULL) { + if (!python_route_auth(obj, hdlr)) { + kore_python_log_error("python_route_install"); + kore_module_handler_free(hdlr); + return (KORE_RESULT_ERROR); + } + } + } + + if (hdlr->path[0] == '/') { + hdlr->type = HANDLER_TYPE_STATIC; + } else { + hdlr->type = HANDLER_TYPE_DYNAMIC; + if (regcomp(&hdlr->rctx, hdlr->path, REG_EXTENDED)) + fatal("failed to compile regex for '%s'", hdlr->path); + } + + TAILQ_FOREACH(entry, &domain->handlers, list) { + if (!strcmp(entry->path, hdlr->path) && + (entry->methods & hdlr->methods)) + fatal("duplicate route for '%s'", route->path); + } + + TAILQ_INSERT_TAIL(&domain->handlers, hdlr, list); + + return (KORE_RESULT_OK); +} + +static struct kore_domain * +python_route_domain_resolve(struct pyroute *route) +{ + struct kore_server *srv; + const char *name; + struct kore_domain *domain; + + if (route->domain != NULL) + return (route->domain); + + if (route->kwargs != NULL) + name = python_string_from_dict(route->kwargs, "domain"); + else + name = NULL; + + if (name != NULL) { + domain = NULL; + LIST_FOREACH(srv, &kore_servers, list) { + TAILQ_FOREACH(domain, &srv->domains, list) { + if (!strcmp(domain->domain, name)) + break; + } + } + + if (domain == NULL) + fatal("domain '%s' does not exist", name); + } else { + if ((domain = kore_domain_byid(1)) != NULL) + fatal("ambiguous domain on route, please specify one"); + if ((domain = kore_domain_byid(0)) == NULL) + fatal("no domains configured, please configure one"); + } + + return (domain); +} + +static int +python_route_methods(PyObject *obj, PyObject *kwargs, + struct kore_module_handle *hdlr) +{ + const char *val; + PyObject *item; + int method; + Py_ssize_t list_len, idx; + + if (!PyList_CheckExact(obj)) + return (KORE_RESULT_ERROR); + + hdlr->methods = 0; + list_len = PyList_Size(obj); + + for (idx = 0; idx < list_len; idx++) { + if ((item = PyList_GetItem(obj, idx)) == NULL) + return (KORE_RESULT_ERROR); + + if ((val = PyUnicode_AsUTF8(item)) == NULL) + return (KORE_RESULT_ERROR); + + if ((method = http_method_value(val)) == 0) { + PyErr_Format(PyExc_RuntimeError, + "unknown HTTP method: %s", val); + return (KORE_RESULT_ERROR); + } + + hdlr->methods |= method; + if (method == HTTP_METHOD_GET) + hdlr->methods |= HTTP_METHOD_HEAD; + + if (!python_route_params(kwargs, hdlr, val, method)) + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int +python_route_params(PyObject *kwargs, struct kore_module_handle *hdlr, const char *method, int type) { Py_ssize_t idx; @@ -5182,7 +5340,7 @@ pydomain_params(PyObject *kwargs, struct kore_module_handle *hdlr, } static int -pydomain_auth(PyObject *dict, struct kore_module_handle *hdlr) +python_route_auth(PyObject *dict, struct kore_module_handle *hdlr) { int type; struct kore_auth *auth; From 582e18d2ec4ffbe4ca3b05499fb9eaee1c16b451 Mon Sep 17 00:00:00 2001 From: Frederic Cambus Date: Wed, 21 Apr 2021 14:36:39 +0200 Subject: [PATCH 025/121] Stop hardcoding HTTP error codes in http_response() calls. Use predefined HTTP_STATUS_* macros instead. --- src/auth.c | 4 ++-- src/jsonrpc.c | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/auth.c b/src/auth.c index df3db8d..4ac836e 100644 --- a/src/auth.c +++ b/src/auth.c @@ -90,12 +90,12 @@ kore_auth_run(struct http_request *req, struct kore_auth *auth) kore_debug("kore_auth_run() for %s failed", req->path); if (auth->redirect == NULL) { - http_response(req, 403, NULL, 0); + http_response(req, HTTP_STATUS_FORBIDDEN, NULL, 0); return (KORE_RESULT_ERROR); } http_response_header(req, "location", auth->redirect); - http_response(req, 302, NULL, 0); + http_response(req, HTTP_STATUS_FOUND, NULL, 0); return (KORE_RESULT_ERROR); } diff --git a/src/jsonrpc.c b/src/jsonrpc.c index 7e295d8..fee40ff 100644 --- a/src/jsonrpc.c +++ b/src/jsonrpc.c @@ -423,13 +423,13 @@ jsonrpc_error(struct jsonrpc_request *req, int code, const char *msg) http_response_header(req->http, "content-type", "application/json"); yajl_gen_get_buf(req->gen, &body, &body_len); succeeded: - http_response(req->http, 200, body, body_len); + http_response(req->http, HTTP_STATUS_OK, body, body_len); if (req->gen != NULL) yajl_gen_clear(req->gen); jsonrpc_destroy_request(req); return (KORE_RESULT_OK); failed: - http_response(req->http, 500, NULL, 0); + http_response(req->http, HTTP_STATUS_INTERNAL_ERROR, NULL, 0); jsonrpc_destroy_request(req); return (KORE_RESULT_OK); } @@ -466,13 +466,13 @@ jsonrpc_result(struct jsonrpc_request *req, http_response_header(req->http, "content-type", "application/json"); yajl_gen_get_buf(req->gen, &body, &body_len); succeeded: - http_response(req->http, 200, body, body_len); + http_response(req->http, HTTP_STATUS_OK, body, body_len); if (req->gen != NULL) yajl_gen_clear(req->gen); jsonrpc_destroy_request(req); return (KORE_RESULT_OK); failed: - http_response(req->http, 500, NULL, 0); + http_response(req->http, HTTP_STATUS_INTERNAL_ERROR, NULL, 0); jsonrpc_destroy_request(req); return (KORE_RESULT_OK); } From 5e84ebdab23e83d9bf9524162c3a95cddb2405b2 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 2 May 2021 16:07:41 +0200 Subject: [PATCH 026/121] Simplify the echo example, kore can just run the file nowadays. --- examples/python-echo/README.md | 11 ++++ examples/python-echo/conf/build.conf | 34 ----------- examples/python-echo/conf/python-echo.conf | 3 - examples/python-echo/src/echo.py | 67 ---------------------- examples/python-echo/src/init.c | 25 -------- 5 files changed, 11 insertions(+), 129 deletions(-) create mode 100644 examples/python-echo/README.md delete mode 100644 examples/python-echo/conf/build.conf delete mode 100644 examples/python-echo/conf/python-echo.conf delete mode 100644 examples/python-echo/src/echo.py delete mode 100644 examples/python-echo/src/init.c diff --git a/examples/python-echo/README.md b/examples/python-echo/README.md new file mode 100644 index 0000000..65b53d8 --- /dev/null +++ b/examples/python-echo/README.md @@ -0,0 +1,11 @@ +Example of using the asynchronous python api to create a simple +echo server. + +Kore must have been built with PYTHON=1. + +On the command-line run the following + +$ kore echo.py + +Then connect to 127.0.0.1 port 6969 using netcat or so and you'll +see it echo back everything you send it. diff --git a/examples/python-echo/conf/build.conf b/examples/python-echo/conf/build.conf deleted file mode 100644 index 253613e..0000000 --- a/examples/python-echo/conf/build.conf +++ /dev/null @@ -1,34 +0,0 @@ -# python-echo build config -# You can switch flavors using: kodev flavor [newflavor] - -# Set to yes if you wish to produce a single binary instead -# of a dynamic library. If you set this to yes you must also -# set kore_source together with kore_flavor. -single_binary=yes -kore_source=../../ -kore_flavor=NOTLS=1 PYTHON=1 - -# The flags below are shared between flavors -cflags=-Wall -Wmissing-declarations -Wshadow -cflags=-Wstrict-prototypes -Wmissing-prototypes -cflags=-Wpointer-arith -Wcast-qual -Wsign-compare - -cxxflags=-Wall -Wmissing-declarations -Wshadow -cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare - -# Mime types for assets served via the builtin asset_serve_* -#mime_add=txt:text/plain; charset=utf-8 -#mime_add=png:image/png -#mime_add=html:text/html; charset=utf-8 - -dev { - # These flags are added to the shared ones when - # you build the "dev" flavor. - cflags=-g - cxxflags=-g -} - -#prod { -# You can specify additional flags here which are only -# included if you build with the "prod" flavor. -#} diff --git a/examples/python-echo/conf/python-echo.conf b/examples/python-echo/conf/python-echo.conf deleted file mode 100644 index f9e88a4..0000000 --- a/examples/python-echo/conf/python-echo.conf +++ /dev/null @@ -1,3 +0,0 @@ -# python-echo configuration - -python_import src/echo.py diff --git a/examples/python-echo/src/echo.py b/examples/python-echo/src/echo.py deleted file mode 100644 index ea42b24..0000000 --- a/examples/python-echo/src/echo.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) 2013-2018 Joris Vink -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# - -import kore -import socket - -class EchoServer: - # Setup socket + wrap it inside of a kore socket so we can use it. - def __init__(self): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setblocking(False) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("127.0.0.1", 6969)) - sock.listen() - - self.conn = kore.socket_wrap(sock) - - # Wait for a new client to connect, then create a new task - # that calls handle_client with the ocnnected client as - # the argument. - async def run(self): - while True: - try: - client = await self.conn.accept() - kore.task_create(self.handle_client(client)) - client = None - except Exception as e: - kore.fatal("exception %s" % e) - - # Each client will run as this co-routine. - # In this case we pass a timeout of 1 second to the recv() call - # which will throw a TimeoutError exception in case the timeout - # is hit before data is read from the socket. - # - # This timeout argument is optional. If none is specified the call - # will wait until data becomes available. - async def handle_client(self, client): - while True: - try: - data = await client.recv(1024, 1000) - if data is None: - break - await client.send(data) - except TimeoutError as e: - print("timed out reading (%s)" % e) - except Exception as e: - print("client got exception %s" % e) - client.close() - -# Setup the server object. -server = EchoServer() - -# Create a task that will execute inside of Kore as a co-routine. -kore.task_create(server.run()) diff --git a/examples/python-echo/src/init.c b/examples/python-echo/src/init.c deleted file mode 100644 index d40c00b..0000000 --- a/examples/python-echo/src/init.c +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2020 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include -#include - -/* Let kore handle the default option parsing. */ -void -kore_parent_configure(int argc, char **argv) -{ - kore_default_getopt(argc, argv); -} From 66dd856bdc962b2d2b3c46074a27be4c9c597fcd Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 2 May 2021 16:08:32 +0200 Subject: [PATCH 027/121] and add the echo.py file of course.. --- examples/python-echo/echo.py | 67 ++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 examples/python-echo/echo.py diff --git a/examples/python-echo/echo.py b/examples/python-echo/echo.py new file mode 100644 index 0000000..ea42b24 --- /dev/null +++ b/examples/python-echo/echo.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2013-2018 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +import kore +import socket + +class EchoServer: + # Setup socket + wrap it inside of a kore socket so we can use it. + def __init__(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(False) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", 6969)) + sock.listen() + + self.conn = kore.socket_wrap(sock) + + # Wait for a new client to connect, then create a new task + # that calls handle_client with the ocnnected client as + # the argument. + async def run(self): + while True: + try: + client = await self.conn.accept() + kore.task_create(self.handle_client(client)) + client = None + except Exception as e: + kore.fatal("exception %s" % e) + + # Each client will run as this co-routine. + # In this case we pass a timeout of 1 second to the recv() call + # which will throw a TimeoutError exception in case the timeout + # is hit before data is read from the socket. + # + # This timeout argument is optional. If none is specified the call + # will wait until data becomes available. + async def handle_client(self, client): + while True: + try: + data = await client.recv(1024, 1000) + if data is None: + break + await client.send(data) + except TimeoutError as e: + print("timed out reading (%s)" % e) + except Exception as e: + print("client got exception %s" % e) + client.close() + +# Setup the server object. +server = EchoServer() + +# Create a task that will execute inside of Kore as a co-routine. +kore.task_create(server.run()) From 0abc9b19ffe51be1ed95ce6bb35832cb0fdb24cd Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 2 May 2021 16:25:46 +0200 Subject: [PATCH 028/121] simplify the python-async example --- examples/python-async/README.md | 4 ++- .../python-async/{src/setup.py => app.py} | 13 ++++--- examples/python-async/{src => }/async_http.py | 1 + examples/python-async/{src => }/async_lock.py | 1 + .../python-async/{src => }/async_process.py | 1 + .../python-async/{src => }/async_queue.py | 1 + .../python-async/{src => }/async_socket.py | 2 ++ examples/python-async/conf/build.conf | 34 ------------------- examples/python-async/conf/python-async.conf | 28 --------------- examples/python-async/src/init.c | 25 -------------- 10 files changed, 18 insertions(+), 92 deletions(-) rename examples/python-async/{src/setup.py => app.py} (78%) rename examples/python-async/{src => }/async_http.py (97%) rename examples/python-async/{src => }/async_lock.py (97%) rename examples/python-async/{src => }/async_process.py (98%) rename examples/python-async/{src => }/async_queue.py (97%) rename examples/python-async/{src => }/async_socket.py (95%) delete mode 100644 examples/python-async/conf/build.conf delete mode 100644 examples/python-async/conf/python-async.conf delete mode 100644 examples/python-async/src/init.c diff --git a/examples/python-async/README.md b/examples/python-async/README.md index 97451e3..422a07f 100644 --- a/examples/python-async/README.md +++ b/examples/python-async/README.md @@ -3,9 +3,11 @@ Kore python async/await examples. This example also shows off the asynchronous HTTP client support and requires libcurl on your machine. +Requires that Kore is built with PYTHON=1 CURL=1 + Run: ``` - $ kodev run + $ kore app.py ``` Test: diff --git a/examples/python-async/src/setup.py b/examples/python-async/app.py similarity index 78% rename from examples/python-async/src/setup.py rename to examples/python-async/app.py index ab01e61..ee77372 100644 --- a/examples/python-async/src/setup.py +++ b/examples/python-async/app.py @@ -16,8 +16,13 @@ import kore -from async_queue import queue_helper +import async_http +import async_queue +import async_socket +import async_process +import async_process -# Kore worker started, start the queue helper coroutine. -def kore_worker_configure(): - kore.task_create(queue_helper()) +kore.server(ip="127.0.0.1", port="8888", tls=False) +kore.domain("*") + +kore.task_create(async_queue.queue_helper()) diff --git a/examples/python-async/src/async_http.py b/examples/python-async/async_http.py similarity index 97% rename from examples/python-async/src/async_http.py rename to examples/python-async/async_http.py index 854919b..1e20bfa 100644 --- a/examples/python-async/src/async_http.py +++ b/examples/python-async/async_http.py @@ -21,6 +21,7 @@ import kore # Handler called for /httpclient +@kore.route("/httpclient", methods=["get"]) async def httpclient(req): # Create an httpclient. client = kore.httpclient("https://kore.io") diff --git a/examples/python-async/src/async_lock.py b/examples/python-async/async_lock.py similarity index 97% rename from examples/python-async/src/async_lock.py rename to examples/python-async/async_lock.py index dceb1a9..20a93da 100644 --- a/examples/python-async/src/async_lock.py +++ b/examples/python-async/async_lock.py @@ -28,6 +28,7 @@ import kore # The shared lock lock = kore.lock() +@kore.route("/lock", methods=["get"]) async def async_lock(req): # A kore.lock should be used with the "async with" syntax. async with lock: diff --git a/examples/python-async/src/async_process.py b/examples/python-async/async_process.py similarity index 98% rename from examples/python-async/src/async_process.py rename to examples/python-async/async_process.py index 2add851..95d4604 100644 --- a/examples/python-async/src/async_process.py +++ b/examples/python-async/async_process.py @@ -25,6 +25,7 @@ import kore import json +@kore.route("/proc", methods=["get"]) async def async_proc(req): # # You may specify a timeout when creating the kore.proc object. diff --git a/examples/python-async/src/async_queue.py b/examples/python-async/async_queue.py similarity index 97% rename from examples/python-async/src/async_queue.py rename to examples/python-async/async_queue.py index 03fd100..ffb6949 100644 --- a/examples/python-async/src/async_queue.py +++ b/examples/python-async/async_queue.py @@ -36,6 +36,7 @@ async def queue_helper(): # Send it on the received queue. obj["rq"].push(msg) +@kore.route("/queue", methods=["get"]) async def async_queue(req): # Create our own queue. rq = kore.queue() diff --git a/examples/python-async/src/async_socket.py b/examples/python-async/async_socket.py similarity index 95% rename from examples/python-async/src/async_socket.py rename to examples/python-async/async_socket.py index baca675..f0dbf77 100644 --- a/examples/python-async/src/async_socket.py +++ b/examples/python-async/async_socket.py @@ -23,6 +23,7 @@ import kore import socket +@kore.route("/socket", methods=["get"]) async def async_socket(req): # Create the socket using Pythons built-in socket class. sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -51,6 +52,7 @@ async def async_socket(req): conn.close() +@kore.route("/socket-test", methods=["get"]) async def socket_test(req): # Delay response a bit, just cause we can. await kore.suspend(5000) diff --git a/examples/python-async/conf/build.conf b/examples/python-async/conf/build.conf deleted file mode 100644 index 87c8b7b..0000000 --- a/examples/python-async/conf/build.conf +++ /dev/null @@ -1,34 +0,0 @@ -# python-async build config -# You can switch flavors using: kodev flavor [newflavor] - -# Set to yes if you wish to produce a single binary instead -# of a dynamic library. If you set this to yes you must also -# set kore_source together with kore_flavor. -single_binary=yes -kore_source=../../ -kore_flavor=PYTHON=1 CURL=1 NOTLS=1 DEBUG=1 - -# The flags below are shared between flavors -cflags=-Wall -Wmissing-declarations -Wshadow -cflags=-Wstrict-prototypes -Wmissing-prototypes -cflags=-Wpointer-arith -Wcast-qual -Wsign-compare - -cxxflags=-Wall -Wmissing-declarations -Wshadow -cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare - -# Mime types for assets served via the builtin asset_serve_* -#mime_add=txt:text/plain; charset=utf-8 -#mime_add=png:image/png -#mime_add=html:text/html; charset=utf-8 - -dev { - # These flags are added to the shared ones when - # you build the "dev" flavor. - cflags=-g - cxxflags=-g -} - -#prod { -# You can specify additional flags here which are only -# included if you build with the "prod" flavor. -#} diff --git a/examples/python-async/conf/python-async.conf b/examples/python-async/conf/python-async.conf deleted file mode 100644 index 6d68786..0000000 --- a/examples/python-async/conf/python-async.conf +++ /dev/null @@ -1,28 +0,0 @@ -# python-async configuration - -server notls { - tls no - bind 127.0.0.1 8888 -} - -python_path src - -python_import ./src/setup.py -python_import ./src/async_lock.py -python_import ./src/async_queue.py -python_import ./src/async_process.py -python_import ./src/async_socket.py -python_import ./src/async_http.py - -domain * { - attach notls - - route /queue async_queue - route /lock async_lock - route /proc async_proc - - route /socket async_socket - route /socket-test socket_test - - route /httpclient httpclient -} diff --git a/examples/python-async/src/init.c b/examples/python-async/src/init.c deleted file mode 100644 index d40c00b..0000000 --- a/examples/python-async/src/init.c +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2020 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include -#include - -/* Let kore handle the default option parsing. */ -void -kore_parent_configure(int argc, char **argv) -{ - kore_default_getopt(argc, argv); -} From 61c06291b62ce238a332a651b26cfca9f531f20d Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 10 May 2021 10:27:32 +0200 Subject: [PATCH 029/121] Add a docker python kore.config.deployment setting. This keeps kore in the foreground will still doing privsep. Useful with upcoming official kore docker images. --- src/config.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 7f3466e..6c2035c 100644 --- a/src/config.c +++ b/src/config.c @@ -1866,7 +1866,9 @@ configure_task_threads(char *option) static int configure_deployment(char *value) { - if (!strcmp(value, "dev") || !strcmp(value, "development")) { + if (!strcmp(value, "docker")) { + kore_foreground = 1; + } else if (!strcmp(value, "dev") || !strcmp(value, "development")) { kore_foreground = 1; skip_runas = 1; skip_chroot = 1; From 12909b98bbb2c120d547949163f810a302fedff9 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 10 May 2021 10:32:54 +0200 Subject: [PATCH 030/121] be explicit --- src/config.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config.c b/src/config.c index 6c2035c..c7c0a6c 100644 --- a/src/config.c +++ b/src/config.c @@ -1868,6 +1868,8 @@ configure_deployment(char *value) { if (!strcmp(value, "docker")) { kore_foreground = 1; + skip_runas = 0; + skip_chroot = 0; } else if (!strcmp(value, "dev") || !strcmp(value, "development")) { kore_foreground = 1; skip_runas = 1; From d4a78de5fcb8431dc85b8a77475c3ff501d05e16 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 10 May 2021 14:51:30 +0200 Subject: [PATCH 031/121] Add madvise to keymgr seccomp rules. --- src/keymgr.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/keymgr.c b/src/keymgr.c index b53ed18..bba1940 100644 --- a/src/keymgr.c +++ b/src/keymgr.c @@ -137,6 +137,9 @@ static struct sock_filter filter_keymgr[] = { #endif #if defined(SYS_mmap2) KORE_SYSCALL_ALLOW(mmap2), +#endif +#if defined(SYS_madvise) + KORE_SYSCALL_ALLOW(madvise), #endif KORE_SYSCALL_ALLOW(munmap), KORE_SYSCALL_ALLOW(clock_gettime), From f39919e98c9051175d920bc6a626e8115e0afbd1 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 10 May 2021 23:05:58 +0200 Subject: [PATCH 032/121] Add readlinkat to seccomp whitelist. --- src/seccomp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/seccomp.c b/src/seccomp.c index f20b3db..7d86aa1 100644 --- a/src/seccomp.c +++ b/src/seccomp.c @@ -90,6 +90,9 @@ static struct sock_filter filter_kore[] = { #if defined(SYS_readlink) KORE_SYSCALL_ALLOW(readlink), #endif +#if defined(SYS_readlinkat) + KORE_SYSCALL_ALLOW(readlinkat), +#endif /* Process related. */ KORE_SYSCALL_ALLOW(exit), From f1a65ef2368d6704edad84176c86068249e25678 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Thu, 3 Jun 2021 14:02:28 +0200 Subject: [PATCH 033/121] Small improvement to the Python kore.timer() api. Do not allow kore.timer() to be called from the parent process as it shouldn't be run there. This makes Kore fail more gracefully. --- src/python.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/python.c b/src/python.c index 8ae91f0..b23aa6b 100644 --- a/src/python.c +++ b/src/python.c @@ -2441,6 +2441,12 @@ python_kore_timer(PyObject *self, PyObject *args, PyObject *kwargs) int flags; struct pytimer *timer; + if (worker == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "kore.timer not supported on parent process"); + return (NULL); + } + if (!PyArg_ParseTuple(args, "OKi", &obj, &ms, &flags)) return (NULL); From 95139925ec82fe71d451114a72c1b6c0fae90677 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 18 Jun 2021 13:00:57 +0200 Subject: [PATCH 034/121] Add query string support to the Python validator API. Now you can specify the qs keyword in a route which can contain validators for the query string. Eg: @kore.route("/", methods=["post"], qs={"id": "^[0-9]+$"}) def index: ... --- src/python.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/python.c b/src/python.c index b23aa6b..d59246f 100644 --- a/src/python.c +++ b/src/python.c @@ -81,7 +81,8 @@ static struct kore_domain *python_route_domain_resolve(struct pyroute *); static int python_route_install(struct pyroute *); static int python_route_params(PyObject *, - struct kore_module_handle *, const char *, int); + struct kore_module_handle *, const char *, + int, int); static int python_route_methods(PyObject *, PyObject *, struct kore_module_handle *); static int python_route_auth(PyObject *, @@ -5267,7 +5268,10 @@ python_route_methods(PyObject *obj, PyObject *kwargs, if (method == HTTP_METHOD_GET) hdlr->methods |= HTTP_METHOD_HEAD; - if (!python_route_params(kwargs, hdlr, val, method)) + if (!python_route_params(kwargs, hdlr, val, method, 0)) + return (KORE_RESULT_ERROR); + + if (!python_route_params(kwargs, hdlr, "qs", method, 1)) return (KORE_RESULT_ERROR); } @@ -5276,7 +5280,7 @@ python_route_methods(PyObject *obj, PyObject *kwargs, static int python_route_params(PyObject *kwargs, struct kore_module_handle *hdlr, - const char *method, int type) + const char *method, int type, int qs) { Py_ssize_t idx; const char *val; @@ -5336,7 +5340,7 @@ python_route_params(PyObject *kwargs, struct kore_module_handle *hdlr, param->validator = vldr; param->name = kore_strdup(val); - if (type == HTTP_METHOD_GET) + if (type == HTTP_METHOD_GET || qs == 1) param->flags = KORE_PARAMS_QUERY_STRING; TAILQ_INSERT_TAIL(&hdlr->params, param, list); From 90056dbdcb8be5575a3258da6250e1f969671e52 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sat, 10 Jul 2021 10:02:46 +0200 Subject: [PATCH 035/121] make python_module_init() non static --- src/python.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python.c b/src/python.c index d59246f..1cb9b8a 100644 --- a/src/python.c +++ b/src/python.c @@ -59,7 +59,8 @@ struct reqcall { TAILQ_HEAD(reqcall_list, reqcall); -static PyMODINIT_FUNC python_module_init(void); +PyMODINIT_FUNC python_module_init(void); + static PyObject *python_import(const char *); static PyObject *pyconnection_alloc(struct connection *); static PyObject *python_callable(PyObject *, const char *); @@ -1580,7 +1581,7 @@ python_runtime_connect(void *addr, struct connection *c) Py_DECREF(pyret); } -static PyMODINIT_FUNC +PyMODINIT_FUNC python_module_init(void) { int i; From 6d7c774ba2fd812c587cf948d8f474b34c7c50af Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sat, 10 Jul 2021 10:02:53 +0200 Subject: [PATCH 036/121] remove support for older openssl apis in keymgr. --- src/keymgr.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/keymgr.c b/src/keymgr.c index bba1940..234dfc8 100644 --- a/src/keymgr.c +++ b/src/keymgr.c @@ -693,12 +693,8 @@ keymgr_rsa_encrypt(struct kore_msg *msg, const void *data, struct key *key) u_int8_t buf[1024]; req = (const struct kore_keyreq *)data; - -#if defined(KORE_OPENSSL_NEWER_API) rsa = EVP_PKEY_get0_RSA(key->pkey); -#else - rsa = key->pkey->pkey.rsa; -#endif + keylen = RSA_size(rsa); if (req->data_len > keylen || keylen > sizeof(buf)) return; @@ -721,11 +717,8 @@ keymgr_ecdsa_sign(struct kore_msg *msg, const void *data, struct key *key) u_int8_t sig[1024]; req = (const struct kore_keyreq *)data; -#if defined(KORE_OPENSSL_NEWER_API) ec = EVP_PKEY_get0_EC_KEY(key->pkey); -#else - ec = key->pkey->pkey.ec; -#endif + len = ECDSA_size(ec); if (req->data_len > len || len > sizeof(sig)) return; @@ -801,14 +794,8 @@ keymgr_acme_init(void) kore_log(LOG_INFO, "loaded existing ACME account key"); } -#if defined(KORE_OPENSSL_NEWER_API) rsa = EVP_PKEY_get0_RSA(key->pkey); RSA_get0_key(rsa, &bn, &be, NULL); -#else - rsa = key->pkey->pkey.rsa; - be = rsa->e; - bn = rsa->n; -#endif e = keymgr_bignum_base64(be); n = keymgr_bignum_base64(bn); From 89085246e532c95fd59ac886218a5ec2d57b2071 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sat, 10 Jul 2021 10:03:01 +0200 Subject: [PATCH 037/121] style nits --- src/fileref.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fileref.c b/src/fileref.c index ad1decf..d1f8425 100644 --- a/src/fileref.c +++ b/src/fileref.c @@ -79,7 +79,7 @@ kore_fileref_create(struct kore_server *srv, const char *path, int fd, if (srv->tls == 0) { ref->fd = fd; } else { - if ((uintmax_t)size> SIZE_MAX) { + if ((uintmax_t)size > SIZE_MAX) { kore_pool_put(&ref_pool, ref); return (NULL); } From 55aaef875d489e1617c8d27433c6d4bc6197bcf9 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 27 Aug 2021 10:00:50 +0200 Subject: [PATCH 038/121] Add support for setting curlopts in kore.httpclient. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Much of the work done by Matthew Norström with minor cleanup by me. --- include/kore/python_methods.h | 22 ++-- misc/curl/python_curlopt.h | 2 +- src/python.c | 187 +++++++++++++++++++++++----------- 3 files changed, 140 insertions(+), 71 deletions(-) diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index 65895f7..da6b85d 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -838,12 +838,16 @@ struct pycurl_slist { LIST_ENTRY(pycurl_slist) list; }; +struct pycurl_data { + struct kore_curl curl; + LIST_HEAD(, pycurl_slist) slists; +}; + struct pycurl_handle { PyObject_HEAD - struct kore_curl curl; - char *url; - struct kore_buf *body; - LIST_HEAD(, pycurl_slist) slists; + char *url; + struct kore_buf *body; + struct pycurl_data data; }; struct pycurl_handle_op { @@ -863,11 +867,11 @@ static PyObject *pycurl_handle_run(struct pycurl_handle *, PyObject *); static PyObject *pycurl_handle_setopt(struct pycurl_handle *, PyObject *); static PyObject *pycurl_handle_setbody(struct pycurl_handle *, PyObject *); -static PyObject *pycurl_handle_setopt_string(struct pycurl_handle *, +static PyObject *pycurl_handle_setopt_string(struct pycurl_data *, int, PyObject *); -static PyObject *pycurl_handle_setopt_long(struct pycurl_handle *, +static PyObject *pycurl_handle_setopt_long(struct pycurl_data *, int, PyObject *); -static PyObject *pycurl_handle_setopt_slist(struct pycurl_handle *, +static PyObject *pycurl_handle_setopt_slist(struct pycurl_data *, int, PyObject *); static PyMethodDef pycurl_handle_methods[] = { @@ -911,6 +915,7 @@ struct pyhttp_client { char *tlskey; char *tlscert; char *cabundle; + PyObject *curlopt; int tlsverify; }; @@ -918,11 +923,12 @@ struct pyhttp_client_op { PyObject_HEAD int state; int headers; - struct kore_curl curl; struct python_coro *coro; struct pyhttp_client *client; + struct pycurl_data data; }; + static PyObject *pyhttp_client_op_await(PyObject *); static PyObject *pyhttp_client_op_iternext(struct pyhttp_client_op *); diff --git a/misc/curl/python_curlopt.h b/misc/curl/python_curlopt.h index 5b43821..9beb89b 100644 --- a/misc/curl/python_curlopt.h +++ b/misc/curl/python_curlopt.h @@ -3,7 +3,7 @@ struct { const char *name; int value; - PyObject *(*cb)(struct pycurl_handle *, int, PyObject *); + PyObject *(*cb)(struct pycurl_data *, int, PyObject *); } py_curlopt[] = { { "CURLOPT_WRITEDATA", 1, NULL }, { "CURLOPT_URL", 2, pycurl_handle_setopt_string }, diff --git a/src/python.c b/src/python.c index 1cb9b8a..256828c 100644 --- a/src/python.c +++ b/src/python.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "kore.h" #include "http.h" @@ -131,6 +132,8 @@ static void python_curl_http_callback(struct kore_curl *, void *); static void python_curl_handle_callback(struct kore_curl *, void *); static PyObject *pyhttp_client_request(struct pyhttp_client *, int, PyObject *); +static PyObject *python_kore_curl_setopt(struct pycurl_data *, + long, PyObject *); #endif static void python_append_path(const char *); @@ -5730,6 +5733,30 @@ pykore_pgsql_result(struct pykore_pgsql *pysql) #endif #if defined(KORE_USE_CURL) +static PyObject * +python_kore_curl_setopt(struct pycurl_data *data, long opt, PyObject *value) +{ + int i; + + for (i = 0; py_curlopt[i].name != NULL; i++) { + if (py_curlopt[i].value == opt) + break; + } + + if (py_curlopt[i].name == NULL) { + PyErr_Format(PyExc_RuntimeError, "invalid option '%d'", opt); + return (NULL); + } + + if (py_curlopt[i].cb == NULL) { + PyErr_Format(PyExc_RuntimeError, "option '%s' not implemented", + py_curlopt[i].name); + return (NULL); + } + + return (py_curlopt[i].cb(data, i, value)); +} + static PyObject * python_kore_curl_handle(PyObject *self, PyObject *args) { @@ -5744,12 +5771,12 @@ python_kore_curl_handle(PyObject *self, PyObject *args) return (NULL); handle->url = kore_strdup(url); - memset(&handle->curl, 0, sizeof(handle->curl)); + memset(&handle->data.curl, 0, sizeof(handle->data.curl)); handle->body = NULL; - LIST_INIT(&handle->slists); + LIST_INIT(&handle->data.slists); - if (!kore_curl_init(&handle->curl, handle->url, KORE_CURL_ASYNC)) { + if (!kore_curl_init(&handle->data.curl, handle->url, KORE_CURL_ASYNC)) { Py_DECREF((PyObject *)handle); PyErr_SetString(PyExc_RuntimeError, "failed to setup call"); return (NULL); @@ -5763,7 +5790,7 @@ pycurl_handle_dealloc(struct pycurl_handle *handle) { struct pycurl_slist *psl; - while ((psl = LIST_FIRST(&handle->slists))) { + while ((psl = LIST_FIRST(&handle->data.slists))) { LIST_REMOVE(psl, list); curl_slist_free_all(psl->slist); kore_free(psl); @@ -5773,7 +5800,7 @@ pycurl_handle_dealloc(struct pycurl_handle *handle) kore_buf_free(handle->body); kore_free(handle->url); - kore_curl_cleanup(&handle->curl); + kore_curl_cleanup(&handle->data.curl); PyObject_Del((PyObject *)handle); } @@ -5812,10 +5839,12 @@ pycurl_handle_setbody(struct pycurl_handle *handle, PyObject *args) kore_buf_append(handle->body, ptr, length); kore_buf_reset(handle->body); - curl_easy_setopt(handle->curl.handle, + curl_easy_setopt(handle->data.curl.handle, CURLOPT_READFUNCTION, kore_curl_frombuf); - curl_easy_setopt(handle->curl.handle, CURLOPT_READDATA, handle->body); - curl_easy_setopt(handle->curl.handle, CURLOPT_UPLOAD, 1); + curl_easy_setopt(handle->data.curl.handle, + CURLOPT_READDATA, handle->body); + + curl_easy_setopt(handle->data.curl.handle, CURLOPT_UPLOAD, 1); Py_RETURN_TRUE; } @@ -5823,34 +5852,17 @@ pycurl_handle_setbody(struct pycurl_handle *handle, PyObject *args) static PyObject * pycurl_handle_setopt(struct pycurl_handle *handle, PyObject *args) { - int i, opt; + int opt; PyObject *value; if (!PyArg_ParseTuple(args, "iO", &opt, &value)) return (NULL); - for (i = 0; py_curlopt[i].name != NULL; i++) { - if (py_curlopt[i].value == opt) - break; - } - - if (py_curlopt[i].name == NULL) { - PyErr_Format(PyExc_RuntimeError, "invalid option '%d'", opt); - return (NULL); - } - - if (py_curlopt[i].cb == NULL) { - PyErr_Format(PyExc_RuntimeError, "option '%s' not implemented", - py_curlopt[i].name); - return (NULL); - } - - return (py_curlopt[i].cb(handle, i, value)); + return (python_kore_curl_setopt(&handle->data, opt, value)); } static PyObject * -pycurl_handle_setopt_string(struct pycurl_handle *handle, int idx, - PyObject *obj) +pycurl_handle_setopt_string(struct pycurl_data *data, int idx, PyObject *obj) { const char *str; @@ -5864,14 +5876,14 @@ pycurl_handle_setopt_string(struct pycurl_handle *handle, int idx, if ((str = PyUnicode_AsUTF8(obj)) == NULL) return (NULL); - curl_easy_setopt(handle->curl.handle, + curl_easy_setopt(data->curl.handle, CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, str); Py_RETURN_TRUE; } static PyObject * -pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj) +pycurl_handle_setopt_long(struct pycurl_data *data, int idx, PyObject *obj) { long val; @@ -5887,14 +5899,14 @@ pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj) if (val == -1 && PyErr_Occurred()) return (NULL); - curl_easy_setopt(handle->curl.handle, + curl_easy_setopt(data->curl.handle, CURLOPTTYPE_LONG + py_curlopt[idx].value, val); Py_RETURN_TRUE; } static PyObject * -pycurl_handle_setopt_slist(struct pycurl_handle *handle, int idx, PyObject *obj) +pycurl_handle_setopt_slist(struct pycurl_data *data, int idx, PyObject *obj) { struct pycurl_slist *psl; PyObject *item; @@ -5928,9 +5940,9 @@ pycurl_handle_setopt_slist(struct pycurl_handle *handle, int idx, PyObject *obj) psl = kore_calloc(1, sizeof(*psl)); psl->slist = slist; - LIST_INSERT_HEAD(&handle->slists, psl, list); + LIST_INSERT_HEAD(&data->slists, psl, list); - curl_easy_setopt(handle->curl.handle, + curl_easy_setopt(data->curl.handle, CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, slist); Py_RETURN_TRUE; @@ -5951,7 +5963,8 @@ pycurl_handle_run(struct pycurl_handle *handle, PyObject *args) op->coro = coro_running; op->state = CURL_CLIENT_OP_RUN; - kore_curl_bind_callback(&handle->curl, python_curl_handle_callback, op); + kore_curl_bind_callback(&handle->data.curl, + python_curl_handle_callback, op); return ((PyObject *)op); } @@ -5978,7 +5991,7 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op) const u_int8_t *response; if (op->state == CURL_CLIENT_OP_RUN) { - kore_curl_run(&op->handle->curl); + kore_curl_run(&op->handle->data.curl); op->state = CURL_CLIENT_OP_RESULT; Py_RETURN_NONE; } @@ -5988,14 +6001,14 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op) op->handle->body = NULL; } - if (!kore_curl_success(&op->handle->curl)) { + if (!kore_curl_success(&op->handle->data.curl)) { /* Do not log the url here, may contain some sensitive data. */ PyErr_Format(PyExc_RuntimeError, "request failed: %s", - kore_curl_strerror(&op->handle->curl)); + kore_curl_strerror(&op->handle->data.curl)); return (NULL); } - kore_curl_response_as_bytes(&op->handle->curl, &response, &len); + kore_curl_response_as_bytes(&op->handle->data.curl, &response, &len); if ((result = PyBytes_FromStringAndSize((const char *)response, len)) == NULL) @@ -6010,6 +6023,7 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op) static PyObject * python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs) { + PyObject *vdict; struct pyhttp_client *client; const char *url, *v; @@ -6022,6 +6036,7 @@ python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs) client->unix = NULL; client->tlskey = NULL; + client->curlopt = NULL; client->tlscert = NULL; client->cabundle = NULL; @@ -6041,6 +6056,18 @@ python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs) if ((v = python_string_from_dict(kwargs, "unix")) != NULL) client->unix = kore_strdup(v); + if ((vdict = PyDict_GetItemString(kwargs, "curlopt")) != NULL) { + if (!PyDict_CheckExact(vdict)) { + Py_DECREF((PyObject *)client); + PyErr_SetString(PyExc_RuntimeError, + "curlopt must be a dict"); + return (NULL); + } + + client->curlopt = vdict; + Py_INCREF(client->curlopt); + } + python_bool_from_dict(kwargs, "tlsverify", &client->tlsverify); } @@ -6064,6 +6091,8 @@ pyhttp_client_dealloc(struct pyhttp_client *client) kore_free(client->tlscert); kore_free(client->cabundle); + Py_XDECREF(client->curlopt); + PyObject_Del((PyObject *)client); } @@ -6119,11 +6148,12 @@ pyhttp_client_options(struct pyhttp_client *client, PyObject *args, static PyObject * pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) { + long opt; struct pyhttp_client_op *op; char *ptr; const char *k, *v; Py_ssize_t length, idx; - PyObject *data, *headers, *key, *item; + PyObject *data, *headers, *key, *item, *value; ptr = NULL; length = 0; @@ -6170,12 +6200,12 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) default: fatal("%s: unknown method %d", __func__, m); } - + op = PyObject_New(struct pyhttp_client_op, &pyhttp_client_op_type); if (op == NULL) return (NULL); - if (!kore_curl_init(&op->curl, client->url, KORE_CURL_ASYNC)) { + if (!kore_curl_init(&op->data.curl, client->url, KORE_CURL_ASYNC)) { Py_DECREF((PyObject *)op); PyErr_SetString(PyExc_RuntimeError, "failed to setup call"); return (NULL); @@ -6184,43 +6214,68 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) op->headers = 0; op->coro = coro_running; op->state = CURL_CLIENT_OP_RUN; + LIST_INIT(&op->data.slists); Py_INCREF(client); op->client = client; - kore_curl_http_setup(&op->curl, m, ptr, length); - kore_curl_bind_callback(&op->curl, python_curl_http_callback, op); + kore_curl_http_setup(&op->data.curl, m, ptr, length); + kore_curl_bind_callback(&op->data.curl, python_curl_http_callback, op); /* Go in with our own bare hands. */ if (client->unix != NULL) { #if defined(__linux__) if (client->unix[0] == '@') { - curl_easy_setopt(op->curl.handle, + curl_easy_setopt(op->data.curl.handle, CURLOPT_ABSTRACT_UNIX_SOCKET, client->unix + 1); } else { - curl_easy_setopt(op->curl.handle, + curl_easy_setopt(op->data.curl.handle, CURLOPT_UNIX_SOCKET_PATH, client->unix); } #else - curl_easy_setopt(op->curl.handle, CURLOPT_UNIX_SOCKET_PATH, + curl_easy_setopt(op->data.curl.handle, CURLOPT_UNIX_SOCKET_PATH, client->unix); #endif } if (client->tlskey != NULL && client->tlscert != NULL) { - curl_easy_setopt(op->curl.handle, CURLOPT_SSLCERT, + curl_easy_setopt(op->data.curl.handle, CURLOPT_SSLCERT, client->tlscert); - curl_easy_setopt(op->curl.handle, CURLOPT_SSLKEY, + curl_easy_setopt(op->data.curl.handle, CURLOPT_SSLKEY, client->tlskey); } if (client->tlsverify == 0) { - curl_easy_setopt(op->curl.handle, CURLOPT_SSL_VERIFYHOST, 0); - curl_easy_setopt(op->curl.handle, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(op->data.curl.handle, + CURLOPT_SSL_VERIFYHOST, 0); + curl_easy_setopt(op->data.curl.handle, + CURLOPT_SSL_VERIFYPEER, 0); + } + + if (client->curlopt != NULL) { + idx = 0; + while (PyDict_Next(client->curlopt, &idx, &key, &value)) { + if (!PyLong_CheckExact(key)) { + Py_DECREF((PyObject *)op); + PyErr_Format(PyExc_RuntimeError, + "invalid key in curlopt keyword"); + return (NULL); + } + + opt = PyLong_AsLong(key); + + item = python_kore_curl_setopt(&op->data, opt, value); + if (item == NULL) { + Py_DECREF((PyObject *)op); + return (NULL); + } + + Py_DECREF(item); + } } if (client->cabundle != NULL) { - curl_easy_setopt(op->curl.handle, CURLOPT_CAINFO, + curl_easy_setopt(op->data.curl.handle, CURLOPT_CAINFO, client->cabundle); } @@ -6237,7 +6292,7 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) return (NULL); } - kore_curl_http_set_header(&op->curl, k, v); + kore_curl_http_set_header(&op->data.curl, k, v); } } @@ -6250,8 +6305,16 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) static void pyhttp_client_op_dealloc(struct pyhttp_client_op *op) { + struct pycurl_slist *psl; + + while ((psl = LIST_FIRST(&op->data.slists))) { + LIST_REMOVE(psl, list); + curl_slist_free_all(psl->slist); + kore_free(psl); + } + Py_DECREF(op->client); - kore_curl_cleanup(&op->curl); + kore_curl_cleanup(&op->data.curl); PyObject_Del((PyObject *)op); } @@ -6271,26 +6334,26 @@ pyhttp_client_op_iternext(struct pyhttp_client_op *op) PyObject *result, *tuple, *dict, *value; if (op->state == CURL_CLIENT_OP_RUN) { - kore_curl_run(&op->curl); + kore_curl_run(&op->data.curl); op->state = CURL_CLIENT_OP_RESULT; Py_RETURN_NONE; } - if (!kore_curl_success(&op->curl)) { + if (!kore_curl_success(&op->data.curl)) { PyErr_Format(PyExc_RuntimeError, "request to '%s' failed: %s", - op->curl.url, kore_curl_strerror(&op->curl)); + op->data.curl.url, kore_curl_strerror(&op->data.curl)); return (NULL); } - kore_curl_response_as_bytes(&op->curl, &response, &len); + kore_curl_response_as_bytes(&op->data.curl, &response, &len); if (op->headers) { - kore_curl_http_parse_headers(&op->curl); + kore_curl_http_parse_headers(&op->data.curl); if ((dict = PyDict_New()) == NULL) return (NULL); - TAILQ_FOREACH(hdr, &op->curl.http.resp_hdrs, list) { + TAILQ_FOREACH(hdr, &op->data.curl.http.resp_hdrs, list) { value = PyUnicode_FromString(hdr->value); if (value == NULL) { Py_DECREF(dict); @@ -6307,13 +6370,13 @@ pyhttp_client_op_iternext(struct pyhttp_client_op *op) Py_DECREF(value); } - if ((tuple = Py_BuildValue("(iOy#)", op->curl.http.status, + if ((tuple = Py_BuildValue("(iOy#)", op->data.curl.http.status, dict, (const char *)response, len)) == NULL) return (NULL); Py_DECREF(dict); } else { - if ((tuple = Py_BuildValue("(iy#)", op->curl.http.status, + if ((tuple = Py_BuildValue("(iy#)", op->data.curl.http.status, (const char *)response, len)) == NULL) return (NULL); } From 355cf87b93207feb7a09c36c27f389531eeeecd3 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 27 Aug 2021 10:06:45 +0200 Subject: [PATCH 039/121] use correct format specifier. --- src/python.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python.c b/src/python.c index 256828c..73f2fd8 100644 --- a/src/python.c +++ b/src/python.c @@ -5744,7 +5744,7 @@ python_kore_curl_setopt(struct pycurl_data *data, long opt, PyObject *value) } if (py_curlopt[i].name == NULL) { - PyErr_Format(PyExc_RuntimeError, "invalid option '%d'", opt); + PyErr_Format(PyExc_RuntimeError, "invalid option '%ld'", opt); return (NULL); } From 3eff4b9790a96f5ce56c4799562f123e6b82c790 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 27 Aug 2021 10:12:11 +0200 Subject: [PATCH 040/121] whitespace fixes --- src/python.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python.c b/src/python.c index 73f2fd8..b8ec42f 100644 --- a/src/python.c +++ b/src/python.c @@ -6200,7 +6200,7 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) default: fatal("%s: unknown method %d", __func__, m); } - + op = PyObject_New(struct pyhttp_client_op, &pyhttp_client_op_type); if (op == NULL) return (NULL); From 3c4acd9ac301001fd55baea2a662d700e0a136ac Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 27 Aug 2021 10:42:40 +0200 Subject: [PATCH 041/121] Allow curlopt keyword at httpclient request level. Now you can set curlopt on kore.httpclient at both the global httpclient object level and individual requests. Eg: client = kore.httpclient("https://kore.io", curlopt={ kore.CURLOPT_VERBOSE: 1 } ) status, body = await client.get( curlopt={ kore.CURLOPT_VERBOSE: 0 } ) --- src/python.c | 93 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/src/python.c b/src/python.c index b8ec42f..8f5d270 100644 --- a/src/python.c +++ b/src/python.c @@ -132,8 +132,10 @@ static void python_curl_http_callback(struct kore_curl *, void *); static void python_curl_handle_callback(struct kore_curl *, void *); static PyObject *pyhttp_client_request(struct pyhttp_client *, int, PyObject *); -static PyObject *python_kore_curl_setopt(struct pycurl_data *, +static PyObject *python_curlopt_set(struct pycurl_data *, long, PyObject *); +static int python_curlopt_from_dict(struct pycurl_data *, + PyObject *); #endif static void python_append_path(const char *); @@ -5734,7 +5736,7 @@ pykore_pgsql_result(struct pykore_pgsql *pysql) #if defined(KORE_USE_CURL) static PyObject * -python_kore_curl_setopt(struct pycurl_data *data, long opt, PyObject *value) +python_curlopt_set(struct pycurl_data *data, long opt, PyObject *value) { int i; @@ -5757,6 +5759,39 @@ python_kore_curl_setopt(struct pycurl_data *data, long opt, PyObject *value) return (py_curlopt[i].cb(data, i, value)); } +static int +python_curlopt_from_dict(struct pycurl_data *data, PyObject *dict) +{ + long opt; + Py_ssize_t idx; + PyObject *key, *value, *obj; + + idx = 0; + + if (!PyDict_CheckExact(dict)) { + PyErr_SetString(PyExc_RuntimeError, + "curlopt must be a dictionary"); + return (KORE_RESULT_ERROR); + } + + while (PyDict_Next(dict, &idx, &key, &value)) { + if (!PyLong_CheckExact(key)) { + PyErr_Format(PyExc_RuntimeError, + "invalid key in curlopt keyword"); + return (KORE_RESULT_ERROR); + } + + opt = PyLong_AsLong(key); + + if ((obj = python_curlopt_set(data, opt, value)) == NULL) + return (KORE_RESULT_ERROR); + + Py_DECREF(obj); + } + + return (KORE_RESULT_OK); +} + static PyObject * python_kore_curl_handle(PyObject *self, PyObject *args) { @@ -5858,7 +5893,7 @@ pycurl_handle_setopt(struct pycurl_handle *handle, PyObject *args) if (!PyArg_ParseTuple(args, "iO", &opt, &value)) return (NULL); - return (python_kore_curl_setopt(&handle->data, opt, value)); + return (python_curlopt_set(&handle->data, opt, value)); } static PyObject * @@ -6023,7 +6058,6 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op) static PyObject * python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs) { - PyObject *vdict; struct pyhttp_client *client; const char *url, *v; @@ -6056,17 +6090,8 @@ python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs) if ((v = python_string_from_dict(kwargs, "unix")) != NULL) client->unix = kore_strdup(v); - if ((vdict = PyDict_GetItemString(kwargs, "curlopt")) != NULL) { - if (!PyDict_CheckExact(vdict)) { - Py_DECREF((PyObject *)client); - PyErr_SetString(PyExc_RuntimeError, - "curlopt must be a dict"); - return (NULL); - } - - client->curlopt = vdict; - Py_INCREF(client->curlopt); - } + client->curlopt = PyDict_GetItemString(kwargs, "curlopt"); + Py_XINCREF(client->curlopt); python_bool_from_dict(kwargs, "tlsverify", &client->tlsverify); } @@ -6148,12 +6173,11 @@ pyhttp_client_options(struct pyhttp_client *client, PyObject *args, static PyObject * pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) { - long opt; struct pyhttp_client_op *op; char *ptr; const char *k, *v; Py_ssize_t length, idx; - PyObject *data, *headers, *key, *item, *value; + PyObject *data, *headers, *key, *obj; ptr = NULL; length = 0; @@ -6253,24 +6277,9 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) } if (client->curlopt != NULL) { - idx = 0; - while (PyDict_Next(client->curlopt, &idx, &key, &value)) { - if (!PyLong_CheckExact(key)) { - Py_DECREF((PyObject *)op); - PyErr_Format(PyExc_RuntimeError, - "invalid key in curlopt keyword"); - return (NULL); - } - - opt = PyLong_AsLong(key); - - item = python_kore_curl_setopt(&op->data, opt, value); - if (item == NULL) { - Py_DECREF((PyObject *)op); - return (NULL); - } - - Py_DECREF(item); + if (!python_curlopt_from_dict(&op->data, client->curlopt)) { + Py_DECREF((PyObject *)op); + return (NULL); } } @@ -6281,13 +6290,13 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) if (headers != NULL) { idx = 0; - while (PyDict_Next(headers, &idx, &key, &item)) { + while (PyDict_Next(headers, &idx, &key, &obj)) { if ((k = PyUnicode_AsUTF8(key)) == NULL) { Py_DECREF((PyObject *)op); return (NULL); } - if ((v = PyUnicode_AsUTF8(item)) == NULL) { + if ((v = PyUnicode_AsUTF8(obj)) == NULL) { Py_DECREF((PyObject *)op); return (NULL); } @@ -6296,8 +6305,16 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) } } - if (kwargs != NULL) + if (kwargs != NULL) { + if ((obj = PyDict_GetItemString(kwargs, "curlopt")) != NULL) { + if (!python_curlopt_from_dict(&op->data, obj)) { + Py_DECREF((PyObject *)op); + return (NULL); + } + } + python_bool_from_dict(kwargs, "return_headers", &op->headers); + } return ((PyObject *)op); } From 5f11f796a8fe0dc57ec803c994aa7b2caf0b7b89 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 5 Sep 2021 17:53:09 +0200 Subject: [PATCH 042/121] Allow configuration to pickup values from environment. Eg: certfile $CERTFILE will pickup the value from the set $CERTFILE environment variable. This works for _any_ Kore configuration option. --- src/config.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/config.c b/src/config.c index c7c0a6c..f287e04 100644 --- a/src/config.c +++ b/src/config.c @@ -56,6 +56,7 @@ /* XXX - This is becoming a clusterfuck. Fix it. */ static int configure_load(char *); +static char *configure_resolve_var(char *); static void configure_check_var(char **, const char *, const char *); #if defined(KORE_SINGLE_BINARY) @@ -392,7 +393,7 @@ void kore_parse_config_file(FILE *fp) { int i, lineno; - char buf[BUFSIZ], *p, *t; + char buf[BUFSIZ], *p, *t, *v; lineno = 1; while ((p = kore_read_line(fp, buf, sizeof(buf))) != NULL) { @@ -475,7 +476,9 @@ kore_parse_config_file(FILE *fp) for (i = 0; config_directives[i].name != NULL; i++) { if (!strcmp(config_directives[i].name, p)) { - if (config_directives[i].configure(t)) + if ((v = configure_resolve_var(t)) == NULL) + fatal("variable %s does not exist", t); + if (config_directives[i].configure(v)) break; fatal("configuration error on line %d", lineno); /* NOTREACHED */ @@ -489,7 +492,9 @@ kore_parse_config_file(FILE *fp) for (i = 0; config_settings[i].name != NULL; i++) { if (!strcmp(config_settings[i].name, p)) { - if (config_settings[i].configure(t)) + if ((v = configure_resolve_var(t)) == NULL) + fatal("variable %s does not exist", t); + if (config_settings[i].configure(v)) break; fatal("configuration error on line %d", lineno); /* NOTREACHED */ @@ -535,6 +540,21 @@ configure_check_var(char **var, const char *other, const char *logmsg) } } +static char * +configure_resolve_var(char *var) +{ + char *v; + + if (var[0] == '$') { + if ((v = getenv(&var[1])) == NULL) + return (NULL); + } else { + v = var; + } + + return (v); +} + static int configure_include(char *path) { From ebe090833d7d4e52667f836420fa30344f6ecbbd Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 5 Sep 2021 18:04:54 +0200 Subject: [PATCH 043/121] add Matthew to BEERS --- BEERS | 1 + 1 file changed, 1 insertion(+) diff --git a/BEERS b/BEERS index f348a72..fc75868 100644 --- a/BEERS +++ b/BEERS @@ -25,6 +25,7 @@ I will note down the beer of your choice between the brackets. [] Manuel Kniep [] Marcin Szczepaniak [] Matt Thompson +[] Matthew Norström [] Nandor Kracser [] Pascal Borreli [] Quentin Perez From 8661aee2f4738c62ed202642adeb5db35a2d4c73 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Sep 2021 13:26:54 +0200 Subject: [PATCH 044/121] only log key info if !kore_quiet --- src/keymgr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/keymgr.c b/src/keymgr.c index 234dfc8..739d3c3 100644 --- a/src/keymgr.c +++ b/src/keymgr.c @@ -577,7 +577,8 @@ keymgr_load_domain_privatekey(struct kore_domain *dom) key->dom = dom; - kore_log(LOG_INFO, "loaded private key for '%s'", dom->domain); + if (!kore_quiet) + kore_log(LOG_INFO, "loaded private key for '%s'", dom->domain); } static struct key * From 7f56c7dbf2688e665bfbef32ed30bce46564dd9e Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Sep 2021 13:28:38 +0200 Subject: [PATCH 045/121] Change how worker processes do logging. Before each worker process would either directly print to stdout if Kore was running in foreground mode, or syslog otherwise. With this commit the workers will submit their log messages to the parent process who will either put it onto stdout or syslog. This change in completely under the hood and users shouldn't care about it. --- Makefile | 6 +-- include/kore/kore.h | 1 + src/kore.c | 3 +- src/log.c | 113 ++++++++++++++++++++++++++++++++++++++++++++ src/utils.c | 40 ---------------- src/worker.c | 3 ++ 6 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 src/log.c diff --git a/Makefile b/Makefile index d5ebd8c..6e67915 100644 --- a/Makefile +++ b/Makefile @@ -21,9 +21,9 @@ VERSION=$(OBJDIR)/version.c PYTHON_CURLOPT=misc/curl/python_curlopt.h S_SRC= src/kore.c src/buf.c src/config.c src/connection.c \ - src/domain.c src/filemap.c src/fileref.c src/json.c src/mem.c \ - src/msg.c src/module.c src/net.c src/pool.c src/runtime.c src/timer.c \ - src/utils.c src/worker.c src/keymgr.c + src/domain.c src/filemap.c src/fileref.c src/json.c src/log.c \ + src/mem.c src/msg.c src/module.c src/net.c src/pool.c src/runtime.c \ + src/timer.c src/utils.c src/worker.c src/keymgr.c FEATURES= FEATURES_INC= diff --git a/include/kore/kore.h b/include/kore/kore.h index 5554d10..b1eb62f 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -659,6 +659,7 @@ struct kore_timer { #define KORE_MSG_CRL 9 #define KORE_MSG_ACCEPT_AVAILABLE 10 #define KORE_PYTHON_SEND_OBJ 11 +#define KORE_MSG_WORKER_LOG 12 #define KORE_MSG_ACME_BASE 100 /* messages for applications should start at 201. */ diff --git a/src/kore.c b/src/kore.c index a500fbc..49d2f07 100644 --- a/src/kore.c +++ b/src/kore.c @@ -205,8 +205,8 @@ main(int argc, char *argv[]) LIST_INIT(&kore_servers); kore_platform_init(); - kore_log_init(); kore_msg_init(); + kore_log_init(); #if !defined(KORE_NO_HTTP) http_parent_init(); #if defined(KORE_USE_CURL) @@ -866,6 +866,7 @@ kore_server_start(int argc, char *argv[]) struct kore_runtime_call *rcall; #endif + printf("kore_foreground = %d\n", kore_foreground); if (kore_foreground == 0) { if (daemon(1, 0) == -1) fatal("cannot daemon(): %s", errno_s); diff --git a/src/log.c b/src/log.c new file mode 100644 index 0000000..017e104 --- /dev/null +++ b/src/log.c @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2021 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include + +#include + +#include "kore.h" + +struct kore_wlog { + int prio; + u_int16_t wid; + size_t loglen; + char logmsg[]; +}; + +static void log_from_worker(struct kore_msg *, const void *); + +void +kore_log_init(void) +{ +#if defined(KORE_SINGLE_BINARY) + extern const char *__progname; + const char *name = kore_strdup(__progname); +#else + const char *name = "kore"; +#endif + + if (!kore_foreground) + openlog(name, LOG_NDELAY | LOG_PID, LOG_DAEMON); + + kore_msg_register(KORE_MSG_WORKER_LOG, log_from_worker); +} + +void +kore_log(int prio, const char *fmt, ...) +{ + va_list args; + const char *str; + struct kore_wlog wlog; + struct kore_buf buf, pkt; + + kore_buf_init(&buf, 128); + + va_start(args, fmt); + kore_buf_appendv(&buf, fmt, args); + va_end(args); + + if (worker != NULL) { + kore_buf_init(&pkt, sizeof(wlog) + buf.offset); + + memset(&wlog, 0, sizeof(wlog)); + + wlog.prio = prio; + wlog.wid = worker->id; + wlog.loglen = buf.offset; + + kore_buf_append(&pkt, &wlog, sizeof(wlog)); + kore_buf_append(&pkt, buf.data, buf.offset); + + kore_msg_send(KORE_MSG_PARENT, KORE_MSG_WORKER_LOG, + pkt.data, pkt.offset); + + kore_buf_cleanup(&pkt); + } else { + str = kore_buf_stringify(&buf, NULL); + + if (kore_foreground) + printf("[parent]: %s\n", str); + else + syslog(prio, "[parent]: %s", str); + } + + kore_buf_cleanup(&buf); +} + +static void +log_from_worker(struct kore_msg *msg, const void *data) +{ + const char *name; + const struct kore_wlog *wlog; + + if (msg->length < sizeof(*wlog)) { + kore_log(LOG_NOTICE, + "too short worker log received (%zu < %zu)", + msg->length, sizeof(*wlog)); + return; + } + + wlog = data; + name = kore_worker_name(wlog->wid); + + if (kore_foreground) { + printf("%s: %.*s\n", + name, (int)wlog->loglen, wlog->logmsg); + } else { + syslog(wlog->prio, "%s: %.*s", + name, (int)wlog->loglen, wlog->logmsg); + } +} diff --git a/src/utils.c b/src/utils.c index 480eba4..edcfac1 100644 --- a/src/utils.c +++ b/src/utils.c @@ -79,46 +79,6 @@ kore_debug_internal(char *file, int line, const char *fmt, ...) } #endif -void -kore_log_init(void) -{ -#if defined(KORE_SINGLE_BINARY) - extern const char *__progname; - const char *name = kore_strdup(__progname); -#else - const char *name = "kore"; -#endif - - if (!kore_foreground) - openlog(name, LOG_NDELAY | LOG_PID, LOG_DAEMON); -} - -void -kore_log(int prio, const char *fmt, ...) -{ - va_list args; - const char *name; - char buf[2048]; - - va_start(args, fmt); - (void)vsnprintf(buf, sizeof(buf), fmt, args); - va_end(args); - - if (worker != NULL) { - name = kore_worker_name(worker->id); - - if (kore_foreground) - printf("%s: %s\n", name, buf); - else - syslog(prio, "%s: %s", name, buf); - } else { - if (kore_foreground) - printf("[parent]: %s\n", buf); - else - syslog(prio, "[parent]: %s", buf); - } -} - size_t kore_strlcpy(char *dst, const char *src, const size_t len) { diff --git a/src/worker.c b/src/worker.c index d0ccfd1..e53c6c0 100644 --- a/src/worker.c +++ b/src/worker.c @@ -357,6 +357,9 @@ kore_worker_entry(struct kore_worker *kw) worker = kw; + if (!kore_foreground) + closelog(); + #if defined(__linux__) kore_seccomp_traceme(); #endif From 06991d22d5ec1b67e2f9f526366fc473d05abe13 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Sep 2021 13:40:33 +0200 Subject: [PATCH 046/121] remove norwegian debug --- src/kore.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/kore.c b/src/kore.c index 49d2f07..e3f9760 100644 --- a/src/kore.c +++ b/src/kore.c @@ -866,7 +866,6 @@ kore_server_start(int argc, char *argv[]) struct kore_runtime_call *rcall; #endif - printf("kore_foreground = %d\n", kore_foreground); if (kore_foreground == 0) { if (daemon(1, 0) == -1) fatal("cannot daemon(): %s", errno_s); From 00ef837d6247ecde831f549bcd6ef5d421b091b8 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Sep 2021 14:15:53 +0200 Subject: [PATCH 047/121] call explicit fflush() on stdout --- src/log.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/log.c b/src/log.c index 017e104..aa97301 100644 --- a/src/log.c +++ b/src/log.c @@ -106,6 +106,7 @@ log_from_worker(struct kore_msg *msg, const void *data) if (kore_foreground) { printf("%s: %.*s\n", name, (int)wlog->loglen, wlog->logmsg); + fflush(stdout); } else { syslog(wlog->prio, "%s: %.*s", name, (int)wlog->loglen, wlog->logmsg); From 0ac54eb48d9be1be815df2e66cc163be776b15eb Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Sep 2021 14:35:04 +0200 Subject: [PATCH 048/121] Add a kore.config.skipchroot to the Python API. If set to True, will skip the chroot() of the Kore workers. This can be handy in case you want to set your deployment target to production or docker so you get user changes but you don't want to chroot the processes. --- src/config.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/config.c b/src/config.c index f287e04..ac3facd 100644 --- a/src/config.c +++ b/src/config.c @@ -155,6 +155,7 @@ static int configure_task_threads(char *); #if defined(KORE_USE_PYTHON) static int configure_deployment(char *); +static int configure_skip_chroot(char *); static int configure_python_path(char *); static int configure_python_import(char *); #endif @@ -266,6 +267,7 @@ static struct { #endif #if defined(KORE_USE_PYTHON) { "deployment", configure_deployment }, + { "skipchroot", configure_skip_chroot }, #endif #if defined(KORE_USE_PGSQL) { "pgsql_conn_max", configure_pgsql_conn_max }, @@ -1883,6 +1885,22 @@ configure_task_threads(char *option) #endif #if defined(KORE_USE_PYTHON) +static int +configure_skip_chroot(char *value) +{ + if (!strcmp(value, "True")) { + skip_chroot = 1; + } else if (!strcmp(value, "False")) { + skip_chroot = 0; + } else { + kore_log(LOG_NOTICE, + "kore.config.skipchroot: bad value '%s'", value); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + static int configure_deployment(char *value) { From 599835e7fd7e07fe45bdec2ea11a13b251450528 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Sep 2021 15:39:38 +0200 Subject: [PATCH 049/121] Python: Only use parameters if needed. We always called kore_pgsql_query_param_fields() regardless if the params keyword was specified or not, instead only use it if actual parameters have been given. Otherwise use the kore_pgsql_query() function directly to execute the query. --- src/python.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/python.c b/src/python.c index 8f5d270..6540693 100644 --- a/src/python.c +++ b/src/python.c @@ -5595,13 +5595,21 @@ pykore_pgsql_iternext(struct pykore_pgsql *pysql) } /* fallthrough */ case PYKORE_PGSQL_QUERY: - if (!kore_pgsql_query_param_fields(&pysql->sql, - pysql->query, pysql->binary, - pysql->param.count, pysql->param.values, - pysql->param.lengths, pysql->param.formats)) { - PyErr_Format(PyExc_RuntimeError, - "pgsql error: %s", pysql->sql.error); - return (NULL); + if (pysql->param.count > 0) { + if (!kore_pgsql_query_param_fields(&pysql->sql, + pysql->query, pysql->binary, + pysql->param.count, pysql->param.values, + pysql->param.lengths, pysql->param.formats)) { + PyErr_Format(PyExc_RuntimeError, + "pgsql error: %s", pysql->sql.error); + return (NULL); + } + } else { + if (!kore_pgsql_query(&pysql->sql, pysql->query)) { + PyErr_Format(PyExc_RuntimeError, + "pgsql error: %s", pysql->sql.error); + return (NULL); + } } pysql->state = PYKORE_PGSQL_WAIT; break; From c8c9a24d99033458650c04c91bcbd2e6a78f4fc6 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 7 Sep 2021 21:15:17 +0200 Subject: [PATCH 050/121] Only set initial python deployment if needed. We should only be setting this if an actual module was specified on the command-line that will be loaded. --- src/python.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python.c b/src/python.c index 6540693..e8e4c9b 100644 --- a/src/python.c +++ b/src/python.c @@ -366,8 +366,10 @@ kore_python_init(void) KORE_FILTER_LEN(filter_python)); #endif - if (!kore_configure_setting("deployment", "dev")) - fatal("failed to set initial deployment"); + if (kore_pymodule) { + if (!kore_configure_setting("deployment", "dev")) + fatal("failed to set initial deployment"); + } } void From 1c33ce01d0326a7c4b4c461ccb30928487c8592f Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 7 Sep 2021 21:58:53 +0200 Subject: [PATCH 051/121] Add kore_build_date to version.c --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 6e67915..c036bc9 100644 --- a/Makefile +++ b/Makefile @@ -176,6 +176,8 @@ $(VERSION): $(OBJDIR) force echo "No version information found (no .git or RELEASE)"; \ exit 1; \ fi + @printf "const char *kore_build_date = \"%s\";\n" \ + `date +"%Y-%m-%d"` >> $(VERSION); $(KODEV): src/cli.c $(MAKE) -C kodev @@ -256,6 +258,8 @@ tools-install: $(OBJDIR)/%.o: src/%.c $(CC) $(CFLAGS) -c $< -o $@ +src/kore.c: $(VERSION) + src/python.c: $(PYTHON_CURLOPT) src/seccomp.c: $(PLATFORM) From 3b20cda11c104eacedb572a739a6b4ffd58b370e Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 7 Sep 2021 21:59:22 +0200 Subject: [PATCH 052/121] Rework worker startup/privsep config. Starting with the privsep config, this commit changes the following: - Removes the root, runas, keymgr_root, keymgr_runas, acme_root and acme_runas configuration options. Instead these are now configured via a privsep configuration context: privsep worker { root /tmp runas nobody } This is also configurable via Python using the new kore.privsep() method: kore.privsep("worker", root="/tmp", runas="nobody", skip=["chroot"]) Tied into this we also better handle worker startup: - Per worker process, wait until it signalled it is ready. - If a worker fails at startup, display its last log lines more clearly. - Don't start acme process if no domain requires acme. - Remove each process its individual startup log message in favour of a generalized one that displays its PID, root and user. - At startup, log the kore version and built-ins in a nicer way. - The worker processes now check things they need to start running before signaling they are ready (such as access to CA certs for TLS client authentication). --- conf/kore.conf.example | 76 ++++++---- include/kore/acme.h | 1 + include/kore/kore.h | 27 ++-- include/kore/python_methods.h | 3 + src/acme.c | 26 ++-- src/config.c | 258 +++++++++++++++++----------------- src/filemap.c | 4 +- src/keymgr.c | 28 ++-- src/kore.c | 77 ++++++---- src/log.c | 28 +++- src/python.c | 67 +++++++++ src/utils.c | 6 +- src/worker.c | 150 +++++++++++++++----- 13 files changed, 485 insertions(+), 266 deletions(-) diff --git a/conf/kore.conf.example b/conf/kore.conf.example index 6f01053..f133b1d 100644 --- a/conf/kore.conf.example +++ b/conf/kore.conf.example @@ -20,16 +20,57 @@ server tls { # tls no #} -# The worker process root directory. If chrooting was not disabled -# at startup the worker processes will chroot into this directory. +# Kore can have multiple settings for each processes that run under it. +# There are 3 different type of processes: # -# If this configuration option is not set, Kore will take the current -# working directory as the root. -root /home/joris/src/kore +# 1) Worker processes, these handle the HTTP requests and your code +# runs inside of these. +# 2) The keymgr process, this handles your domain private keys +# and signing during the TLS handshakes. It also holds your +# ACME account-key and will sign ACME requests. +# 3) The acme process, this talks to the ACME servers. +# +# You can individually turn on/off chrooting and dropping user +# privileges per process. The -n and -r command-line options +# are a global override for skipping chroot or dropping user +# permissions on all processes. +# +# If no root/runas options are set in a process, it will inherit the +# default values from the worker processes. +# +# The worker processes will get the current working directory or +# current user if no options where specified for it. +# +# Configures the worker processes. +privsep worker { + # The user the workers will run as. + runas kore -# Worker processes will run as the specified user. If this option is -# missing Kore will run as the current user. -runas joris + # The root directory for the worker processes, if chroot isn't + # skipped, this is the directory it will chroot into. + # + # If not set, Kore will take the current working directory. + root /var/chroot/kore + + # We could configure this process to not chroot and only + # chdir into its root directory. + #skip chroot +} + +# Configures the keymgr process. +# If TLS is enabled you will need to specify paths to the domain +# certificate and key that Kore will load. This loading is done +# from the keymgr (separate process) and all paths must be relative +# to the keymgr process its root configuration option. +privsep keymgr { + # The user the keymgr will run as. + runas keymgr + + # The root directory for the keymgr process. In this example + # we do not turn off chroot for this process so the keymgr + # will chroot into this directory. + root /etc/keymgr +} # How many worker processes Kore will spawn. If the directive # worker_set_affinity is set to 1 (the default) Kore will automatically @@ -88,25 +129,6 @@ workers 4 # NOTE: This file location must be inside your chrooted environment. #rand_file random.data -# Key manager specific options. -# If TLS is enabled you will need to specify paths to the domain -# certificate and key that Kore will load. This loading is done -# from the keymgr (separate process) and all paths must be relative -# to the keymgr_root configuration option. -# -# keymgr_root The root path the keymgr will chdir into. -# If chroot was not disable at startup time -# the keymgr process will chroot into here. -# -# keymgr_runas The user to run the keymgr as. -# -# If privsep and chrooting is enabled at startup time but these -# configuration options were not set, they will take over the -# values from the 'root' and 'runas' configuration options. -# -#keymgr_root -#keymgr_runas - # Filemap settings # filemap_index Name of the file to be used as the directory # index for a filemap. diff --git a/include/kore/acme.h b/include/kore/acme.h index b6dbd76..04391df 100644 --- a/include/kore/acme.h +++ b/include/kore/acme.h @@ -54,6 +54,7 @@ int kore_acme_tls_alpn(SSL *, const unsigned char **, unsigned char *, const unsigned char *, unsigned int, void *); extern char *acme_email; +extern int acme_domains; extern char *acme_provider; #if defined(__cplusplus) diff --git a/include/kore/kore.h b/include/kore/kore.h index b1eb62f..c6b0174 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -450,9 +450,17 @@ struct kore_alog_header { u_int16_t loglen; } __attribute__((packed)); +struct kore_privsep { + char *root; + char *runas; + int skip_runas; + int skip_chroot; +}; + struct kore_worker { u_int16_t id; u_int16_t cpu; + int ready; int running; #if defined(__linux__) int tracing; @@ -464,6 +472,7 @@ struct kore_worker { int restarted; u_int64_t time_locked; struct kore_module_handle *active_hdlr; + struct kore_privsep *ps; /* Used by the workers to store accesslogs. */ struct { @@ -701,8 +710,6 @@ extern int skip_runas; extern int kore_foreground; extern char *kore_pidfile; -extern char *kore_root_path; -extern char *kore_runas_user; extern char *kore_tls_cipher_list; extern volatile sig_atomic_t sig_recv; @@ -711,15 +718,16 @@ extern int tls_version; extern DH *tls_dhparam; extern char *rand_file; extern int keymgr_active; -extern char *keymgr_runas_user; -extern char *keymgr_root_path; -extern char *acme_runas_user; -extern char *acme_root_path; + +extern struct kore_privsep worker_privsep; +extern struct kore_privsep keymgr_privsep; +extern struct kore_privsep acme_privsep; extern u_int8_t nlisteners; extern u_int16_t cpu_count; extern u_int8_t worker_count; extern const char *kore_version; +extern const char *kore_build_date; extern int worker_policy; extern u_int8_t worker_set_affinity; extern u_int32_t worker_rlimit_nofiles; @@ -742,12 +750,13 @@ void kore_proctitle(const char *); void kore_default_getopt(int, char **); void kore_worker_reap(void); -void kore_worker_init(void); +int kore_worker_init(void); +void kore_worker_privsep(void); +void kore_worker_started(void); void kore_worker_make_busy(void); void kore_worker_shutdown(void); void kore_worker_dispatch_signal(int); -void kore_worker_privdrop(const char *, const char *); -void kore_worker_spawn(u_int16_t, u_int16_t, u_int16_t); +int kore_worker_spawn(u_int16_t, u_int16_t, u_int16_t); int kore_worker_keymgr_response_verify(struct kore_msg *, const void *, struct kore_domain **); diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index da6b85d..8ca9c62 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -62,6 +62,8 @@ static PyObject *python_kore_sendobj(PyObject *, PyObject *, PyObject *); static PyObject *python_kore_server(PyObject *, PyObject *, PyObject *); +static PyObject *python_kore_privsep(PyObject *, PyObject *, + PyObject *); #if defined(KORE_USE_PGSQL) @@ -108,6 +110,7 @@ static struct PyMethodDef pykore_methods[] = { METHOD("domain", python_kore_domain, METH_VARARGS | METH_KEYWORDS), METHOD("server", python_kore_server, METH_VARARGS | METH_KEYWORDS), METHOD("gather", python_kore_gather, METH_VARARGS | METH_KEYWORDS), + METHOD("privsep", python_kore_privsep, METH_VARARGS | METH_KEYWORDS), METHOD("sendobj", python_kore_sendobj, METH_VARARGS | METH_KEYWORDS), METHOD("websocket_broadcast", python_websocket_broadcast, METH_VARARGS), #if defined(KORE_USE_PGSQL) diff --git a/src/acme.c b/src/acme.c index 3cca0c2..038e172 100644 --- a/src/acme.c +++ b/src/acme.c @@ -265,10 +265,10 @@ static char *account_url = NULL; static u_int8_t acme_alpn_name[] = { 0xa, 'a', 'c', 'm', 'e', '-', 't', 'l', 's', '/', '1' }; +struct kore_privsep acme_privsep; +int acme_domains = 0; char *acme_email = NULL; char *acme_provider = NULL; -char *acme_root_path = NULL; -char *acme_runas_user = NULL; u_int32_t acme_request_timeout = 8; void @@ -310,7 +310,7 @@ kore_acme_run(void) #if defined(KORE_USE_PYTHON) kore_msg_unregister(KORE_PYTHON_SEND_OBJ); #endif - kore_worker_privdrop(acme_runas_user, acme_root_path); + kore_worker_privsep(); #if defined(__OpenBSD__) if (unveil("/etc/ssl/", "r") == -1) @@ -321,14 +321,10 @@ kore_acme_run(void) http_init(); - if (!kore_quiet) { - kore_log(LOG_NOTICE, - "acme worker started (pid#%d)", worker->pid); - } - LIST_INIT(&orders); LIST_INIT(&signops); + kore_worker_started(); acme_parse_directory(); while (quit != 1) { @@ -434,7 +430,7 @@ kore_acme_tls_challenge_use_cert(SSL *ssl, struct kore_domain *dom) return; } - kore_log(LOG_NOTICE, "[%s] acme-tls/1 challenge requested", + kore_log(LOG_INFO, "[%s] acme-tls/1 challenge requested", dom->domain); if ((c = SSL_get_ex_data(ssl, 0)) == NULL) @@ -989,7 +985,7 @@ acme_order_remove(struct acme_order *order, const char *reason) kore_free(auth); } - kore_log(LOG_NOTICE, "[%s] order removed (%s)", order->domain, reason); + kore_log(LOG_INFO, "[%s] order removed (%s)", order->domain, reason); if (strcmp(reason, "completed")) acme_order_retry(order->domain); @@ -1366,7 +1362,7 @@ cleanup: } else { order->auths--; if (order->auths == 0) { - kore_log(LOG_NOTICE, + kore_log(LOG_INFO, "[%s:auth] authentications done", order->domain); order->state = ACME_ORDER_STATE_RUNNING; } @@ -1389,12 +1385,12 @@ acme_challenge_tls_alpn_01(struct acme_order *order, acme_challenge_tls_alpn_01_create(order, challenge); break; case ACME_STATUS_PROCESSING: - kore_log(LOG_NOTICE, + kore_log(LOG_INFO, "[%s:auth:challenge:tls-alpn-01] processing", order->domain); break; case ACME_STATUS_VALID: - kore_log(LOG_NOTICE, + kore_log(LOG_INFO, "[%s:auth:challenge:tls-alpn-01] valid", order->domain); ret = KORE_RESULT_OK; @@ -1419,7 +1415,7 @@ acme_challenge_tls_alpn_01_create(struct acme_order *order, u_int8_t digest[SHA256_DIGEST_LENGTH]; if (challenge->flags & ACME_FLAG_CHALLENGE_CREATED) { - kore_log(LOG_NOTICE, + kore_log(LOG_INFO, "[%s:auth:challenge:tls-alpn-01] pending keymgr", order->domain); return; @@ -1427,7 +1423,7 @@ acme_challenge_tls_alpn_01_create(struct acme_order *order, challenge->flags |= ACME_FLAG_CHALLENGE_CREATED; - kore_log(LOG_NOTICE, + kore_log(LOG_INFO, "[%s:auth:challenge:tls-alpn-01] requested from keymgr", order->domain); diff --git a/src/config.c b/src/config.c index ac3facd..219d154 100644 --- a/src/config.c +++ b/src/config.c @@ -69,8 +69,6 @@ static int configure_file(char *); #if defined(KORE_USE_ACME) static int configure_acme(char *); -static int configure_acme_root(char *); -static int configure_acme_runas(char *); static int configure_acme_email(char *); static int configure_acme_provider(char *); #endif @@ -82,8 +80,7 @@ static int configure_bind(char *); static int configure_bind_unix(char *); static int configure_attach(char *); static int configure_domain(char *); -static int configure_root(char *); -static int configure_runas(char *); +static int configure_privsep(char *); static int configure_workers(char *); static int configure_pidfile(char *); static int configure_rlimit_nofiles(char *); @@ -92,6 +89,9 @@ static int configure_accept_threshold(char *); static int configure_death_policy(char *); static int configure_set_affinity(char *); static int configure_socket_backlog(char *); +static int configure_privsep_skip(char *); +static int configure_privsep_root(char *); +static int configure_privsep_runas(char *); #if defined(KORE_USE_PLATFORM_PLEDGE) static int configure_add_pledge(char *); @@ -103,8 +103,6 @@ static int configure_certkey(char *); static int configure_tls_version(char *); static int configure_tls_cipher(char *); static int configure_tls_dhparam(char *); -static int configure_keymgr_root(char *); -static int configure_keymgr_runas(char *); static int configure_client_verify(char *); static int configure_client_verify_depth(char *); @@ -155,9 +153,6 @@ static int configure_task_threads(char *); #if defined(KORE_USE_PYTHON) static int configure_deployment(char *); -static int configure_skip_chroot(char *); -static int configure_python_path(char *); -static int configure_python_import(char *); #endif #if defined(KORE_USE_CURL) @@ -180,18 +175,18 @@ static struct { { "bind", configure_bind }, { "load", configure_load }, { "domain", configure_domain }, + { "privsep", configure_privsep }, { "server", configure_server }, { "attach", configure_attach }, { "certkey", configure_certkey }, { "certfile", configure_certfile }, { "include", configure_include }, { "unix", configure_bind_unix }, + { "skip", configure_privsep_skip }, + { "root", configure_privsep_root }, + { "runas", configure_privsep_runas }, { "client_verify", configure_client_verify }, { "client_verify_depth", configure_client_verify_depth }, -#if defined(KORE_USE_PYTHON) - { "python_path", configure_python_path }, - { "python_import", configure_python_import }, -#endif #if !defined(KORE_NO_HTTP) { "route", configure_route}, { "filemap", configure_filemap }, @@ -217,9 +212,6 @@ static struct { const char *name; int (*configure)(char *); } config_settings[] = { - { "root", configure_root }, - { "chroot", configure_root }, - { "runas", configure_runas }, { "workers", configure_workers }, { "worker_max_connections", configure_max_connections }, { "worker_rlimit_nofiles", configure_rlimit_nofiles }, @@ -232,11 +224,7 @@ static struct { { "tls_cipher", configure_tls_cipher }, { "tls_dhparam", configure_tls_dhparam }, { "rand_file", configure_rand_file }, - { "keymgr_runas", configure_keymgr_runas }, - { "keymgr_root", configure_keymgr_root }, #if defined(KORE_USE_ACME) - { "acme_runas", configure_acme_runas }, - { "acme_root", configure_acme_root }, { "acme_email", configure_acme_email }, { "acme_provider", configure_acme_provider }, #endif @@ -267,7 +255,6 @@ static struct { #endif #if defined(KORE_USE_PYTHON) { "deployment", configure_deployment }, - { "skipchroot", configure_skip_chroot }, #endif #if defined(KORE_USE_PGSQL) { "pgsql_conn_max", configure_pgsql_conn_max }, @@ -302,12 +289,14 @@ static struct kore_module_handle *current_handler = NULL; extern const char *__progname; static struct kore_domain *current_domain = NULL; static struct kore_server *current_server = NULL; +static struct kore_privsep *current_privsep = NULL; void kore_parse_config(void) { FILE *fp; BIO *bio; + struct passwd *pwd; char path[PATH_MAX]; if (finalized) @@ -345,10 +334,10 @@ kore_parse_config(void) if (!kore_module_loaded()) fatal("no application module was loaded"); - if (kore_root_path == NULL) { + if (worker_privsep.root == NULL) { if (getcwd(path, sizeof(path)) == NULL) fatal("getcwd: %s", errno_s); - kore_root_path = kore_strdup(path); + worker_privsep.root = kore_strdup(path); if (!kore_quiet) { kore_log(LOG_NOTICE, "privsep: no root path set, " @@ -356,37 +345,54 @@ kore_parse_config(void) } } - if (getuid() != 0 && skip_chroot == 0) - fatal("cannot chroot, use -n to skip it"); + if (worker_privsep.runas == NULL) { + if ((pwd = getpwuid(getuid())) == NULL) + fatal("getpwuid: %s", errno_s); - if (skip_runas != 1 && kore_runas_user == NULL) - fatal("missing runas user, use -r to skip it"); + worker_privsep.runas = kore_strdup(pwd->pw_name); + if (!kore_quiet) { + kore_log(LOG_NOTICE, "privsep: no runas user set, " + "using current user %s", worker_privsep.runas); + } - if (getuid() != 0 && skip_runas == 0) - fatal("cannot drop privileges, use -r to skip it"); + endpwent(); + } - if (skip_runas) { - if (!kore_quiet) - kore_log(LOG_WARNING, "privsep: will not change user"); - } else { - configure_check_var(&keymgr_runas_user, kore_runas_user, - "privsep: no keymgr_runas set, using 'runas` user"); + configure_check_var(&keymgr_privsep.runas, worker_privsep.runas, + "privsep: no keymgr runas set, using 'privsep.worker.runas`"); #if defined(KORE_USE_ACME) - configure_check_var(&acme_runas_user, kore_runas_user, - "privsep: no acme_runas set, using 'runas` user"); + configure_check_var(&acme_privsep.runas, worker_privsep.runas, + "privsep: no acme runas set, using 'privsep.worker.runas`"); +#endif + + configure_check_var(&keymgr_privsep.root, worker_privsep.root, + "privsep: no keymgr root set, using 'privsep.worker.root`"); +#if defined(KORE_USE_ACME) + configure_check_var(&acme_privsep.root, worker_privsep.root, + "privsep: no acme root set, using 'privsep.worker.root`"); +#endif + + if (skip_chroot) { + worker_privsep.skip_chroot = 1; + keymgr_privsep.skip_chroot = 1; +#if defined(KORE_USE_ACME) + acme_privsep.skip_chroot = 1; #endif } - configure_check_var(&keymgr_root_path, kore_root_path, - "privsep: no keymgr_root set, using 'root` directory"); - + if (skip_runas) { + worker_privsep.skip_runas = 1; + keymgr_privsep.skip_runas = 1; #if defined(KORE_USE_ACME) - configure_check_var(&acme_root_path, kore_root_path, - "privsep: no acme_root set, using 'root` directory"); + acme_privsep.skip_runas = 1; #endif + } + + if (skip_runas && !kore_quiet) + kore_log(LOG_NOTICE, "privsep: skipping all runas options"); if (skip_chroot && !kore_quiet) - kore_log(LOG_WARNING, "privsep: will not chroot"); + kore_log(LOG_NOTICE, "privsep: skipping all chroot options"); finalized = 1; } @@ -404,6 +410,12 @@ kore_parse_config_file(FILE *fp) continue; } + if (!strcmp(p, "}") && current_privsep != NULL) { + lineno++; + current_privsep = NULL; + continue; + } + if (!strcmp(p, "}") && current_server != NULL) { lineno++; kore_server_finalize(current_server); @@ -649,7 +661,7 @@ configure_acme(char *yesno) kore_acme_get_paths(current_domain->domain, ¤t_domain->certkey, ¤t_domain->certfile); - + acme_domains++; } else { printf("invalid '%s' for yes|no acme option\n", yesno); return (KORE_RESULT_ERROR); @@ -658,24 +670,6 @@ configure_acme(char *yesno) return (KORE_RESULT_OK); } -static int -configure_acme_runas(char *user) -{ - kore_free(acme_runas_user); - acme_runas_user = kore_strdup(user); - - return (KORE_RESULT_OK); -} - -static int -configure_acme_root(char *root) -{ - kore_free(acme_root_path); - acme_root_path = kore_strdup(root); - - return (KORE_RESULT_OK); -} - static int configure_acme_email(char *email) { @@ -937,21 +931,89 @@ configure_certkey(char *path) } static int -configure_keymgr_runas(char *user) +configure_privsep(char *options) { - if (keymgr_runas_user != NULL) - kore_free(keymgr_runas_user); - keymgr_runas_user = kore_strdup(user); + char *argv[3]; + + if (current_privsep != NULL) { + printf("nested privsep contexts are not allowed\n"); + return (KORE_RESULT_ERROR); + } + + kore_split_string(options, " ", argv, 3); + + if (argv[0] == NULL || argv[1] == NULL) { + printf("invalid privsep context\n"); + return (KORE_RESULT_ERROR); + } + + if (strcmp(argv[1], "{")) { + printf("privsep context not opened correctly\n"); + return (KORE_RESULT_ERROR); + } + + if (!strcmp(argv[0], "worker")) { + current_privsep = &worker_privsep; + } else if (!strcmp(argv[0], "keymgr")) { + current_privsep = &keymgr_privsep; +#if defined(KORE_USE_ACME) + } else if (!strcmp(argv[0], "keymgr")) { + current_privsep = &acme_privsep; +#endif + } else { + printf("unknown privsep context: %s\n", argv[0]); + return (KORE_RESULT_ERROR); + } return (KORE_RESULT_OK); } static int -configure_keymgr_root(char *root) +configure_privsep_runas(char *user) { - if (keymgr_root_path != NULL) - kore_free(keymgr_root_path); - keymgr_root_path = kore_strdup(root); + if (current_privsep == NULL) { + printf("runas not specified in privsep context\n"); + return (KORE_RESULT_ERROR); + } + + if (current_privsep->runas != NULL) + kore_free(current_privsep->runas); + + current_privsep->runas = kore_strdup(user); + + return (KORE_RESULT_OK); +} + +static int +configure_privsep_root(char *root) +{ + if (current_privsep == NULL) { + printf("root not specified in privsep context\n"); + return (KORE_RESULT_ERROR); + } + + if (current_privsep->root != NULL) + kore_free(current_privsep->root); + + current_privsep->root = kore_strdup(root); + + return (KORE_RESULT_OK); +} + +static int +configure_privsep_skip(char *option) +{ + if (current_privsep == NULL) { + printf("skip not specified in privsep context\n"); + return (KORE_RESULT_ERROR); + } + + if (!strcmp(option, "chroot")) { + current_privsep->skip_chroot = 1; + } else { + printf("unknown skip option '%s'\n", option); + return (KORE_RESULT_ERROR); + } return (KORE_RESULT_OK); } @@ -1709,26 +1771,6 @@ configure_websocket_timeout(char *option) #endif /* !KORE_NO_HTTP */ -static int -configure_root(char *path) -{ - if (kore_root_path != NULL) - kore_free(kore_root_path); - kore_root_path = kore_strdup(path); - - return (KORE_RESULT_OK); -} - -static int -configure_runas(char *user) -{ - if (kore_runas_user != NULL) - kore_free(kore_runas_user); - kore_runas_user = kore_strdup(user); - - return (KORE_RESULT_OK); -} - static int configure_workers(char *option) { @@ -1885,22 +1927,6 @@ configure_task_threads(char *option) #endif #if defined(KORE_USE_PYTHON) -static int -configure_skip_chroot(char *value) -{ - if (!strcmp(value, "True")) { - skip_chroot = 1; - } else if (!strcmp(value, "False")) { - skip_chroot = 0; - } else { - kore_log(LOG_NOTICE, - "kore.config.skipchroot: bad value '%s'", value); - return (KORE_RESULT_ERROR); - } - - return (KORE_RESULT_OK); -} - static int configure_deployment(char *value) { @@ -1925,26 +1951,6 @@ configure_deployment(char *value) return (KORE_RESULT_OK); } -static int -configure_python_path(char *path) -{ - kore_python_path(path); - - return (KORE_RESULT_OK); -} - -static int -configure_python_import(char *module) -{ - char *argv[3]; - - kore_split_string(module, " ", argv, 3); - if (argv[0] == NULL) - return (KORE_RESULT_ERROR); - - kore_module_load(argv[0], argv[1], KORE_MODULE_PYTHON); - return (KORE_RESULT_OK); -} #endif #if defined(KORE_USE_PLATFORM_PLEDGE) diff --git a/src/filemap.c b/src/filemap.c index 8d4dc66..89918d8 100644 --- a/src/filemap.c +++ b/src/filemap.c @@ -69,9 +69,9 @@ kore_filemap_create(struct kore_domain *dom, const char *path, const char *root) if (root[0] != '/' || root[sz - 1] != '/') return (KORE_RESULT_ERROR); - if (kore_root_path != NULL) { + if (worker_privsep.root != NULL) { len = snprintf(fpath, sizeof(fpath), "%s/%s", - kore_root_path, path); + worker_privsep.root, path); if (len == -1 || (size_t)len >= sizeof(fpath)) fatal("kore_filemap_create: failed to concat paths"); } else { diff --git a/src/keymgr.c b/src/keymgr.c index 739d3c3..535ef81 100644 --- a/src/keymgr.c +++ b/src/keymgr.c @@ -248,9 +248,8 @@ static void keymgr_rsa_encrypt(struct kore_msg *, const void *, static void keymgr_ecdsa_sign(struct kore_msg *, const void *, struct key *); -int keymgr_active = 0; -char *keymgr_root_path = NULL; -char *keymgr_runas_user = NULL; +struct kore_privsep keymgr_privsep; +int keymgr_active = 0; #if defined(__OpenBSD__) #if defined(KORE_USE_ACME) @@ -293,10 +292,7 @@ kore_keymgr_run(void) #if defined(KORE_USE_PYTHON) kore_msg_unregister(KORE_PYTHON_SEND_OBJ); #endif - kore_worker_privdrop(keymgr_runas_user, keymgr_root_path); - - if (!kore_quiet) - kore_log(LOG_NOTICE, "key manager started (pid#%d)", getpid()); + kore_worker_privsep(); if (rand_file != NULL) { keymgr_load_randfile(); @@ -321,6 +317,8 @@ kore_keymgr_run(void) X509V3_EXT_add_alias(acme_oid, NID_subject_key_identifier); #endif + kore_worker_started(); + while (quit != 1) { now = kore_time_ms(); if ((now - last_seed) > RAND_POLL_INTERVAL) { @@ -367,7 +365,7 @@ kore_keymgr_cleanup(int final) struct key *key, *next; if (final && !kore_quiet) - kore_log(LOG_NOTICE, "cleaning up keys"); + kore_log(LOG_INFO, "cleaning up keys"); if (initialized == 0) return; @@ -788,7 +786,7 @@ keymgr_acme_init(void) ACME_RENEWAL_TIMER, NULL, 0); if (key->pkey == NULL) { - kore_log(LOG_NOTICE, "generating new ACME account key"); + kore_log(LOG_INFO, "generating new ACME account key"); key->pkey = kore_rsakey_generate(KORE_ACME_ACCOUNT_KEY); needsreg = 1; } else { @@ -828,7 +826,7 @@ keymgr_acme_domainkey(struct kore_domain *dom, struct key *key) { char *p; - kore_log(LOG_NOTICE, "generated new domain key for %s", dom->domain); + kore_log(LOG_INFO, "generated new domain key for %s", dom->domain); if ((p = strrchr(dom->certkey, '/')) == NULL) fatalx("invalid certkey path '%s'", dom->certkey); @@ -1029,7 +1027,7 @@ keymgr_acme_install_cert(const void *data, size_t len, struct key *key) fd = open(key->dom->certfile, O_CREAT | O_TRUNC | O_WRONLY, 0700); if (fd == -1) - fatal("open(%s): %s", key->dom->certfile, errno_s); + fatalx("open(%s): %s", key->dom->certfile, errno_s); kore_log(LOG_INFO, "writing %zu bytes of data", len); @@ -1038,14 +1036,14 @@ keymgr_acme_install_cert(const void *data, size_t len, struct key *key) if (ret == -1) { if (errno == EINTR) continue; - fatal("write(%s): %s", key->dom->certfile, errno_s); + fatalx("write(%s): %s", key->dom->certfile, errno_s); } break; } if ((size_t)ret != len) { - fatal("incorrect write on %s (%zd/%zu)", + fatalx("incorrect write on %s (%zd/%zu)", key->dom->certfile, ret, len); } @@ -1102,7 +1100,7 @@ keymgr_acme_challenge_cert(const void *data, size_t len, struct key *key) slen = snprintf(hex + (idx * 2), sizeof(hex) - (idx * 2), "%02x", digest[idx]); if (slen == -1 || (size_t)slen >= sizeof(hex)) - fatal("failed to convert digest to hex"); + fatalx("failed to convert digest to hex"); } if ((x509 = X509_new()) == NULL) @@ -1176,7 +1174,7 @@ keymgr_acme_csr(const struct kore_keyreq *req, struct key *key) kore_log(LOG_INFO, "[%s] creating CSR", req->domain); if ((csr = X509_REQ_new()) == NULL) - fatal("X509_REQ_new: %s", ssl_errno_s); + fatalx("X509_REQ_new: %s", ssl_errno_s); if (!X509_REQ_set_version(csr, 3)) fatalx("X509_REQ_set_version(): %s", ssl_errno_s); diff --git a/src/kore.c b/src/kore.c index e3f9760..c4669da 100644 --- a/src/kore.c +++ b/src/kore.c @@ -64,12 +64,12 @@ u_int8_t worker_count = 0; char **kore_argv = NULL; int kore_foreground = 0; char *kore_progname = NULL; -char *kore_root_path = NULL; -char *kore_runas_user = NULL; u_int32_t kore_socket_backlog = 5000; char *kore_pidfile = KORE_PIDFILE_DEFAULT; char *kore_tls_cipher_list = KORE_DEFAULT_CIPHER_LIST; +struct kore_privsep worker_privsep; + extern char **environ; extern char *__progname; static size_t proctitle_maxlen = 0; @@ -113,9 +113,9 @@ usage(void) #endif printf("\t-f\tstart in foreground\n"); printf("\t-h\tthis help text\n"); - printf("\t-n\tdo not chroot\n"); + printf("\t-n\tdo not chroot on any worker\n"); printf("\t-q\tonly log errors\n"); - printf("\t-r\tdo not drop privileges\n"); + printf("\t-r\tdo not change user on any worker\n"); printf("\t-v\tdisplay %s build information\n", __progname); printf("\nFind more information on https://kore.io\n"); @@ -226,6 +226,34 @@ main(int argc, char *argv[]) kore_module_init(); kore_server_sslstart(); + if (!kore_quiet) { + kore_log(LOG_INFO, "%s %s starting, built=%s", + __progname, kore_version, kore_build_date); + kore_log(LOG_INFO, "built-ins: " +#if defined(__linux__) + "seccomp " +#endif +#if defined(KORE_USE_PGSQL) + "pgsql " +#endif +#if defined(KORE_USE_TASKS) + "tasks " +#endif +#if defined(KORE_USE_JSONRPC) + "jsonrpc " +#endif +#if defined(KORE_USE_PYTHON) + "python " +#endif +#if defined(KORE_USE_ACME) + "acme " +#endif +#if defined(KORE_USE_CURL) + "curl " +#endif + ); + } + #if !defined(KORE_SINGLE_BINARY) && !defined(KORE_USE_PYTHON) if (config_file == NULL) usage(); @@ -276,7 +304,7 @@ main(int argc, char *argv[]) kore_server_start(argc, argv); if (!kore_quiet) - kore_log(LOG_NOTICE, "server shutting down"); + kore_log(LOG_INFO, "server shutting down"); kore_worker_shutdown(); @@ -292,7 +320,7 @@ main(int argc, char *argv[]) kore_server_cleanup(); if (!kore_quiet) - kore_log(LOG_NOTICE, "goodbye"); + kore_log(LOG_INFO, "goodbye"); #if defined(KORE_USE_PYTHON) kore_python_cleanup(); @@ -554,10 +582,10 @@ kore_server_finalize(struct kore_server *srv) proto = "http"; if (l->family == AF_UNIX) { - kore_log(LOG_NOTICE, "%s serving %s on %s", + kore_log(LOG_INFO, "%s serving %s on %s", srv->name, proto, l->host); } else { - kore_log(LOG_NOTICE, "%s serving %s on %s:%s", + kore_log(LOG_INFO, "%s serving %s on %s:%s", srv->name, proto, l->host, l->port); } } @@ -881,25 +909,6 @@ kore_server_start(int argc, char *argv[]) kore_pid = getpid(); kore_write_kore_pid(); - if (!kore_quiet) { - kore_log(LOG_NOTICE, "%s is starting up", __progname); -#if defined(__linux__) - kore_log(LOG_NOTICE, "seccomp sandbox enabled"); -#endif -#if defined(KORE_USE_PGSQL) - kore_log(LOG_NOTICE, "pgsql built-in enabled"); -#endif -#if defined(KORE_USE_TASKS) - kore_log(LOG_NOTICE, "tasks built-in enabled"); -#endif -#if defined(KORE_USE_JSONRPC) - kore_log(LOG_NOTICE, "jsonrpc built-in enabled"); -#endif -#if defined(KORE_USE_PYTHON) - kore_log(LOG_NOTICE, "python built-in enabled"); -#endif - } - #if !defined(KORE_SINGLE_BINARY) && !defined(KORE_USE_PYTHON) kore_call_parent_configure(argc, argv); #endif @@ -922,7 +931,19 @@ kore_server_start(int argc, char *argv[]) } kore_platform_proctitle("[parent]"); - kore_worker_init(); + + if (!kore_worker_init()) { + kore_log(LOG_ERR, "last worker log lines:"); + kore_log(LOG_ERR, "====================================="); + net_init(); + kore_connection_init(); + kore_platform_event_init(); + kore_msg_parent_init(); + kore_platform_event_wait(10); + kore_worker_dispatch_signal(SIGQUIT); + kore_log(LOG_ERR, "====================================="); + return; + } /* Set worker_max_connections for kore_connection_init(). */ tmp = worker_max_connections; diff --git a/src/log.c b/src/log.c index aa97301..42af50b 100644 --- a/src/log.c +++ b/src/log.c @@ -27,6 +27,7 @@ struct kore_wlog { char logmsg[]; }; +static void log_print(int, const char *, ...); static void log_from_worker(struct kore_msg *, const void *); void @@ -79,7 +80,7 @@ kore_log(int prio, const char *fmt, ...) str = kore_buf_stringify(&buf, NULL); if (kore_foreground) - printf("[parent]: %s\n", str); + log_print(prio, "[parent]: %s\n", str); else syslog(prio, "[parent]: %s", str); } @@ -104,11 +105,32 @@ log_from_worker(struct kore_msg *msg, const void *data) name = kore_worker_name(wlog->wid); if (kore_foreground) { - printf("%s: %.*s\n", + log_print(wlog->prio, "%s: %.*s\n", name, (int)wlog->loglen, wlog->logmsg); - fflush(stdout); } else { syslog(wlog->prio, "%s: %.*s", name, (int)wlog->loglen, wlog->logmsg); } } + +static void +log_print(int prio, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + + switch (prio) { + case LOG_ERR: + case LOG_WARNING: + case LOG_NOTICE: + case LOG_INFO: + case LOG_DEBUG: + break; + } + + vprintf(fmt, args); + fflush(stdout); + + va_end(args); +} diff --git a/src/python.c b/src/python.c index e8e4c9b..77c732b 100644 --- a/src/python.c +++ b/src/python.c @@ -1847,6 +1847,73 @@ python_kore_server(PyObject *self, PyObject *args, PyObject *kwargs) Py_RETURN_NONE; } +static PyObject * +python_kore_privsep(PyObject *self, PyObject *args, PyObject *kwargs) +{ + struct kore_privsep *ps; + const char *val; + PyObject *skip, *obj; + Py_ssize_t list_len, idx; + + if (!PyArg_ParseTuple(args, "s", &val)) + return (NULL); + + if (!strcmp(val, "worker")) { + ps = &worker_privsep; + } else if (!strcmp(val, "keymgr")) { + ps = &keymgr_privsep; +#if defined(KORE_USE_ACME) + } else if (!strcmp(val, "keymgr")) { + ps = &acme_privsep; +#endif + } else { + PyErr_Format(PyExc_RuntimeError, + "unknown privsep process '%s'", val); + return (NULL); + } + + if ((val = python_string_from_dict(kwargs, "root")) != NULL) { + kore_free(ps->root); + ps->root = kore_strdup(val); + } + + if ((val = python_string_from_dict(kwargs, "runas")) != NULL) { + kore_free(ps->runas); + ps->runas = kore_strdup(val); + } + + if ((skip = PyDict_GetItemString(kwargs, "skip")) != NULL) { + if (!PyList_CheckExact(skip)) { + PyErr_Format(PyExc_RuntimeError, + "privsep skip keyword needs to be a list"); + return (NULL); + } + + list_len = PyList_Size(skip); + + for (idx = 0; idx < list_len; idx++) { + if ((obj = PyList_GetItem(skip, idx)) == NULL) + return (NULL); + + if (!PyUnicode_Check(obj)) + return (NULL); + + if ((val = PyUnicode_AsUTF8AndSize(obj, NULL)) == NULL) + return (NULL); + + if (!strcmp(val, "chroot")) { + ps->skip_chroot = 1; + } else { + PyErr_Format(PyExc_RuntimeError, + "unknown skip keyword '%s'", val); + return (NULL); + } + } + } + + Py_RETURN_NONE; +} + static PyObject * python_kore_prerequest(PyObject *self, PyObject *args) { diff --git a/src/utils.c b/src/utils.c index edcfac1..0756a3c 100644 --- a/src/utils.c +++ b/src/utils.c @@ -652,14 +652,10 @@ fatal_log(const char *fmt, va_list args) extern const char *kore_progname; (void)vsnprintf(buf, sizeof(buf), fmt, args); - - if (!kore_foreground) - kore_log(LOG_ERR, "%s", buf); + kore_log(LOG_ERR, "%s", buf); if (worker != NULL && worker->id == KORE_WORKER_KEYMGR) kore_keymgr_cleanup(1); - - printf("%s: %s\n", kore_progname, buf); } static int diff --git a/src/worker.c b/src/worker.c index e53c6c0..df86675 100644 --- a/src/worker.c +++ b/src/worker.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include "kore.h" @@ -77,6 +79,7 @@ struct wlock { static int worker_trylock(void); static void worker_unlock(void); static void worker_reaper(pid_t, int); +static void worker_domain_check(struct kore_domain *); static inline int worker_acceptlock_obtain(void); static inline void worker_acceptlock_release(void); @@ -99,7 +102,7 @@ u_int32_t worker_max_connections = 512; u_int32_t worker_active_connections = 0; int worker_policy = KORE_WORKER_POLICY_RESTART; -void +int kore_worker_init(void) { size_t len; @@ -149,27 +152,33 @@ kore_worker_init(void) for (idx = KORE_WORKER_BASE; idx < worker_count; idx++) { if (cpu >= cpu_count) cpu = 0; - kore_worker_spawn(idx, id++, cpu++); + if (!kore_worker_spawn(idx, id++, cpu++)) + return (KORE_RESULT_ERROR); } if (keymgr_active) { #if defined(KORE_USE_ACME) /* The ACME process is only started if we need it. */ - if (acme_provider) { - kore_worker_spawn(KORE_WORKER_ACME_IDX, - KORE_WORKER_ACME, 0); + if (acme_domains) { + if (!kore_worker_spawn(KORE_WORKER_ACME_IDX, + KORE_WORKER_ACME, 0)) + return (KORE_RESULT_ERROR); } #endif /* Now we can start the keymgr. */ - kore_worker_spawn(KORE_WORKER_KEYMGR_IDX, - KORE_WORKER_KEYMGR, 0); + if (!kore_worker_spawn(KORE_WORKER_KEYMGR_IDX, + KORE_WORKER_KEYMGR, 0)) + return (KORE_RESULT_ERROR); } + + return (KORE_RESULT_OK); } -void +int kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu) { + int cnt; struct kore_worker *kw; kw = WORKER(idx); @@ -178,6 +187,7 @@ kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu) kw->has_lock = 0; kw->active_hdlr = NULL; kw->running = 1; + kw->ready = 0; if (socketpair(AF_UNIX, SOCK_STREAM, 0, kw->pipe) == -1) fatal("socketpair(): %s", errno_s); @@ -186,6 +196,20 @@ kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu) !kore_connection_nonblock(kw->pipe[1], 0)) fatal("could not set pipe fds to nonblocking: %s", errno_s); + switch (id) { + case KORE_WORKER_KEYMGR: + kw->ps = &keymgr_privsep; + break; +#if defined(KORE_USE_ACME) + case KORE_WORKER_ACME: + kw->ps = &acme_privsep; + break; +#endif + default: + kw->ps = &worker_privsep; + break; + } + kw->pid = fork(); if (kw->pid == -1) fatal("could not spawn worker child: %s", errno_s); @@ -193,8 +217,24 @@ kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu) if (kw->pid == 0) { kw->pid = getpid(); kore_worker_entry(kw); - /* NOTREACHED */ + exit(1); + } else { + for (cnt = 0; cnt < 25; cnt++) { + if (kw->ready == 1) + break; + usleep(10000); + } + + if (kw->ready == 0) { + kore_log(LOG_NOTICE, + "worker %d failed to start, shutting down", + kw->id); + + return (KORE_RESULT_ERROR); + } } + + return (KORE_RESULT_OK); } struct kore_worker * @@ -282,37 +322,43 @@ kore_worker_dispatch_signal(int sig) } void -kore_worker_privdrop(const char *runas, const char *root) +kore_worker_privsep(void) { rlim_t fd; struct rlimit rl; - struct passwd *pw = NULL; + struct passwd *pw; - if (root == NULL) - fatalx("no root directory for kore_worker_privdrop"); + if (worker == NULL) + fatalx("%s called with no worker", __func__); + + pw = NULL; /* Must happen before chroot. */ - if (skip_runas == 0) { - if (runas == NULL) - fatalx("no runas user given and -r not specified"); - pw = getpwnam(runas); - if (pw == NULL) { + if (worker->ps->skip_runas == 0) { + if (worker->ps->runas == NULL) { + fatalx("no runas user given for %s", + kore_worker_name(worker->id)); + } + + if ((pw = getpwnam(worker->ps->runas)) == NULL) { fatalx("cannot getpwnam(\"%s\") for user: %s", - runas, errno_s); + worker->ps->runas, errno_s); } } - if (skip_chroot == 0) { - if (chroot(root) == -1) { + if (worker->ps->skip_chroot == 0) { + if (chroot(worker->ps->root) == -1) { fatalx("cannot chroot(\"%s\"): %s", - root, errno_s); + worker->ps->root, errno_s); } if (chdir("/") == -1) fatalx("cannot chdir(\"/\"): %s", errno_s); } else { - if (chdir(root) == -1) - fatalx("cannot chdir(\"%s\"): %s", root, errno_s); + if (chdir(worker->ps->root) == -1) { + fatalx("cannot chdir(\"%s\"): %s", + worker->ps->root, errno_s); + } } if (getrlimit(RLIMIT_NOFILE, &rl) == -1) { @@ -332,7 +378,7 @@ kore_worker_privdrop(const char *runas, const char *root) worker_rlimit_nofiles, errno_s); } - if (skip_runas == 0) { + if (worker->ps->skip_runas == 0) { if (setgroups(1, &pw->pw_gid) || #if defined(__MACH__) || defined(NetBSD) setgid(pw->pw_gid) || setegid(pw->pw_gid) || @@ -341,7 +387,7 @@ kore_worker_privdrop(const char *runas, const char *root) setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) || setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid)) #endif - fatalx("cannot drop privileges"); + fatalx("cannot drop privileges (%s)", errno_s); } kore_platform_sandbox(); @@ -370,7 +416,6 @@ kore_worker_entry(struct kore_worker *kw) kore_platform_worker_setcpu(kw); kore_pid = kw->pid; - kore_signal_setup(); if (kw->id == KORE_WORKER_KEYMGR) { @@ -394,7 +439,7 @@ kore_worker_entry(struct kore_worker *kw) kore_task_init(); #endif - kore_worker_privdrop(kore_runas_user, kore_root_path); + kore_worker_privsep(); #if !defined(KORE_NO_HTTP) http_init(); @@ -435,12 +480,6 @@ kore_worker_entry(struct kore_worker *kw) if (nlisteners == 0) worker_no_lock = 1; - if (!kore_quiet) { - kore_log(LOG_NOTICE, - "worker %d started (cpu#%d, pid#%d)", - kw->id, kw->cpu, kw->pid); - } - rcall = kore_runtime_getcall("kore_worker_configure"); if (rcall != NULL) { kore_runtime_execute(rcall); @@ -448,6 +487,9 @@ kore_worker_entry(struct kore_worker *kw) } kore_module_onload(); + kore_domain_callback(worker_domain_check); + + kore_worker_started(); worker->restarted = 0; for (;;) { @@ -672,6 +714,40 @@ kore_worker_keymgr_response_verify(struct kore_msg *msg, const void *data, return (KORE_RESULT_OK); } +void +kore_worker_started(void) +{ + const char *chroot; + + if (worker->ps->skip_chroot) + chroot = "root"; + else + chroot = "chroot"; + + if (!kore_quiet) { + kore_log(LOG_NOTICE, + "process started (#%d %s=%s%s%s)", + getpid(), chroot, worker->ps->root, + worker->ps->skip_runas ? "" : " user=", + worker->ps->skip_runas ? "" : worker->ps->runas); + } + + worker->ready = 1; +} + +static void +worker_domain_check(struct kore_domain *dom) +{ + struct stat st; + + if (dom->cafile != NULL) { + if (stat(dom->cafile, &st) == -1) + fatalx("'%s': %s", dom->cafile, errno_s); + if (access(dom->cafile, R_OK) == -1) + fatalx("'%s': not readable", dom->cafile, errno_s); + } +} + static void worker_reaper(pid_t pid, int status) { @@ -758,9 +834,11 @@ worker_reaper(pid_t pid, int status) kore_log(LOG_NOTICE, "restarting worker %d", kw->id); kw->restarted = 1; kore_msg_parent_remove(kw); - kore_worker_spawn(idx, kw->id, kw->cpu); - kore_msg_parent_add(kw); + if (!kore_worker_spawn(idx, kw->id, kw->cpu)) + (void)raise(SIGQUIT); + + kore_msg_parent_add(kw); break; } } From bbae4be6f18c6308e1fbc7aeb25cd9dfd96c6987 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 7 Sep 2021 22:06:18 +0200 Subject: [PATCH 053/121] remove unused kore_progname. --- src/utils.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils.c b/src/utils.c index 0756a3c..eddb754 100644 --- a/src/utils.c +++ b/src/utils.c @@ -649,7 +649,6 @@ static void fatal_log(const char *fmt, va_list args) { char buf[2048]; - extern const char *kore_progname; (void)vsnprintf(buf, sizeof(buf), fmt, args); kore_log(LOG_ERR, "%s", buf); From 9fd30db5985e10feabb7ae6c50909db54cc63655 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 7 Sep 2021 22:14:28 +0200 Subject: [PATCH 054/121] Change timeout for worker startup a bit. Also give some feedback we are waiting for process startup. --- src/worker.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/worker.c b/src/worker.c index df86675..ac03fb3 100644 --- a/src/worker.c +++ b/src/worker.c @@ -146,6 +146,8 @@ kore_worker_init(void) kw->lb.offset = 0; } + kore_log(LOG_INFO, "starting worker processes"); + /* Now start all the workers. */ id = 1; cpu = 1; @@ -172,6 +174,8 @@ kore_worker_init(void) return (KORE_RESULT_ERROR); } + kore_log(LOG_INFO, "all worker processes started"); + return (KORE_RESULT_OK); } @@ -219,10 +223,10 @@ kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu) kore_worker_entry(kw); exit(1); } else { - for (cnt = 0; cnt < 25; cnt++) { + for (cnt = 0; cnt < 50; cnt++) { if (kw->ready == 1) break; - usleep(10000); + usleep(100000); } if (kw->ready == 0) { From 116f935e1070c95537cc222a5c2a6499700864a4 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 7 Sep 2021 22:19:21 +0200 Subject: [PATCH 055/121] use the correct name for acme. --- src/python.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python.c b/src/python.c index 77c732b..1d9439e 100644 --- a/src/python.c +++ b/src/python.c @@ -1863,7 +1863,7 @@ python_kore_privsep(PyObject *self, PyObject *args, PyObject *kwargs) } else if (!strcmp(val, "keymgr")) { ps = &keymgr_privsep; #if defined(KORE_USE_ACME) - } else if (!strcmp(val, "keymgr")) { + } else if (!strcmp(val, "acme")) { ps = &acme_privsep; #endif } else { From 1af0772ebe593567a0eeab0191064f2bc29c5d5f Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 7 Sep 2021 22:27:02 +0200 Subject: [PATCH 056/121] Don't add acme to msg framework if not needed. --- src/msg.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/msg.c b/src/msg.c index 13c275a..977a368 100644 --- a/src/msg.c +++ b/src/msg.c @@ -22,6 +22,10 @@ #include "kore.h" #include "http.h" +#if defined(KORE_USE_ACME) +#include "acme.h" +#endif + struct msg_type { u_int8_t id; void (*cb)(struct kore_msg *, const void *); @@ -62,10 +66,14 @@ kore_msg_parent_init(void) continue; } -#if !defined(KORE_USE_ACME) - if (idx == KORE_WORKER_ACME_IDX) + if (idx == KORE_WORKER_ACME_IDX) { +#if defined(KORE_USE_ACME) + if (acme_domains == 0) + continue; +#else continue; #endif + } kw = kore_worker_data(idx); kore_msg_parent_add(kw); From a6677b873f2a521c1f03d54ce200b15ed7ca96ac Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 7 Sep 2021 23:05:25 +0200 Subject: [PATCH 057/121] On linux, keep track of seccomp tracing properly. With the new process startup code we must handle the SIGSTOP from the processes if seccomp_tracing is enabled. Otherwise they just hang indefinitely and we assume they failed to start, which is somewhat true. --- src/worker.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/worker.c b/src/worker.c index ac03fb3..6c70073 100644 --- a/src/worker.c +++ b/src/worker.c @@ -184,6 +184,9 @@ kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu) { int cnt; struct kore_worker *kw; +#if defined(__linux__) + int status; +#endif kw = WORKER(idx); kw->id = id; @@ -227,6 +230,16 @@ kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu) if (kw->ready == 1) break; usleep(100000); +#if defined(__linux__) + /* + * If seccomp_tracing is enabled, make sure we + * handle the SIGSTOP from the child processes. + */ + if (kore_seccomp_tracing) { + if (waitpid(kw->pid, &status, WNOHANG) > 0) + kore_seccomp_trace(kw->pid, status); + } +#endif } if (kw->ready == 0) { From ff6bae6513eb5f1c86ef6ccf938bdc03807b9905 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 7 Sep 2021 23:26:36 +0200 Subject: [PATCH 058/121] move startup log back into kore_server_start(). --- src/kore.c | 56 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/kore.c b/src/kore.c index c4669da..ecf315c 100644 --- a/src/kore.c +++ b/src/kore.c @@ -226,34 +226,6 @@ main(int argc, char *argv[]) kore_module_init(); kore_server_sslstart(); - if (!kore_quiet) { - kore_log(LOG_INFO, "%s %s starting, built=%s", - __progname, kore_version, kore_build_date); - kore_log(LOG_INFO, "built-ins: " -#if defined(__linux__) - "seccomp " -#endif -#if defined(KORE_USE_PGSQL) - "pgsql " -#endif -#if defined(KORE_USE_TASKS) - "tasks " -#endif -#if defined(KORE_USE_JSONRPC) - "jsonrpc " -#endif -#if defined(KORE_USE_PYTHON) - "python " -#endif -#if defined(KORE_USE_ACME) - "acme " -#endif -#if defined(KORE_USE_CURL) - "curl " -#endif - ); - } - #if !defined(KORE_SINGLE_BINARY) && !defined(KORE_USE_PYTHON) if (config_file == NULL) usage(); @@ -894,6 +866,34 @@ kore_server_start(int argc, char *argv[]) struct kore_runtime_call *rcall; #endif + if (!kore_quiet) { + kore_log(LOG_INFO, "%s %s starting, built=%s", + __progname, kore_version, kore_build_date); + kore_log(LOG_INFO, "built-ins: " +#if defined(__linux__) + "seccomp " +#endif +#if defined(KORE_USE_PGSQL) + "pgsql " +#endif +#if defined(KORE_USE_TASKS) + "tasks " +#endif +#if defined(KORE_USE_JSONRPC) + "jsonrpc " +#endif +#if defined(KORE_USE_PYTHON) + "python " +#endif +#if defined(KORE_USE_ACME) + "acme " +#endif +#if defined(KORE_USE_CURL) + "curl " +#endif + ); + } + if (kore_foreground == 0) { if (daemon(1, 0) == -1) fatal("cannot daemon(): %s", errno_s); From b77d727f729b4c2fd0b7cc1638cbed02c1d326c9 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 10 Sep 2021 13:34:57 +0200 Subject: [PATCH 059/121] Add a logfile configuration option. This will log all output from Kore processes to the specified file. --- include/kore/kore.h | 1 + src/config.c | 9 +++++++++ src/log.c | 17 +++++++++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/include/kore/kore.h b/include/kore/kore.h index c6b0174..d7e59c5 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -855,6 +855,7 @@ int kore_connection_accept(struct listener *, u_int64_t kore_time_ms(void); void kore_log_init(void); +void kore_log_file(const char *); #if defined(KORE_USE_PYTHON) int kore_configure_setting(const char *, char *); diff --git a/src/config.c b/src/config.c index 219d154..3b66cf6 100644 --- a/src/config.c +++ b/src/config.c @@ -81,6 +81,7 @@ static int configure_bind_unix(char *); static int configure_attach(char *); static int configure_domain(char *); static int configure_privsep(char *); +static int configure_logfile(char *); static int configure_workers(char *); static int configure_pidfile(char *); static int configure_rlimit_nofiles(char *); @@ -212,6 +213,7 @@ static struct { const char *name; int (*configure)(char *); } config_settings[] = { + { "logfile", configure_logfile }, { "workers", configure_workers }, { "worker_max_connections", configure_max_connections }, { "worker_rlimit_nofiles", configure_rlimit_nofiles }, @@ -1771,6 +1773,13 @@ configure_websocket_timeout(char *option) #endif /* !KORE_NO_HTTP */ +static int +configure_logfile(char *path) +{ + kore_log_file(path); + return (KORE_RESULT_OK); +} + static int configure_workers(char *option) { diff --git a/src/log.c b/src/log.c index 42af50b..ab2cce7 100644 --- a/src/log.c +++ b/src/log.c @@ -30,6 +30,8 @@ struct kore_wlog { static void log_print(int, const char *, ...); static void log_from_worker(struct kore_msg *, const void *); +static FILE *fp = NULL; + void kore_log_init(void) { @@ -40,12 +42,23 @@ kore_log_init(void) const char *name = "kore"; #endif + fp = stdout; + if (!kore_foreground) openlog(name, LOG_NDELAY | LOG_PID, LOG_DAEMON); kore_msg_register(KORE_MSG_WORKER_LOG, log_from_worker); } +void +kore_log_file(const char *path) +{ + if ((fp = fopen(path, "a")) == NULL) { + fp = stdout; + fatal("fopen(%s): %s", path, errno_s); + } +} + void kore_log(int prio, const char *fmt, ...) { @@ -129,8 +142,8 @@ log_print(int prio, const char *fmt, ...) break; } - vprintf(fmt, args); - fflush(stdout); + vfprintf(fp, fmt, args); + fflush(fp); va_end(args); } From eb0b8f21e37113736602db9bba210496b860e186 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 12 Sep 2021 14:13:24 +0200 Subject: [PATCH 060/121] Add http_response_close() to the C API. This is the same as http_response() except it will automatically close the connection after the response is sent. This is a bit easier than setting CONN_CLOSE_EMPTY yourself manually. --- include/kore/http.h | 2 ++ src/http.c | 33 ++++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/include/kore/http.h b/include/kore/http.h index 20df6ca..db8dce3 100644 --- a/include/kore/http.h +++ b/include/kore/http.h @@ -357,6 +357,8 @@ int http_body_digest(struct http_request *, char *, size_t); int http_redirect_add(struct kore_domain *, const char *, int, const char *); void http_response(struct http_request *, int, const void *, size_t); +void http_response_close(struct http_request *, int, + const void *, size_t); void http_response_fileref(struct http_request *, int, struct kore_fileref *); void http_serveable(struct http_request *, const void *, diff --git a/src/http.c b/src/http.c index fd46591..993e8fb 100644 --- a/src/http.c +++ b/src/http.c @@ -580,21 +580,44 @@ http_serveable(struct http_request *req, const void *data, size_t len, } void -http_response(struct http_request *req, int status, const void *d, size_t l) +http_response(struct http_request *req, int code, const void *d, size_t l) { if (req->owner == NULL) return; - kore_debug("http_response(%p, %d, %p, %zu)", req, status, d, l); + kore_debug("%s(%p, %d, %p, %zu)", __func__, req, code, d, l); + + req->status = code; - req->status = status; switch (req->owner->proto) { case CONN_PROTO_HTTP: case CONN_PROTO_WEBSOCKET: - http_response_normal(req, req->owner, status, d, l); + http_response_normal(req, req->owner, code, d, l); break; default: - fatal("http_response() bad proto %d", req->owner->proto); + fatal("%s: bad proto %d", __func__, req->owner->proto); + /* NOTREACHED. */ + } +} + +void +http_response_close(struct http_request *req, int code, const void *d, size_t l) +{ + if (req->owner == NULL) + return; + + kore_debug("%s(%p, %d, %p, %zu)", __func__, req, code, d, l); + + req->status = code; + req->owner->flags |= CONN_CLOSE_EMPTY; + + switch (req->owner->proto) { + case CONN_PROTO_HTTP: + case CONN_PROTO_WEBSOCKET: + http_response_normal(req, req->owner, code, d, l); + break; + default: + fatal("%s: bad proto %d", __func__, req->owner->proto); /* NOTREACHED. */ } } From cdd681d602980f8d6862a168eea5f3b40bd5a033 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 12 Sep 2021 14:30:33 +0200 Subject: [PATCH 061/121] Let http_response_header() handle duplicates. If a response header was previously by an application for an HTTP request http_response_header() will now overwrite the previous value. --- src/http.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/http.c b/src/http.c index 993e8fb..665aa3a 100644 --- a/src/http.c +++ b/src/http.c @@ -400,11 +400,24 @@ http_response_header(struct http_request *req, { struct http_header *hdr; + hdr = NULL; kore_debug("http_response_header(%p, %s, %s)", req, header, value); - hdr = kore_pool_get(&http_header_pool); + TAILQ_FOREACH(hdr, &req->resp_headers, list) { + if (!strcasecmp(hdr->header, header)) { + TAILQ_REMOVE(&req->resp_headers, hdr, list); + kore_free(hdr->header); + kore_free(hdr->value); + break; + } + } + + if (hdr == NULL) + hdr = kore_pool_get(&http_header_pool); + hdr->header = kore_strdup(header); hdr->value = kore_strdup(value); + TAILQ_INSERT_TAIL(&(req->resp_headers), hdr, list); } From 862bf1a5f671d06cba35c6d794d9d7c1bcdaf6be Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 12 Sep 2021 15:10:06 +0200 Subject: [PATCH 062/121] Add http_response_json(). Since its an HTTP response function it functions like http_response() but takes a kore_json_item pointer that it will automatically convert to a kore_buf and send/free using http_response_stream(). While here fix a problem with http_response_stream() which could end up not calling the cb() in case of HTTP_METHOD_HEAD. Since the behaviour is that it should call cb() when done it should do so immediately. --- src/http.c | 605 ++++++++++++++++++++++++++++------------------------- 1 file changed, 323 insertions(+), 282 deletions(-) diff --git a/src/http.c b/src/http.c index 665aa3a..adb7a43 100644 --- a/src/http.c +++ b/src/http.c @@ -128,6 +128,7 @@ static const char *pretty_error_fmt = "\n\n"; static int http_body_recv(struct netbuf *); +static int http_release_buffer(struct netbuf *); static void http_error_response(struct connection *, int); static void http_write_response_cookie(struct http_cookie *); static void http_argument_add(struct http_request *, char *, char *, @@ -635,6 +636,35 @@ http_response_close(struct http_request *req, int code, const void *d, size_t l) } } +void +http_response_json(struct http_request *req, int status, + struct kore_json_item *json) +{ + struct kore_buf *buf; + + if (req->owner == NULL) + return; + + kore_debug("%s(%p, %d)", __func__, req, code); + + buf = kore_buf_alloc(1024); + kore_json_item_tobuf(json, buf); + kore_json_item_free(json); + + req->status = status; + http_response_header(req, "content-type", "application/json"); + + switch (req->owner->proto) { + case CONN_PROTO_HTTP: + http_response_stream(req, status, buf->data, buf->offset, + http_release_buffer, buf); + break; + default: + fatal("%s: bad proto %d", __func__, req->owner->proto); + /* NOTREACHED. */ + } +} + void http_response_stream(struct http_request *req, int status, void *base, size_t len, int (*cb)(struct netbuf *), void *arg) @@ -651,13 +681,16 @@ http_response_stream(struct http_request *req, int status, void *base, http_response_normal(req, req->owner, status, NULL, len); break; default: - fatal("http_response_stream() bad proto %d", req->owner->proto); + fatal("%s: bad proto %d", __func__, req->owner->proto); /* NOTREACHED. */ } - if (req->method != HTTP_METHOD_HEAD) { - net_send_stream(req->owner, base, len, cb, &nb); - nb->extra = arg; + net_send_stream(req->owner, base, len, cb, &nb); + nb->extra = arg; + + if (req->method == HTTP_METHOD_HEAD) { + nb->s_off = nb->b_len; + net_remove_netbuf(req->owner, nb); } } @@ -1566,6 +1599,292 @@ http_redirect_add(struct kore_domain *dom, const char *path, int status, return (KORE_RESULT_OK); } +const char * +http_status_text(int status) +{ + const char *r; + + switch (status) { + case HTTP_STATUS_CONTINUE: + r = "Continue"; + break; + case HTTP_STATUS_SWITCHING_PROTOCOLS: + r = "Switching Protocols"; + break; + case HTTP_STATUS_OK: + r = "OK"; + break; + case HTTP_STATUS_CREATED: + r = "Created"; + break; + case HTTP_STATUS_ACCEPTED: + r = "Accepted"; + break; + case HTTP_STATUS_NON_AUTHORITATIVE: + r = "Non-Authoritative Information"; + break; + case HTTP_STATUS_NO_CONTENT: + r = "No Content"; + break; + case HTTP_STATUS_RESET_CONTENT: + r = "Reset Content"; + break; + case HTTP_STATUS_PARTIAL_CONTENT: + r = "Partial Content"; + break; + case HTTP_STATUS_MULTIPLE_CHOICES: + r = "Multiple Choices"; + break; + case HTTP_STATUS_MOVED_PERMANENTLY: + r = "Moved Permanently"; + break; + case HTTP_STATUS_FOUND: + r = "Found"; + break; + case HTTP_STATUS_SEE_OTHER: + r = "See Other"; + break; + case HTTP_STATUS_NOT_MODIFIED: + r = "Not Modified"; + break; + case HTTP_STATUS_USE_PROXY: + r = "Use Proxy"; + break; + case HTTP_STATUS_TEMPORARY_REDIRECT: + r = "Temporary Redirect"; + break; + case HTTP_STATUS_BAD_REQUEST: + r = "Bad Request"; + break; + case HTTP_STATUS_UNAUTHORIZED: + r = "Unauthorized"; + break; + case HTTP_STATUS_PAYMENT_REQUIRED: + r = "Payment Required"; + break; + case HTTP_STATUS_FORBIDDEN: + r = "Forbidden"; + break; + case HTTP_STATUS_NOT_FOUND: + r = "Not Found"; + break; + case HTTP_STATUS_METHOD_NOT_ALLOWED: + r = "Method Not Allowed"; + break; + case HTTP_STATUS_NOT_ACCEPTABLE: + r = "Not Acceptable"; + break; + case HTTP_STATUS_PROXY_AUTH_REQUIRED: + r = "Proxy Authentication Required"; + break; + case HTTP_STATUS_REQUEST_TIMEOUT: + r = "Request Time-out"; + break; + case HTTP_STATUS_CONFLICT: + r = "Conflict"; + break; + case HTTP_STATUS_GONE: + r = "Gone"; + break; + case HTTP_STATUS_LENGTH_REQUIRED: + r = "Length Required"; + break; + case HTTP_STATUS_PRECONDITION_FAILED: + r = "Precondition Failed"; + break; + case HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE: + r = "Request Entity Too Large"; + break; + case HTTP_STATUS_REQUEST_URI_TOO_LARGE: + r = "Request-URI Too Large"; + break; + case HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE: + r = "Unsupported Media Type"; + break; + case HTTP_STATUS_REQUEST_RANGE_INVALID: + r = "Requested range not satisfiable"; + break; + case HTTP_STATUS_EXPECTATION_FAILED: + r = "Expectation Failed"; + break; + case HTTP_STATUS_MISDIRECTED_REQUEST: + r = "Misdirected Request"; + break; + case HTTP_STATUS_INTERNAL_ERROR: + r = "Internal Server Error"; + break; + case HTTP_STATUS_NOT_IMPLEMENTED: + r = "Not Implemented"; + break; + case HTTP_STATUS_BAD_GATEWAY: + r = "Bad Gateway"; + break; + case HTTP_STATUS_SERVICE_UNAVAILABLE: + r = "Service Unavailable"; + break; + case HTTP_STATUS_GATEWAY_TIMEOUT: + r = "Gateway Time-out"; + break; + case HTTP_STATUS_BAD_VERSION: + r = "HTTP Version not supported"; + break; + default: + r = ""; + break; + } + + return (r); +} + +const char * +http_method_text(int method) +{ + char *r; + + switch(method) { + case HTTP_METHOD_GET: + r = "GET"; + break; + case HTTP_METHOD_POST: + r = "POST"; + break; + case HTTP_METHOD_PUT: + r = "PUT"; + break; + case HTTP_METHOD_DELETE: + r = "DELETE"; + break; + case HTTP_METHOD_HEAD: + r = "HEAD"; + break; + case HTTP_METHOD_OPTIONS: + r = "OPTIONS"; + break; + case HTTP_METHOD_PATCH: + r = "PATCH"; + break; + default: + r = ""; + break; + } + + return (r); +} + +int +http_method_value(const char *method) +{ + if (!strcasecmp(method, "GET")) + return (HTTP_METHOD_GET); + + if (!strcasecmp(method, "POST")) + return (HTTP_METHOD_POST); + + if (!strcasecmp(method, "PUT")) + return (HTTP_METHOD_PUT); + + if (!strcasecmp(method, "DELETE")) + return (HTTP_METHOD_DELETE); + + if (!strcasecmp(method, "HEAD")) + return (HTTP_METHOD_HEAD); + + if (!strcasecmp(method, "OPTIONS")) + return (HTTP_METHOD_OPTIONS); + + if (!strcasecmp(method, "PATCH")) + return (HTTP_METHOD_PATCH); + + return (0); +} + +int +http_media_register(const char *ext, const char *type) +{ + struct http_media_type *media; + + LIST_FOREACH(media, &http_media_types, list) { + if (!strcasecmp(media->ext, ext)) + return (KORE_RESULT_ERROR); + } + + media = kore_calloc(1, sizeof(*media)); + media->ext = kore_strdup(ext); + media->type = kore_strdup(type); + + LIST_INSERT_HEAD(&http_media_types, media, list); + + return (KORE_RESULT_OK); +} + +const char * +http_media_type(const char *path) +{ + const char *p; + struct http_media_type *media; + + if ((p = strrchr(path, '.')) == NULL) + return (NULL); + + p++; + if (*p == '\0') + return (NULL); + + LIST_FOREACH(media, &http_media_types, list) { + if (!strcasecmp(media->ext, p)) + return (media->type); + } + + return (NULL); +} + +char * +http_validate_header(char *header) +{ + u_int8_t idx; + char *p, *value; + + for (p = header; *p != '\0'; p++) { + idx = *p; + if (idx > HTTP_MAP_LIMIT) + return (NULL); + + if (*p == ':') { + *(p)++ = '\0'; + break; + } + + if (http_token[idx] == 0x00) + return (NULL); + } + + while (isspace(*(unsigned char *)p)) + p++; + + if (*p == '\0') + return (NULL); + + value = p; + while (*p != '\0') { + idx = *p; + if (idx > HTTP_MAP_LIMIT) + return (NULL); + if (http_field_content[idx] == 0x00) + return (NULL); + p++; + } + + return (value); +} + +static int +http_release_buffer(struct netbuf *nb) +{ + kore_buf_free(nb->extra); + + return (KORE_RESULT_OK); +} + static int http_check_redirect(struct http_request *req, struct kore_domain *dom) { @@ -2279,281 +2598,3 @@ http_write_response_cookie(struct http_cookie *ck) kore_buf_appendf(header_buf, "set-cookie: %s\r\n", kore_buf_stringify(ckhdr_buf, NULL)); } - -const char * -http_status_text(int status) -{ - const char *r; - - switch (status) { - case HTTP_STATUS_CONTINUE: - r = "Continue"; - break; - case HTTP_STATUS_SWITCHING_PROTOCOLS: - r = "Switching Protocols"; - break; - case HTTP_STATUS_OK: - r = "OK"; - break; - case HTTP_STATUS_CREATED: - r = "Created"; - break; - case HTTP_STATUS_ACCEPTED: - r = "Accepted"; - break; - case HTTP_STATUS_NON_AUTHORITATIVE: - r = "Non-Authoritative Information"; - break; - case HTTP_STATUS_NO_CONTENT: - r = "No Content"; - break; - case HTTP_STATUS_RESET_CONTENT: - r = "Reset Content"; - break; - case HTTP_STATUS_PARTIAL_CONTENT: - r = "Partial Content"; - break; - case HTTP_STATUS_MULTIPLE_CHOICES: - r = "Multiple Choices"; - break; - case HTTP_STATUS_MOVED_PERMANENTLY: - r = "Moved Permanently"; - break; - case HTTP_STATUS_FOUND: - r = "Found"; - break; - case HTTP_STATUS_SEE_OTHER: - r = "See Other"; - break; - case HTTP_STATUS_NOT_MODIFIED: - r = "Not Modified"; - break; - case HTTP_STATUS_USE_PROXY: - r = "Use Proxy"; - break; - case HTTP_STATUS_TEMPORARY_REDIRECT: - r = "Temporary Redirect"; - break; - case HTTP_STATUS_BAD_REQUEST: - r = "Bad Request"; - break; - case HTTP_STATUS_UNAUTHORIZED: - r = "Unauthorized"; - break; - case HTTP_STATUS_PAYMENT_REQUIRED: - r = "Payment Required"; - break; - case HTTP_STATUS_FORBIDDEN: - r = "Forbidden"; - break; - case HTTP_STATUS_NOT_FOUND: - r = "Not Found"; - break; - case HTTP_STATUS_METHOD_NOT_ALLOWED: - r = "Method Not Allowed"; - break; - case HTTP_STATUS_NOT_ACCEPTABLE: - r = "Not Acceptable"; - break; - case HTTP_STATUS_PROXY_AUTH_REQUIRED: - r = "Proxy Authentication Required"; - break; - case HTTP_STATUS_REQUEST_TIMEOUT: - r = "Request Time-out"; - break; - case HTTP_STATUS_CONFLICT: - r = "Conflict"; - break; - case HTTP_STATUS_GONE: - r = "Gone"; - break; - case HTTP_STATUS_LENGTH_REQUIRED: - r = "Length Required"; - break; - case HTTP_STATUS_PRECONDITION_FAILED: - r = "Precondition Failed"; - break; - case HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE: - r = "Request Entity Too Large"; - break; - case HTTP_STATUS_REQUEST_URI_TOO_LARGE: - r = "Request-URI Too Large"; - break; - case HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE: - r = "Unsupported Media Type"; - break; - case HTTP_STATUS_REQUEST_RANGE_INVALID: - r = "Requested range not satisfiable"; - break; - case HTTP_STATUS_EXPECTATION_FAILED: - r = "Expectation Failed"; - break; - case HTTP_STATUS_MISDIRECTED_REQUEST: - r = "Misdirected Request"; - break; - case HTTP_STATUS_INTERNAL_ERROR: - r = "Internal Server Error"; - break; - case HTTP_STATUS_NOT_IMPLEMENTED: - r = "Not Implemented"; - break; - case HTTP_STATUS_BAD_GATEWAY: - r = "Bad Gateway"; - break; - case HTTP_STATUS_SERVICE_UNAVAILABLE: - r = "Service Unavailable"; - break; - case HTTP_STATUS_GATEWAY_TIMEOUT: - r = "Gateway Time-out"; - break; - case HTTP_STATUS_BAD_VERSION: - r = "HTTP Version not supported"; - break; - default: - r = ""; - break; - } - - return (r); -} - -const char * -http_method_text(int method) -{ - char *r; - - switch(method) { - case HTTP_METHOD_GET: - r = "GET"; - break; - case HTTP_METHOD_POST: - r = "POST"; - break; - case HTTP_METHOD_PUT: - r = "PUT"; - break; - case HTTP_METHOD_DELETE: - r = "DELETE"; - break; - case HTTP_METHOD_HEAD: - r = "HEAD"; - break; - case HTTP_METHOD_OPTIONS: - r = "OPTIONS"; - break; - case HTTP_METHOD_PATCH: - r = "PATCH"; - break; - default: - r = ""; - break; - } - - return (r); -} - -int -http_method_value(const char *method) -{ - if (!strcasecmp(method, "GET")) - return (HTTP_METHOD_GET); - - if (!strcasecmp(method, "POST")) - return (HTTP_METHOD_POST); - - if (!strcasecmp(method, "PUT")) - return (HTTP_METHOD_PUT); - - if (!strcasecmp(method, "DELETE")) - return (HTTP_METHOD_DELETE); - - if (!strcasecmp(method, "HEAD")) - return (HTTP_METHOD_HEAD); - - if (!strcasecmp(method, "OPTIONS")) - return (HTTP_METHOD_OPTIONS); - - if (!strcasecmp(method, "PATCH")) - return (HTTP_METHOD_PATCH); - - return (0); -} - -int -http_media_register(const char *ext, const char *type) -{ - struct http_media_type *media; - - LIST_FOREACH(media, &http_media_types, list) { - if (!strcasecmp(media->ext, ext)) - return (KORE_RESULT_ERROR); - } - - media = kore_calloc(1, sizeof(*media)); - media->ext = kore_strdup(ext); - media->type = kore_strdup(type); - - LIST_INSERT_HEAD(&http_media_types, media, list); - - return (KORE_RESULT_OK); -} - -const char * -http_media_type(const char *path) -{ - const char *p; - struct http_media_type *media; - - if ((p = strrchr(path, '.')) == NULL) - return (NULL); - - p++; - if (*p == '\0') - return (NULL); - - LIST_FOREACH(media, &http_media_types, list) { - if (!strcasecmp(media->ext, p)) - return (media->type); - } - - return (NULL); -} - -char * -http_validate_header(char *header) -{ - u_int8_t idx; - char *p, *value; - - for (p = header; *p != '\0'; p++) { - idx = *p; - if (idx > HTTP_MAP_LIMIT) - return (NULL); - - if (*p == ':') { - *(p)++ = '\0'; - break; - } - - if (http_token[idx] == 0x00) - return (NULL); - } - - while (isspace(*(unsigned char *)p)) - p++; - - if (*p == '\0') - return (NULL); - - value = p; - while (*p != '\0') { - idx = *p; - if (idx > HTTP_MAP_LIMIT) - return (NULL); - if (http_field_content[idx] == 0x00) - return (NULL); - p++; - } - - return (value); -} From 4b4767ac39c336be84bb9e5f74067d2bf32cbcfb Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 12 Sep 2021 15:12:48 +0200 Subject: [PATCH 063/121] Missed committing http.h from previous commit. --- include/kore/http.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/kore/http.h b/include/kore/http.h index db8dce3..7d823e5 100644 --- a/include/kore/http.h +++ b/include/kore/http.h @@ -357,6 +357,8 @@ int http_body_digest(struct http_request *, char *, size_t); int http_redirect_add(struct kore_domain *, const char *, int, const char *); void http_response(struct http_request *, int, const void *, size_t); +void http_response_json(struct http_request *, int, + struct kore_json_item *); void http_response_close(struct http_request *, int, const void *, size_t); void http_response_fileref(struct http_request *, int, From 77848e07088c30fac9f2a836e321d5ed0ab426cf Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 12 Sep 2021 15:19:37 +0200 Subject: [PATCH 064/121] Always use logfile when set, even if !foreground. --- src/log.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/log.c b/src/log.c index ab2cce7..1edea65 100644 --- a/src/log.c +++ b/src/log.c @@ -92,7 +92,7 @@ kore_log(int prio, const char *fmt, ...) } else { str = kore_buf_stringify(&buf, NULL); - if (kore_foreground) + if (kore_foreground || fp != stdout) log_print(prio, "[parent]: %s\n", str); else syslog(prio, "[parent]: %s", str); @@ -117,7 +117,7 @@ log_from_worker(struct kore_msg *msg, const void *data) wlog = data; name = kore_worker_name(wlog->wid); - if (kore_foreground) { + if (kore_foreground || fp != stdout) { log_print(wlog->prio, "%s: %.*s\n", name, (int)wlog->loglen, wlog->logmsg); } else { From 450aabbea17e0d99a563fdf35932eed23f9f0c2c Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 13 Sep 2021 15:07:43 +0200 Subject: [PATCH 065/121] Add timestamp prefix to log when not using syslog. --- src/log.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/log.c b/src/log.c index 1edea65..7e3570f 100644 --- a/src/log.c +++ b/src/log.c @@ -16,6 +16,7 @@ #include +#include #include #include "kore.h" @@ -129,7 +130,10 @@ log_from_worker(struct kore_msg *msg, const void *data) static void log_print(int prio, const char *fmt, ...) { + struct tm *t; + time_t now; va_list args; + char tbuf[32]; va_start(args, fmt); @@ -142,6 +146,12 @@ log_print(int prio, const char *fmt, ...) break; } + time(&now); + t = localtime(&now); + + if (strftime(tbuf, sizeof(tbuf), "%y-%m-%d %H:%S:%M", t) > 0) + fprintf(fp, "%s ", tbuf); + vfprintf(fp, fmt, args); fflush(fp); From 983f5a03f5c034975b2803a02571914e30008c29 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 13 Sep 2021 15:33:42 +0200 Subject: [PATCH 066/121] Initial mem and log earlier. Kill the kodev mention. --- src/kore.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/kore.c b/src/kore.c index ecf315c..b692752 100644 --- a/src/kore.c +++ b/src/kore.c @@ -168,6 +168,9 @@ main(int argc, char *argv[]) #endif kore_mem_init(); + kore_msg_init(); + kore_log_init(); + kore_progname = kore_strdup(argv[0]); kore_proctitle_setup(); @@ -194,10 +197,6 @@ main(int argc, char *argv[]) if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) fatal("%s: not a directory or file", kore_pymodule); } - -#elif !defined(KORE_SINGLE_BINARY) - if (argc > 0) - fatal("did you mean to run `kodev' instead?"); #endif kore_pid = getpid(); @@ -205,8 +204,6 @@ main(int argc, char *argv[]) LIST_INIT(&kore_servers); kore_platform_init(); - kore_msg_init(); - kore_log_init(); #if !defined(KORE_NO_HTTP) http_parent_init(); #if defined(KORE_USE_CURL) From 824d6421d5a006c5b6575be0b1b24388acfedf7a Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 13 Sep 2021 22:54:05 +0200 Subject: [PATCH 067/121] Use correct format for strftime(). --- src/log.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/log.c b/src/log.c index 7e3570f..8af92bc 100644 --- a/src/log.c +++ b/src/log.c @@ -149,8 +149,8 @@ log_print(int prio, const char *fmt, ...) time(&now); t = localtime(&now); - if (strftime(tbuf, sizeof(tbuf), "%y-%m-%d %H:%S:%M", t) > 0) - fprintf(fp, "%s ", tbuf); + if (strftime(tbuf, sizeof(tbuf), "%y-%m-%d %H:%M:%S", t) > 0) + fprintf(fp, "[%ld] %s ", now, tbuf); vfprintf(fp, fmt, args); fflush(fp); From b6ec4081d569972f5d37d0b9acb44cdb33c8f43b Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 13 Sep 2021 22:55:09 +0200 Subject: [PATCH 068/121] Oops, remove debug. --- src/log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/log.c b/src/log.c index 8af92bc..beac13d 100644 --- a/src/log.c +++ b/src/log.c @@ -150,7 +150,7 @@ log_print(int prio, const char *fmt, ...) t = localtime(&now); if (strftime(tbuf, sizeof(tbuf), "%y-%m-%d %H:%M:%S", t) > 0) - fprintf(fp, "[%ld] %s ", now, tbuf); + fprintf(fp, "%s ", tbuf); vfprintf(fp, fmt, args); fflush(fp); From 41511c16836953ed50443da3c308582b07009882 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 14 Sep 2021 09:30:17 +0200 Subject: [PATCH 069/121] Log timestamps in UTC for, add milliseconds. This is when using the normal foreground logs or a specified logfile. --- src/log.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/log.c b/src/log.c index beac13d..16e075a 100644 --- a/src/log.c +++ b/src/log.c @@ -130,10 +130,10 @@ log_from_worker(struct kore_msg *msg, const void *data) static void log_print(int prio, const char *fmt, ...) { - struct tm *t; - time_t now; - va_list args; - char tbuf[32]; + struct tm *t; + struct timespec ts; + va_list args; + char tbuf[32]; va_start(args, fmt); @@ -146,11 +146,11 @@ log_print(int prio, const char *fmt, ...) break; } - time(&now); - t = localtime(&now); + (void)clock_gettime(CLOCK_REALTIME, &ts); + t = gmtime(&ts.tv_sec); - if (strftime(tbuf, sizeof(tbuf), "%y-%m-%d %H:%M:%S", t) > 0) - fprintf(fp, "%s ", tbuf); + if (strftime(tbuf, sizeof(tbuf), "%Y-%m-%d %H:%M:%S", t) > 0) + fprintf(fp, "%s.%ld UTC ", tbuf, ts.tv_nsec / 1000000); vfprintf(fp, fmt, args); fflush(fp); From a2d48feeb7e279d0e84068e1d7195b85eba4137a Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 14 Sep 2021 09:36:33 +0200 Subject: [PATCH 070/121] Lets make sure milliseconds are formatted nicely. --- src/log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/log.c b/src/log.c index 16e075a..0e73341 100644 --- a/src/log.c +++ b/src/log.c @@ -150,7 +150,7 @@ log_print(int prio, const char *fmt, ...) t = gmtime(&ts.tv_sec); if (strftime(tbuf, sizeof(tbuf), "%Y-%m-%d %H:%M:%S", t) > 0) - fprintf(fp, "%s.%ld UTC ", tbuf, ts.tv_nsec / 1000000); + fprintf(fp, "%s.%03ld UTC ", tbuf, ts.tv_nsec / 1000000); vfprintf(fp, fmt, args); fflush(fp); From 2576427dc07321bd80c9ccbf18d5f5bb3221f942 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 14 Sep 2021 15:45:34 +0200 Subject: [PATCH 071/121] Simplify if a worker needs to be added to msg. Just look if the kore_worker data structure its ps member is non NULL. If it is, it was started by the parent process at some point. --- src/msg.c | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/msg.c b/src/msg.c index 977a368..a01f6ac 100644 --- a/src/msg.c +++ b/src/msg.c @@ -60,23 +60,9 @@ kore_msg_parent_init(void) struct kore_worker *kw; for (idx = 0; idx < worker_count; idx++) { - if (keymgr_active == 0) { - if (idx == KORE_WORKER_KEYMGR_IDX || - idx == KORE_WORKER_ACME_IDX) - continue; - } - - if (idx == KORE_WORKER_ACME_IDX) { -#if defined(KORE_USE_ACME) - if (acme_domains == 0) - continue; -#else - continue; -#endif - } - kw = kore_worker_data(idx); - kore_msg_parent_add(kw); + if (kw->ps != NULL) + kore_msg_parent_add(kw); } kore_msg_register(KORE_MSG_SHUTDOWN, msg_type_shutdown); From e98a4ddab58d4e8dce21a4dc512f05cc278685d8 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 15 Sep 2021 11:09:52 +0200 Subject: [PATCH 072/121] Change how routes are configured in Kore. Routes are now configured in a context per route: route /path { handler handler_name methods get post head validate qs:get id v_id } All route related configurations are per-route, allowing multiple routes for the same path (for different methods). The param context is removed and merged into the route context now so that you use the validate keyword to specify what needs validating. --- Makefile | 2 +- examples/async-curl/conf/async-curl.conf | 11 +- examples/cookies/conf/cookies.conf | 16 +- examples/cpp/conf/cpp.conf | 6 +- examples/cpp/dh2048.pem | 8 - examples/generic/conf/generic.conf | 2 - examples/integers/README.md | 6 +- examples/integers/conf/integers.conf | 12 +- examples/json/conf/json.conf | 8 +- examples/json_yajl/.gitignore | 5 - examples/json_yajl/README.md | 21 - examples/json_yajl/conf/build.conf | 19 - examples/json_yajl/conf/json_yajl.conf | 18 - examples/json_yajl/src/json_yajl.c | 98 ---- examples/jsonrpc/conf/jsonrpc.conf | 11 +- examples/memtag/conf/memtag.conf | 6 +- examples/messaging/conf/messaging.conf | 12 +- examples/nohttp/conf/nohttp.conf | 2 - examples/parameters/conf/parameters.conf | 15 +- examples/pgsql-sync/conf/pgsql-sync.conf | 8 +- examples/pgsql/conf/pgsql.conf | 11 +- examples/pipe_task/conf/pipe_task.conf | 12 +- examples/pipe_task/src/pipe_task.c | 14 +- examples/sse/conf/sse.conf | 15 +- examples/tasks/conf/tasks.conf | 16 +- examples/tls-proxy/conf/tls-proxy.conf | 1 - examples/upload/conf/upload.conf | 6 +- examples/video_stream/conf/video_stream.conf | 11 +- examples/websocket/conf/websocket.conf | 11 +- include/kore/http.h | 2 +- include/kore/kore.h | 71 +-- src/accesslog.c | 2 +- src/cli.c | 5 +- src/config.c | 503 ++++++++++--------- src/domain.c | 25 +- src/filemap.c | 19 +- src/http.c | 18 +- src/module.c | 128 +---- src/python.c | 87 ++-- src/route.c | 138 +++++ src/worker.c | 19 +- 41 files changed, 631 insertions(+), 769 deletions(-) delete mode 100755 examples/cpp/dh2048.pem delete mode 100755 examples/json_yajl/.gitignore delete mode 100644 examples/json_yajl/README.md delete mode 100644 examples/json_yajl/conf/build.conf delete mode 100755 examples/json_yajl/conf/json_yajl.conf delete mode 100755 examples/json_yajl/src/json_yajl.c create mode 100644 src/route.c diff --git a/Makefile b/Makefile index c036bc9..fd8cb8b 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ ifneq ("$(NOHTTP)", "") FEATURES+=-DKORE_NO_HTTP else S_SRC+= src/auth.c src/accesslog.c src/http.c \ - src/validator.c src/websocket.c + src/route.c src/validator.c src/websocket.c endif ifneq ("$(PGSQL)", "") diff --git a/examples/async-curl/conf/async-curl.conf b/examples/async-curl/conf/async-curl.conf index 9f7c8fd..85c623a 100644 --- a/examples/async-curl/conf/async-curl.conf +++ b/examples/async-curl/conf/async-curl.conf @@ -5,8 +5,6 @@ server tls { } workers 1 -tls_dhparam dh2048.pem - pledge dns domain * { @@ -15,6 +13,11 @@ domain * { certfile cert/server.pem certkey cert/key.pem - route / http - route /ftp ftp + route / { + handler http + } + + route /ftp { + handler ftp + } } diff --git a/examples/cookies/conf/cookies.conf b/examples/cookies/conf/cookies.conf index 34f3dfd..1c0b219 100644 --- a/examples/cookies/conf/cookies.conf +++ b/examples/cookies/conf/cookies.conf @@ -6,15 +6,21 @@ server tls { load ./cookies.so -tls_dhparam dh2048.pem - domain * { attach tls certfile cert/server.pem certkey cert/key.pem - route / serve_cookies - route /secure serve_cookies - route /vault serve_cookies + route / { + handler serve_cookies + } + + route /secure { + handler serve_cookies + } + + route /vault { + handler serve_cookies + } } diff --git a/examples/cpp/conf/cpp.conf b/examples/cpp/conf/cpp.conf index 9cb58c7..6ed806f 100755 --- a/examples/cpp/conf/cpp.conf +++ b/examples/cpp/conf/cpp.conf @@ -5,12 +5,14 @@ server tls { } load ./cpp.so -tls_dhparam dh2048.pem domain * { attach tls certfile cert/server.pem certkey cert/key.pem - route / page + + route / { + handler page + } } diff --git a/examples/cpp/dh2048.pem b/examples/cpp/dh2048.pem deleted file mode 100755 index 511de69..0000000 --- a/examples/cpp/dh2048.pem +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN DH PARAMETERS----- -MIIBCAKCAQEAn4f4Qn5SudFjEYPWTbUaOTLUH85YWmmPFW1+b5bRa9ygr+1wfamv -VKVT7jO8c4msSNikUf6eEfoH0H4VTCaj+Habwu+Sj+I416r3mliMD4SjNsUJrBrY -Y0QV3ZUgZz4A8ARk/WwQcRl8+ZXJz34IaLwAcpyNhoV46iHVxW0ty8ND0U4DIku/ -PNayKimu4BXWXk4RfwNVP59t8DQKqjshZ4fDnbotskmSZ+e+FHrd+Kvrq/WButvV -Bzy9fYgnUlJ82g/bziCI83R2xAdtH014fR63MpElkqdNeChb94pPbEdFlNUvYIBN -xx2vTUQMqRbB4UdG2zuzzr5j98HDdblQ+wIBAg== ------END DH PARAMETERS----- \ No newline at end of file diff --git a/examples/generic/conf/generic.conf b/examples/generic/conf/generic.conf index 18cdcca..5b43814 100644 --- a/examples/generic/conf/generic.conf +++ b/examples/generic/conf/generic.conf @@ -6,8 +6,6 @@ server tls { load ./generic.so example_load -tls_dhparam dh2048.pem - http_body_max 1024000000 http_body_disk_offload 1024000 diff --git a/examples/integers/README.md b/examples/integers/README.md index c2c1748..974a965 100644 --- a/examples/integers/README.md +++ b/examples/integers/README.md @@ -2,13 +2,13 @@ Test parameter to integer conversions. Run: ``` - # kodev run + $ kodev run ``` Test: ``` - # curl -i -k https://127.0.0.1:8888/?id=123123 - # curl -i -k https://127.0.0.1:8888/?id=-123123 + $ curl -i -k https://127.0.0.1:8888/?id=123123 + $ curl -i -k https://127.0.0.1:8888/?id=-123123 ``` The correct integer types should only be represented in the output. diff --git a/examples/integers/conf/integers.conf b/examples/integers/conf/integers.conf index 0493ff7..76d7941 100755 --- a/examples/integers/conf/integers.conf +++ b/examples/integers/conf/integers.conf @@ -9,8 +9,6 @@ load ./integers.so workers 2 worker_max_connections 5000 -tls_dhparam dh2048.pem - validator v_id regex ^-?[0-9]*.?[0-9]+$ domain * { @@ -18,10 +16,12 @@ domain * { certfile cert/server.pem certkey cert/key.pem - route / page - # allowed parameters in the query string for GETs - params qs:get / { - validate id v_id + route / { + handler page + methods get + + # allowed parameters in the query string for GETs + validate get id v_id } } diff --git a/examples/json/conf/json.conf b/examples/json/conf/json.conf index b896c13..3e26448 100755 --- a/examples/json/conf/json.conf +++ b/examples/json/conf/json.conf @@ -6,14 +6,14 @@ server tls { load ./json.so -tls_dhparam dh2048.pem - domain 127.0.0.1 { attach tls certfile cert/server.pem certkey cert/key.pem - route / page - restrict / post + route / { + handler page + methods post + } } diff --git a/examples/json_yajl/.gitignore b/examples/json_yajl/.gitignore deleted file mode 100755 index 28ab2ed..0000000 --- a/examples/json_yajl/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.o -.objs -json_yajl.so -assets.h -cert diff --git a/examples/json_yajl/README.md b/examples/json_yajl/README.md deleted file mode 100644 index 3c12590..0000000 --- a/examples/json_yajl/README.md +++ /dev/null @@ -1,21 +0,0 @@ -This example demonstrates how you can use external libs in your application. - -In this case we link against yajl (Yet Another JSON library) in order to -parse a JSON string that was POSTed to the server. - -Take a peek at conf/build.conf for different build flavors and how to -link to other libraries. - -Run: -``` - $ kodev run -``` - -Test: -``` - $ curl -i -k -d '{"foo":{"bar": "Hello world"}}' https://127.0.0.1:8888 -``` - -The result should echo back the foo.bar JSON path value: Hello world. - -The yajl repo is available @ https://github.com/lloyd/yajl diff --git a/examples/json_yajl/conf/build.conf b/examples/json_yajl/conf/build.conf deleted file mode 100644 index ec76646..0000000 --- a/examples/json_yajl/conf/build.conf +++ /dev/null @@ -1,19 +0,0 @@ -# json_yajl build config -# You can switch flavors using: kodev flavor [newflavor] - -# The cflags below are shared between flavors -cflags=-Wall -Wmissing-declarations -Wshadow -cflags=-Wstrict-prototypes -Wmissing-prototypes -cflags=-Wpointer-arith -Wcast-qual -Wsign-compare - -dev { - # These cflags are added to the shared ones when - # you build the "dev" flavor. - cflags=-g - ldflags=-lyajl -} - -#prod { -# You can specify additional CFLAGS here which are only -# included if you build with the "prod" flavor. -#} diff --git a/examples/json_yajl/conf/json_yajl.conf b/examples/json_yajl/conf/json_yajl.conf deleted file mode 100755 index 29d2c01..0000000 --- a/examples/json_yajl/conf/json_yajl.conf +++ /dev/null @@ -1,18 +0,0 @@ -# Placeholder configuration - -server tls { - bind 127.0.0.1 8888 -} - -load ./json_yajl.so - -tls_dhparam dh2048.pem - -domain 127.0.0.1 { - attach tls - - certfile cert/server.pem - certkey cert/key.pem - - route / page -} diff --git a/examples/json_yajl/src/json_yajl.c b/examples/json_yajl/src/json_yajl.c deleted file mode 100755 index 8490e7f..0000000 --- a/examples/json_yajl/src/json_yajl.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2013-2018 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include -#include - -#include - -int page(struct http_request *); - -int -page(struct http_request *req) -{ - ssize_t ret; - struct kore_buf *buf; - char *body; - yajl_val node, v; - char eb[1024]; - u_int8_t data[BUFSIZ]; - const char *path[] = { "foo", "bar", NULL }; - - /* We only allow POST/PUT methods. */ - if (req->method != HTTP_METHOD_POST && - req->method != HTTP_METHOD_PUT) { - http_response_header(req, "allow", "POST, PUT"); - http_response(req, HTTP_STATUS_METHOD_NOT_ALLOWED, NULL, 0); - return (KORE_RESULT_OK); - } - - /* - * Read the entire received body into a memory buffer. - */ - buf = kore_buf_alloc(128); - for (;;) { - ret = http_body_read(req, data, sizeof(data)); - if (ret == -1) { - kore_buf_free(buf); - kore_log(LOG_NOTICE, "error reading body"); - http_response(req, 500, NULL, 0); - return (KORE_RESULT_OK); - } - - if (ret == 0) - break; - - kore_buf_append(buf, data, ret); - } - - /* Grab our body data as a NUL-terminated string. */ - body = kore_buf_stringify(buf, NULL); - - /* Parse the body via yajl now. */ - node = yajl_tree_parse(body, eb, sizeof(eb)); - if (node == NULL) { - if (strlen(eb)) { - kore_log(LOG_NOTICE, "parse error: %s", eb); - } else { - kore_log(LOG_NOTICE, "parse error: unknown"); - } - - kore_buf_free(buf); - http_response(req, 400, NULL, 0); - return (KORE_RESULT_OK); - } - - /* Reuse old buffer, don't need it anymore for body. */ - kore_buf_reset(buf); - - /* Attempt to grab foo.bar from the JSON tree. */ - v = yajl_tree_get(node, path, yajl_t_string); - if (v == NULL) { - kore_buf_appendf(buf, "no such path: foo.bar\n"); - } else { - kore_buf_appendf(buf, "foo.bar = '%s'\n", YAJL_GET_STRING(v)); - } - - /* Release the JSON tree now. */ - yajl_tree_free(node); - - /* Respond to the client. */ - http_response(req, 200, buf->data, buf->offset); - kore_buf_free(buf); - - return (KORE_RESULT_OK); -} diff --git a/examples/jsonrpc/conf/jsonrpc.conf b/examples/jsonrpc/conf/jsonrpc.conf index c8ff8d9..f727f12 100644 --- a/examples/jsonrpc/conf/jsonrpc.conf +++ b/examples/jsonrpc/conf/jsonrpc.conf @@ -6,14 +6,17 @@ server tls { load ./jsonrpc.so -tls_dhparam dh2048.pem - domain * { attach tls certfile cert/server.pem certkey cert/key.pem - route / homepage - route /v1 v1 + route / { + handler homepage + } + + route /v1 { + handler v1 + } } diff --git a/examples/memtag/conf/memtag.conf b/examples/memtag/conf/memtag.conf index f981b6c..5e9fc7b 100644 --- a/examples/memtag/conf/memtag.conf +++ b/examples/memtag/conf/memtag.conf @@ -6,13 +6,13 @@ server tls { load ./memtag.so init -tls_dhparam dh2048.pem - domain * { attach tls certfile cert/server.pem certkey cert/key.pem - route / page + route / { + handler page + } } diff --git a/examples/messaging/conf/messaging.conf b/examples/messaging/conf/messaging.conf index f8680ae..e195cb0 100644 --- a/examples/messaging/conf/messaging.conf +++ b/examples/messaging/conf/messaging.conf @@ -5,14 +5,18 @@ server tls { } load ./messaging.so init -tls_dhparam dh2048.pem -workers 4 domain * { attach tls certfile cert/server.pem certkey cert/key.pem - route / page - route /shutdown page_shutdown + + route / { + handler page + } + + route /shutdown { + handler page_shutdown + } } diff --git a/examples/nohttp/conf/nohttp.conf b/examples/nohttp/conf/nohttp.conf index d5912ad..409f37a 100644 --- a/examples/nohttp/conf/nohttp.conf +++ b/examples/nohttp/conf/nohttp.conf @@ -4,8 +4,6 @@ server tls { bind 127.0.0.1 8888 connection_setup } -tls_dhparam dh2048.pem - domain * { attach tls diff --git a/examples/parameters/conf/parameters.conf b/examples/parameters/conf/parameters.conf index c51798c..b6fe08c 100755 --- a/examples/parameters/conf/parameters.conf +++ b/examples/parameters/conf/parameters.conf @@ -6,8 +6,6 @@ server tls { load ./parameters.so -tls_dhparam dh2048.pem - # The validator used to validate the 'id' parameter # defined below. We'll use a simple regex to make sure # it only matches positive numbers. @@ -19,15 +17,10 @@ domain * { certfile cert/server.pem certkey cert/key.pem - route / page + route / { + handler page + methods get - # The parameters allowed for "/" (GET method). - # - # If you would want to declare parameters available - # to the page handler for POST, swap the 'get' setting - # to 'post' instead, Kore takes care of the rest. - params qs:get / { - # Validate the id parameter with the v_id validator. - validate id v_id + validate qs:get id v_id } } diff --git a/examples/pgsql-sync/conf/pgsql-sync.conf b/examples/pgsql-sync/conf/pgsql-sync.conf index 543c2b4..9ef54fa 100644 --- a/examples/pgsql-sync/conf/pgsql-sync.conf +++ b/examples/pgsql-sync/conf/pgsql-sync.conf @@ -4,13 +4,15 @@ server tls { bind 127.0.0.1 8888 } -load ./pgsql-sync.so init -tls_dhparam dh2048.pem +load ./pgsql-sync.so init domain * { attach tls certfile cert/server.pem certkey cert/key.pem - route / page + + route / { + handler page + } } diff --git a/examples/pgsql/conf/pgsql.conf b/examples/pgsql/conf/pgsql.conf index faeb3c9..49e0133 100755 --- a/examples/pgsql/conf/pgsql.conf +++ b/examples/pgsql/conf/pgsql.conf @@ -10,8 +10,6 @@ server other { bind 127.0.0.1 8889 connection_new } -tls_dhparam dh2048.pem - http_keepalive_time 0 domain * { @@ -20,6 +18,11 @@ domain * { certfile cert/server.pem certkey cert/key.pem - route / page - route /hello hello + route / { + handler page + } + + route /hello { + handler hello + } } diff --git a/examples/pipe_task/conf/pipe_task.conf b/examples/pipe_task/conf/pipe_task.conf index 4be394e..56b40ac 100755 --- a/examples/pipe_task/conf/pipe_task.conf +++ b/examples/pipe_task/conf/pipe_task.conf @@ -4,8 +4,6 @@ server tls { bind 127.0.0.1 8888 } -tls_dhparam dh2048.pem - websocket_maxframe 65536 websocket_timeout 10000 @@ -15,6 +13,12 @@ domain * { certfile cert/server.pem certkey cert/key.pem - route / page - route /connect page_ws_connect + route / { + handler page + } + + route /connect { + handler page_ws_connect + methods get + } } diff --git a/examples/pipe_task/src/pipe_task.c b/examples/pipe_task/src/pipe_task.c index 0b981b5..472e503 100755 --- a/examples/pipe_task/src/pipe_task.c +++ b/examples/pipe_task/src/pipe_task.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -49,17 +50,12 @@ void pipe_data_available(struct kore_task *); /* Our pipe reader. */ struct kore_task pipe_task; -/* Module init function (see config). */ -int -init(int state) +void +kore_worker_configure(void) { - /* Do not allow reload. */ - if (state == KORE_MODULE_UNLOAD) - return (KORE_RESULT_ERROR); - /* Only do this on a dedicated worker. */ if (worker->id != 1) - return (KORE_RESULT_OK); + return; /* Create our task. */ kore_task_create(&pipe_task, pipe_reader); @@ -69,8 +65,6 @@ init(int state) /* Start the task. */ kore_task_run(&pipe_task); - - return (KORE_RESULT_OK); } /* Called whenever we get a new websocket connection. */ diff --git a/examples/sse/conf/sse.conf b/examples/sse/conf/sse.conf index e373708..03cfb2c 100755 --- a/examples/sse/conf/sse.conf +++ b/examples/sse/conf/sse.conf @@ -4,10 +4,8 @@ server tls { bind 127.0.0.1 8888 } -load ./sse.so -tls_dhparam dh2048.pem - -http_keepalive_time 600 +load ./sse.so +http_keepalive_time 600 domain * { attach tls @@ -15,6 +13,11 @@ domain * { certfile cert/server.pem certkey cert/key.pem - route / page - route /subscribe subscribe + route / { + handler page + } + + route /subscribe { + handler subscribe + } } diff --git a/examples/tasks/conf/tasks.conf b/examples/tasks/conf/tasks.conf index b6f34df..3768391 100644 --- a/examples/tasks/conf/tasks.conf +++ b/examples/tasks/conf/tasks.conf @@ -4,8 +4,6 @@ server tls { bind 127.0.0.1 8888 } -tls_dhparam dh2048.pem - task_threads 4 worker_max_connections 1000 http_keepalive_time 0 @@ -19,14 +17,14 @@ domain * { certkey cert/key.pem accesslog kore_access.log - route / page_handler - route /post_back post_back - - params qs:get / { - validate user v_user + route / { + handler page_handler + validate qs:get user v_user } - params post /post_back { - validate user v_user + route /post_back { + handler post_back + methods post + validate post user v_user } } diff --git a/examples/tls-proxy/conf/tls-proxy.conf b/examples/tls-proxy/conf/tls-proxy.conf index 8b0742b..2a58dff 100755 --- a/examples/tls-proxy/conf/tls-proxy.conf +++ b/examples/tls-proxy/conf/tls-proxy.conf @@ -1,7 +1,6 @@ # Kore as a TLS proxy configuration. load ./tls-proxy.so -tls_dhparam dh2048.pem # # Bind the proxy to a given IP and port. For every diff --git a/examples/upload/conf/upload.conf b/examples/upload/conf/upload.conf index 1a73855..8958ff5 100644 --- a/examples/upload/conf/upload.conf +++ b/examples/upload/conf/upload.conf @@ -6,8 +6,6 @@ server tls { load ./upload.so -tls_dhparam dh2048.pem - http_body_max 1024000000 http_body_disk_offload 4096 @@ -17,5 +15,7 @@ domain * { certfile cert/server.pem certkey cert/key.pem - route / page + route / { + handler page + } } diff --git a/examples/video_stream/conf/video_stream.conf b/examples/video_stream/conf/video_stream.conf index c44f753..c918e6f 100755 --- a/examples/video_stream/conf/video_stream.conf +++ b/examples/video_stream/conf/video_stream.conf @@ -6,8 +6,6 @@ server tls { load ./video_stream.so init -tls_dhparam dh2048.pem - http_keepalive_time 600 domain * { @@ -17,6 +15,11 @@ domain * { certkey cert/key.pem accesslog access.log - route / asset_serve_video_html - route ^/[a-z]*.[a-z0-9]{3}$ video_stream + route / { + handler asset_serve_video_html + } + + route ^/[a-z]*.[a-z0-9]{3}$ { + handler video_stream + } } diff --git a/examples/websocket/conf/websocket.conf b/examples/websocket/conf/websocket.conf index 228f8d1..ac1ef9c 100755 --- a/examples/websocket/conf/websocket.conf +++ b/examples/websocket/conf/websocket.conf @@ -6,8 +6,6 @@ server tls { load ./websocket.so -tls_dhparam dh2048.pem - # Increase workers so connections are spread # across them to demonstrate WEBSOCKET_BROADCAST_GLOBAL. workers 4 @@ -21,6 +19,11 @@ domain * { certfile cert/server.pem certkey cert/key.pem - route / page - route /connect page_ws_connect + route / { + handler page + } + + route /connect { + handler page_ws_connect + } } diff --git a/include/kore/http.h b/include/kore/http.h index 7d823e5..9b3cf6c 100644 --- a/include/kore/http.h +++ b/include/kore/http.h @@ -268,7 +268,7 @@ struct http_request { void *hdlr_extra; size_t state_len; char *query_string; - struct kore_module_handle *hdlr; + struct kore_route *rt; struct http_runlock_queue *runlock; void (*onfree)(struct http_request *); diff --git a/include/kore/kore.h b/include/kore/kore.h index d7e59c5..ecb0749 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -304,6 +304,33 @@ struct kore_runtime_call { struct kore_runtime *runtime; }; +#if !defined(KORE_NO_HTTP) + +struct kore_route_params { + char *name; + int flags; + u_int8_t method; + struct kore_validator *validator; + + TAILQ_ENTRY(kore_route_params) list; +}; + +struct kore_route { + char *path; + char *func; + int type; + int errors; + regex_t rctx; + struct kore_domain *dom; + struct kore_runtime_call *rcall; + struct kore_auth *auth; + int methods; + TAILQ_HEAD(, kore_route_params) params; + TAILQ_ENTRY(kore_route) list; +}; + +#endif + struct kore_domain { u_int16_t id; int logerr; @@ -327,7 +354,7 @@ struct kore_domain { SSL_CTX *ssl_ctx; int x509_verify_depth; #if !defined(KORE_NO_HTTP) - TAILQ_HEAD(, kore_module_handle) handlers; + TAILQ_HEAD(, kore_route) routes; TAILQ_HEAD(, http_redirect) redirects; #endif TAILQ_ENTRY(kore_domain) list; @@ -363,15 +390,6 @@ LIST_HEAD(kore_server_list, kore_server); #define KORE_PARAMS_QUERY_STRING 0x0001 -struct kore_handler_params { - char *name; - int flags; - u_int8_t method; - struct kore_validator *validator; - - TAILQ_ENTRY(kore_handler_params) list; -}; - #define KORE_AUTH_TYPE_COOKIE 1 #define KORE_AUTH_TYPE_HEADER 2 #define KORE_AUTH_TYPE_REQUEST 3 @@ -420,23 +438,6 @@ struct kore_module { TAILQ_ENTRY(kore_module) list; }; -#if !defined(KORE_NO_HTTP) - -struct kore_module_handle { - char *path; - char *func; - int type; - int errors; - regex_t rctx; - struct kore_domain *dom; - struct kore_runtime_call *rcall; - struct kore_auth *auth; - int methods; - TAILQ_HEAD(, kore_handler_params) params; - TAILQ_ENTRY(kore_module_handle) list; -}; -#endif - /* * The workers get a 128KB log buffer per worker, and parent will fetch their * logs when it reached at least 75% of that or if its been > 1 second since @@ -471,7 +472,7 @@ struct kore_worker { u_int8_t has_lock; int restarted; u_int64_t time_locked; - struct kore_module_handle *active_hdlr; + struct kore_route *active_route; struct kore_privsep *ps; /* Used by the workers to store accesslogs. */ @@ -959,12 +960,16 @@ int kore_domain_attach(struct kore_domain *, struct kore_server *); void kore_domain_tlsinit(struct kore_domain *, int, const void *, size_t); void kore_domain_crl_add(struct kore_domain *, const void *, size_t); + #if !defined(KORE_NO_HTTP) -int kore_module_handler_new(struct kore_domain *, const char *, - const char *, const char *, int); -void kore_module_handler_free(struct kore_module_handle *); -int kore_module_handler_find(struct http_request *, - struct kore_domain *, int, struct kore_module_handle **); +void kore_route_reload(void); +void kore_route_free(struct kore_route *); +void kore_route_callback(struct kore_route *, const char *); + +struct kore_route *kore_route_create(struct kore_domain *, + const char *, int); +int kore_route_lookup(struct http_request *, + struct kore_domain *, int, struct kore_route **); #endif struct kore_runtime_call *kore_runtime_getcall(const char *); diff --git a/src/accesslog.c b/src/accesslog.c index 287fa5d..84af979 100644 --- a/src/accesslog.c +++ b/src/accesslog.c @@ -195,7 +195,7 @@ kore_accesslog(struct http_request *req) } hdr->loglen = len; - hdr->domain = req->hdlr->dom->id; + hdr->domain = req->rt->dom->id; worker->lb.offset += (size_t)len; break; diff --git a/src/cli.c b/src/cli.c index f102146..5ed97b1 100644 --- a/src/cli.c +++ b/src/cli.c @@ -293,7 +293,10 @@ static const char *config_data = "\tcertfile\tcert/server.pem\n" "\tcertkey\t\tcert/key.pem\n" "\n" - "\troute\t/\tpage\n" + "\troute / {\n" + "\t\thandler page\n" + "\t}\n" + "\n" "}\n"; static const char *build_data = diff --git a/src/config.c b/src/config.c index 3b66cf6..9d1d421 100644 --- a/src/config.c +++ b/src/config.c @@ -109,12 +109,13 @@ static int configure_client_verify_depth(char *); #if !defined(KORE_NO_HTTP) static int configure_route(char *); +static int configure_route_handler(char *); +static int configure_route_methods(char *); static int configure_filemap(char *); static int configure_return(char *); static int configure_redirect(char *); static int configure_static_handler(char *); static int configure_dynamic_handler(char *); -static int configure_restrict(char *); static int configure_accesslog(char *); static int configure_http_header_max(char *); static int configure_http_header_timeout(char *); @@ -132,7 +133,6 @@ static int configure_http_body_disk_path(char *); static int configure_http_server_version(char *); static int configure_http_pretty_error(char *); static int configure_validator(char *); -static int configure_params(char *); static int configure_validate(char *); static int configure_authentication(char *); static int configure_authentication_uri(char *); @@ -189,16 +189,16 @@ static struct { { "client_verify", configure_client_verify }, { "client_verify_depth", configure_client_verify_depth }, #if !defined(KORE_NO_HTTP) - { "route", configure_route}, + { "route", configure_route }, + { "handler", configure_route_handler }, + { "methods", configure_route_methods }, { "filemap", configure_filemap }, { "redirect", configure_redirect }, { "return", configure_return }, { "static", configure_static_handler }, { "dynamic", configure_dynamic_handler }, { "accesslog", configure_accesslog }, - { "restrict", configure_restrict }, { "validator", configure_validator }, - { "params", configure_params }, { "validate", configure_validate }, { "authentication", configure_authentication }, { "authentication_uri", configure_authentication_uri }, @@ -282,10 +282,8 @@ char *config_file = NULL; #endif #if !defined(KORE_NO_HTTP) -static u_int8_t current_method = 0; -static int current_flags = 0; static struct kore_auth *current_auth = NULL; -static struct kore_module_handle *current_handler = NULL; +static struct kore_route *current_route = NULL; #endif extern const char *__progname; @@ -341,10 +339,8 @@ kore_parse_config(void) fatal("getcwd: %s", errno_s); worker_privsep.root = kore_strdup(path); - if (!kore_quiet) { - kore_log(LOG_NOTICE, "privsep: no root path set, " - "using working directory"); - } + if (!kore_quiet) + kore_log(LOG_NOTICE, "privsep: no root path set"); } if (worker_privsep.runas == NULL) { @@ -352,26 +348,24 @@ kore_parse_config(void) fatal("getpwuid: %s", errno_s); worker_privsep.runas = kore_strdup(pwd->pw_name); - if (!kore_quiet) { - kore_log(LOG_NOTICE, "privsep: no runas user set, " - "using current user %s", worker_privsep.runas); - } + if (!kore_quiet) + kore_log(LOG_NOTICE, "privsep: no runas user set"); endpwent(); } configure_check_var(&keymgr_privsep.runas, worker_privsep.runas, - "privsep: no keymgr runas set, using 'privsep.worker.runas`"); + "privsep: no keymgr runas set"); #if defined(KORE_USE_ACME) configure_check_var(&acme_privsep.runas, worker_privsep.runas, - "privsep: no acme runas set, using 'privsep.worker.runas`"); + "privsep: no acme runas set"); #endif configure_check_var(&keymgr_privsep.root, worker_privsep.root, - "privsep: no keymgr root set, using 'privsep.worker.root`"); + "privsep: no keymgr root set"); #if defined(KORE_USE_ACME) configure_check_var(&acme_privsep.root, worker_privsep.root, - "privsep: no acme root set, using 'privsep.worker.root`"); + "privsep: no acme root set"); #endif if (skip_chroot) { @@ -426,11 +420,9 @@ kore_parse_config_file(FILE *fp) } #if !defined(KORE_NO_HTTP) - if (!strcmp(p, "}") && current_handler != NULL) { + if (!strcmp(p, "}") && current_route != NULL) { lineno++; - current_flags = 0; - current_method = 0; - current_handler = NULL; + current_route = NULL; continue; } @@ -476,7 +468,8 @@ kore_parse_config_file(FILE *fp) } if ((t = strchr(p, ' ')) == NULL) { - printf("ignoring \"%s\" on line %d\n", p, lineno++); + kore_log(LOG_NOTICE, + "ignoring \"%s\" on line %d", p, lineno++); continue; } @@ -486,7 +479,8 @@ kore_parse_config_file(FILE *fp) t = kore_text_trim(t, strlen(t)); if (strlen(p) == 0 || strlen(t) == 0) { - printf("ignoring \"%s\" on line %d\n", p, lineno++); + kore_log(LOG_NOTICE, + "ignoring \"%s\" on line %d", p, lineno++); continue; } @@ -517,8 +511,10 @@ kore_parse_config_file(FILE *fp) } } - if (config_settings[i].name == NULL) - printf("ignoring \"%s\" on line %d\n", p, lineno); + if (config_settings[i].name == NULL) { + kore_log(LOG_NOTICE, + "ignoring \"%s\" on line %d", p, lineno); + } lineno++; } @@ -592,24 +588,24 @@ configure_server(char *options) char *argv[3]; if (current_server != NULL) { - printf("nested server contexts are not allowed\n"); + kore_log(LOG_ERR, "nested server contexts are not allowed"); return (KORE_RESULT_ERROR); } kore_split_string(options, " ", argv, 3); if (argv[0] == NULL || argv[1] == NULL) { - printf("invalid server context\n"); + kore_log(LOG_ERR, "server context invalid"); return (KORE_RESULT_ERROR); } if (strcmp(argv[1], "{")) { - printf("server context not opened correctly\n"); + kore_log(LOG_ERR, "server context not opened correctly"); return (KORE_RESULT_ERROR); } if ((srv = kore_server_lookup(argv[0])) != NULL) { - printf("a server with name '%s' already exists\n", srv->name); + kore_log(LOG_ERR, "server with name '%s' exists", srv->name); return (KORE_RESULT_ERROR); } @@ -622,7 +618,7 @@ static int configure_tls(char *yesno) { if (current_server == NULL) { - printf("bind directive not inside a server context\n"); + kore_log(LOG_ERR, "bind keyword not inside a server context"); return (KORE_RESULT_ERROR); } @@ -631,7 +627,7 @@ configure_tls(char *yesno) } else if (!strcmp(yesno, "yes")) { current_server->tls = 1; } else { - printf("invalid '%s' for yes|no tls option\n", yesno); + kore_log(LOG_ERR, "invalid '%s' for yes|no tls option", yesno); return (KORE_RESULT_ERROR); } @@ -643,12 +639,13 @@ static int configure_acme(char *yesno) { if (current_domain == NULL) { - printf("acme directive not inside a domain context\n"); + kore_log(LOG_ERR, "acme keyword not inside a domain context"); return (KORE_RESULT_ERROR); } if (strchr(current_domain->domain, '*')) { - printf("wildcards not supported due to lack of dns-01\n"); + kore_log(LOG_ERR, + "wildcards not supported due to lack of dns-01"); return (KORE_RESULT_ERROR); } @@ -665,7 +662,7 @@ configure_acme(char *yesno) ¤t_domain->certkey, ¤t_domain->certfile); acme_domains++; } else { - printf("invalid '%s' for yes|no acme option\n", yesno); + kore_log(LOG_ERR, "invalid '%s' for yes|no acme option", yesno); return (KORE_RESULT_ERROR); } @@ -698,7 +695,7 @@ configure_bind(char *options) char *argv[4]; if (current_server == NULL) { - printf("bind directive not inside a server context\n"); + kore_log(LOG_ERR, "bind keyword not inside a server context"); return (KORE_RESULT_ERROR); } @@ -715,7 +712,8 @@ configure_bind_unix(char *options) char *argv[3]; if (current_server == NULL) { - printf("bind_unix directive not inside a server context\n"); + kore_log(LOG_ERR, + "bind_unix keyword not inside a server context"); return (KORE_RESULT_ERROR); } @@ -800,7 +798,9 @@ configure_tls_version(char *version) } else if (!strcmp(version, "both")) { tls_version = KORE_TLS_VERSION_BOTH; } else { - printf("unknown value for tls_version: %s\n", version); + kore_log(LOG_ERR, + "unknown value for tls_version: %s (use 1.3, 1.2, both)", + version); return (KORE_RESULT_ERROR); } @@ -811,7 +811,7 @@ static int configure_tls_cipher(char *cipherlist) { if (strcmp(kore_tls_cipher_list, KORE_DEFAULT_CIPHER_LIST)) { - printf("tls_cipher specified twice\n"); + kore_log(LOG_ERR, "tls_cipher specified twice"); return (KORE_RESULT_ERROR); } @@ -825,12 +825,12 @@ configure_tls_dhparam(char *path) BIO *bio; if (tls_dhparam != NULL) { - printf("tls_dhparam specified twice\n"); + kore_log(LOG_ERR, "tls_dhparam specified twice"); return (KORE_RESULT_ERROR); } if ((bio = BIO_new_file(path, "r")) == NULL) { - printf("%s did not exist\n", path); + kore_log(LOG_ERR, "tls_dhparam file '%s' not accessible", path); return (KORE_RESULT_ERROR); } @@ -838,7 +838,7 @@ configure_tls_dhparam(char *path) BIO_free(bio); if (tls_dhparam == NULL) { - printf("PEM_read_bio_DHparams(): %s\n", ssl_errno_s); + kore_log(LOG_ERR, "PEM_read_bio_DHparams(): %s", ssl_errno_s); return (KORE_RESULT_ERROR); } @@ -851,13 +851,14 @@ configure_client_verify_depth(char *value) int err, depth; if (current_domain == NULL) { - printf("client_verify_depth not specified in domain context\n"); + kore_log(LOG_ERR, + "client_verify_depth keyword not in domain context"); return (KORE_RESULT_ERROR); } depth = kore_strtonum(value, 10, 0, INT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad client_verify_depth value: %s\n", value); + kore_log(LOG_ERR, "bad client_verify_depth value: %s", value); return (KORE_RESULT_ERROR); } @@ -872,18 +873,19 @@ configure_client_verify(char *options) char *argv[3]; if (current_domain == NULL) { - printf("client_verify not specified in domain context\n"); + kore_log(LOG_ERR, + "client_verify keyword not in domain context"); return (KORE_RESULT_ERROR); } kore_split_string(options, " ", argv, 3); if (argv[0] == NULL) { - printf("client_verify is missing a parameter\n"); + kore_log(LOG_ERR, "client_verify is missing a parameter"); return (KORE_RESULT_ERROR); } if (current_domain->cafile != NULL) { - printf("client_verify already set for %s\n", + kore_log(LOG_ERR, "client_verify already set for '%s'", current_domain->domain); return (KORE_RESULT_ERROR); } @@ -910,7 +912,8 @@ static int configure_certfile(char *path) { if (current_domain == NULL) { - printf("certfile not specified in domain context\n"); + kore_log(LOG_ERR, + "certfile keyword not specified in domain context"); return (KORE_RESULT_ERROR); } @@ -923,7 +926,8 @@ static int configure_certkey(char *path) { if (current_domain == NULL) { - printf("certkey not specified in domain text\n"); + kore_log(LOG_ERR, + "certkey keyword not specified in domain context"); return (KORE_RESULT_ERROR); } @@ -938,19 +942,19 @@ configure_privsep(char *options) char *argv[3]; if (current_privsep != NULL) { - printf("nested privsep contexts are not allowed\n"); + kore_log(LOG_ERR, "nested privsep contexts are not allowed"); return (KORE_RESULT_ERROR); } kore_split_string(options, " ", argv, 3); if (argv[0] == NULL || argv[1] == NULL) { - printf("invalid privsep context\n"); + kore_log(LOG_ERR, "invalid privsep context"); return (KORE_RESULT_ERROR); } if (strcmp(argv[1], "{")) { - printf("privsep context not opened correctly\n"); + kore_log(LOG_ERR, "privsep context not opened correctly"); return (KORE_RESULT_ERROR); } @@ -963,7 +967,7 @@ configure_privsep(char *options) current_privsep = &acme_privsep; #endif } else { - printf("unknown privsep context: %s\n", argv[0]); + kore_log(LOG_ERR, "unknown privsep context: %s", argv[0]); return (KORE_RESULT_ERROR); } @@ -974,7 +978,7 @@ static int configure_privsep_runas(char *user) { if (current_privsep == NULL) { - printf("runas not specified in privsep context\n"); + kore_log(LOG_ERR, "runas keyword not in privsep context"); return (KORE_RESULT_ERROR); } @@ -990,7 +994,7 @@ static int configure_privsep_root(char *root) { if (current_privsep == NULL) { - printf("root not specified in privsep context\n"); + kore_log(LOG_ERR, "root keyword not in privsep context"); return (KORE_RESULT_ERROR); } @@ -1006,14 +1010,14 @@ static int configure_privsep_skip(char *option) { if (current_privsep == NULL) { - printf("skip not specified in privsep context\n"); + kore_log(LOG_ERR, "skip keyword not in privsep context"); return (KORE_RESULT_ERROR); } if (!strcmp(option, "chroot")) { current_privsep->skip_chroot = 1; } else { - printf("unknown skip option '%s'\n", option); + kore_log(LOG_ERR, "unknown skip option '%s'", option); return (KORE_RESULT_ERROR); } @@ -1026,24 +1030,24 @@ configure_domain(char *options) char *argv[3]; if (current_domain != NULL) { - printf("nested domain contexts are not allowed\n"); + kore_log(LOG_ERR, "nested domain contexts are not allowed"); return (KORE_RESULT_ERROR); } kore_split_string(options, " ", argv, 3); if (argv[0] == NULL || argv[1] == NULL) { - printf("invalid domain context\n"); + kore_log(LOG_ERR, "invalid domain context"); return (KORE_RESULT_ERROR); } if (strcmp(argv[1], "{")) { - printf("domain context not opened correctly\n"); + kore_log(LOG_ERR, "domain context not opened correctly"); return (KORE_RESULT_ERROR); } if (strlen(argv[0]) >= KORE_DOMAINNAME_LEN - 1) { - printf("domain name '%s' too long\n", argv[0]); + kore_log(LOG_ERR, "domain name '%s' too long", argv[0]); return (KORE_RESULT_ERROR); } @@ -1058,23 +1062,23 @@ configure_attach(char *name) struct kore_server *srv; if (current_domain == NULL) { - printf("attach not specified in domain context\n"); + kore_log(LOG_ERR, "attach keyword not in domain context"); return (KORE_RESULT_ERROR); } if (current_domain->server != NULL) { - printf("domain '%s' already attached to server\n", + kore_log(LOG_ERR, "domain '%s' already attached to server", current_domain->domain); return (KORE_RESULT_ERROR); } if ((srv = kore_server_lookup(name)) == NULL) { - printf("server '%s' does not exist\n", name); + kore_log(LOG_ERR, "server '%s' does not exist", name); return (KORE_RESULT_ERROR); } if (!kore_domain_attach(current_domain, srv)) { - printf("failed to attach '%s' to '%s'\n", + kore_log(LOG_ERR, "failed to attach '%s' to '%s'", current_domain->domain, name); return (KORE_RESULT_ERROR); } @@ -1100,18 +1104,24 @@ configure_dynamic_handler(char *options) static int configure_route(char *options) { - int type; - char *argv[4]; + struct kore_route *rt; + int type; + char *argv[4]; if (current_domain == NULL) { - printf("route not specified in domain context\n"); + kore_log(LOG_ERR, "route keyword not in domain context"); + return (KORE_RESULT_ERROR); + } + + if (current_route != NULL) { + kore_log(LOG_ERR, "nested route contexts not allowed"); return (KORE_RESULT_ERROR); } kore_split_string(options, " ", argv, 4); - if (argv[0] == NULL || argv[1] == NULL) { - printf("missing parameters for route \n"); + if (argv[1] == NULL || strcmp(argv[1], "{")) { + kore_log(LOG_ERR, "invalid route context"); return (KORE_RESULT_ERROR); } @@ -1120,12 +1130,72 @@ configure_route(char *options) else type = HANDLER_TYPE_DYNAMIC; - if (!kore_module_handler_new(current_domain, - argv[0], argv[1], argv[2], type)) { - printf("cannot create route for %s\n", argv[0]); + if ((rt = kore_route_create(current_domain, argv[0], type)) == NULL) { + kore_log(LOG_ERR, + "failed to create route handler for '%s'", argv[0]); return (KORE_RESULT_ERROR); } + current_route = rt; + + return (KORE_RESULT_OK); +} + +static int +configure_route_handler(char *name) +{ + if (current_route == NULL) { + kore_log(LOG_ERR, + "handler keyword not inside of route context"); + return (KORE_RESULT_ERROR); + } + + kore_route_callback(current_route, name); + + return (KORE_RESULT_OK); +} + +static int +configure_route_methods(char *options) +{ + int i, cnt; + char *argv[10]; + + if (current_route == NULL) { + kore_log(LOG_ERR, + "methods keyword not inside of route context"); + return (KORE_RESULT_ERROR); + } + + cnt = kore_split_string(options, " ", argv, 10); + if (cnt < 1) { + kore_log(LOG_ERR, + "bad methods option '%s', missing methods", options); + return (KORE_RESULT_ERROR); + } + + current_route->methods = 0; + + for (i = 0; i < cnt; i++) { + if (!strcasecmp(argv[i], "post")) { + current_route->methods |= HTTP_METHOD_POST; + } else if (!strcasecmp(argv[i], "get")) { + current_route->methods |= HTTP_METHOD_GET; + } else if (!strcasecmp(argv[i], "put")) { + current_route->methods |= HTTP_METHOD_PUT; + } else if (!strcasecmp(argv[i], "delete")) { + current_route->methods |= HTTP_METHOD_DELETE; + } else if (!strcasecmp(argv[i], "head")) { + current_route->methods |= HTTP_METHOD_HEAD; + } else if (!strcasecmp(argv[i], "patch")) { + current_route->methods |= HTTP_METHOD_PATCH; + } else { + kore_log(LOG_ERR, "unknown method: %s in method for %s", + argv[i], current_route->path); + return (KORE_RESULT_ERROR); + } + } + return (KORE_RESULT_OK); } @@ -1136,24 +1206,25 @@ configure_return(char *options) int elm, status, err; if (current_domain == NULL) { - printf("return outside of domain context\n"); + kore_log(LOG_ERR, "return keyword not in domain context"); return (KORE_RESULT_ERROR); } elm = kore_split_string(options, " ", argv, 3); if (elm != 2) { - printf("missing parameters for return\n"); + kore_log(LOG_ERR, "missing parameters for return"); return (KORE_RESULT_ERROR); } status = kore_strtonum(argv[1], 10, 400, 600, &err); if (err != KORE_RESULT_OK) { - printf("invalid status code on return (%s)\n", argv[1]); + kore_log(LOG_ERR, + "invalid status code on return (%s)", argv[1]); return (KORE_RESULT_ERROR); } if (!http_redirect_add(current_domain, argv[0], status, NULL)) { - printf("invalid regex on return path\n"); + kore_log(LOG_ERR, "invalid regex on return path"); return (KORE_RESULT_ERROR); } @@ -1167,24 +1238,25 @@ configure_redirect(char *options) int elm, status, err; if (current_domain == NULL) { - printf("redirect outside of domain context\n"); + kore_log(LOG_ERR, "redirect keyword not in domain context"); return (KORE_RESULT_ERROR); } elm = kore_split_string(options, " ", argv, 4); if (elm != 3) { - printf("missing parameters for redirect\n"); + kore_log(LOG_ERR, "missing parameters for redirect"); return (KORE_RESULT_ERROR); } status = kore_strtonum(argv[1], 10, 300, 399, &err); if (err != KORE_RESULT_OK) { - printf("invalid status code on redirect (%s)\n", argv[1]); + kore_log(LOG_ERR, + "invalid status code on redirect (%s)", argv[1]); return (KORE_RESULT_ERROR); } if (!http_redirect_add(current_domain, argv[0], status, argv[2])) { - printf("invalid regex on redirect path\n"); + kore_log(LOG_ERR, "invalid regex on redirect path"); return (KORE_RESULT_ERROR); } @@ -1197,19 +1269,19 @@ configure_filemap(char *options) char *argv[3]; if (current_domain == NULL) { - printf("filemap outside of domain context\n"); + kore_log(LOG_ERR, "filemap keyword not in domain context"); return (KORE_RESULT_ERROR); } kore_split_string(options, " ", argv, 3); if (argv[0] == NULL || argv[1] == NULL) { - printf("missing parameters for filemap\n"); + kore_log(LOG_ERR, "missing parameters for filemap"); return (KORE_RESULT_ERROR); } if (!kore_filemap_create(current_domain, argv[1], argv[0])) { - printf("cannot create filemap for %s\n", argv[1]); + kore_log(LOG_ERR, "cannot create filemap for %s", argv[1]); return (KORE_RESULT_ERROR); } @@ -1225,7 +1297,7 @@ configure_accesslog(char *path) } if (current_domain->accesslog != -1) { - printf("domain %s already has an open accesslog\n", + kore_log(LOG_ERR, "domain '%s' already has an open accesslog", current_domain->domain); return (KORE_RESULT_ERROR); } @@ -1234,67 +1306,13 @@ configure_accesslog(char *path) O_CREAT | O_APPEND | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (current_domain->accesslog == -1) { - printf("accesslog open(%s): %s\n", path, errno_s); + kore_log(LOG_ERR, "accesslog open(%s): %s", path, errno_s); return (KORE_RESULT_ERROR); } return (KORE_RESULT_OK); } -static int -configure_restrict(char *options) -{ - struct kore_module_handle *hdlr; - int i, cnt; - char *argv[10]; - - if (current_domain == NULL) { - printf("restrict not used in domain context\n"); - return (KORE_RESULT_ERROR); - } - - cnt = kore_split_string(options, " ", argv, 10); - if (cnt < 2) { - printf("bad restrict option '%s', missing methods\n", options); - return (KORE_RESULT_ERROR); - } - - hdlr = NULL; - TAILQ_FOREACH(hdlr, &(current_domain->handlers), list) { - if (!strcmp(hdlr->path, argv[0])) - break; - } - - if (hdlr == NULL) { - printf("bad restrict option handler '%s' not found\n", argv[0]); - return (KORE_RESULT_ERROR); - } - - hdlr->methods = 0; - - for (i = 1; i < cnt; i++) { - if (!strcasecmp(argv[i], "post")) { - hdlr->methods |= HTTP_METHOD_POST; - } else if (!strcasecmp(argv[i], "get")) { - hdlr->methods |= HTTP_METHOD_GET; - } else if (!strcasecmp(argv[i], "put")) { - hdlr->methods |= HTTP_METHOD_PUT; - } else if (!strcasecmp(argv[i], "delete")) { - hdlr->methods |= HTTP_METHOD_DELETE; - } else if (!strcasecmp(argv[i], "head")) { - hdlr->methods |= HTTP_METHOD_HEAD; - } else if (!strcasecmp(argv[i], "patch")) { - hdlr->methods |= HTTP_METHOD_PATCH; - } else { - printf("unknown method: %s in restrict for %s\n", - argv[i], argv[0]); - return (KORE_RESULT_ERROR); - } - } - - return (KORE_RESULT_OK); -} - static int configure_filemap_ext(char *ext) { @@ -1321,7 +1339,7 @@ configure_http_media_type(char *type) extensions = strchr(type, ' '); if (extensions == NULL) { - printf("bad http_media_type value: %s\n", type); + kore_log(LOG_ERR, "bad http_media_type value '%s'", type); return (KORE_RESULT_ERROR); } @@ -1330,13 +1348,14 @@ configure_http_media_type(char *type) kore_split_string(extensions, " \t", ext, 10); for (i = 0; ext[i] != NULL; i++) { if (!http_media_register(ext[i], type)) { - printf("duplicate extension found: %s\n", ext[i]); + kore_log(LOG_ERR, + "duplicate extension found '%s'", ext[i]); return (KORE_RESULT_ERROR); } } if (i == 0) { - printf("missing extensions in: %s\n", type); + kore_log(LOG_ERR, "missing extensions in '%s'", type); return (KORE_RESULT_ERROR); } @@ -1350,7 +1369,7 @@ configure_http_header_max(char *option) http_header_max = kore_strtonum(option, 10, 1, 65535, &err); if (err != KORE_RESULT_OK) { - printf("bad http_header_max value: %s\n", option); + kore_log(LOG_ERR, "bad http_header_max value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1364,7 +1383,7 @@ configure_http_header_timeout(char *option) http_header_timeout = kore_strtonum(option, 10, 1, 65535, &err); if (err != KORE_RESULT_OK) { - printf("bad http_header_timeout value: %s\n", option); + kore_log(LOG_ERR, "bad http_header_timeout value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1378,7 +1397,7 @@ configure_http_body_max(char *option) http_body_max = kore_strtonum(option, 10, 0, LONG_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad http_body_max value: %s\n", option); + kore_log(LOG_ERR, "bad http_body_max value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1392,7 +1411,7 @@ configure_http_body_timeout(char *option) http_body_timeout = kore_strtonum(option, 10, 1, 65535, &err); if (err != KORE_RESULT_OK) { - printf("bad http_body_timeout value: %s\n", option); + kore_log(LOG_ERR, "bad http_body_timeout value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1406,7 +1425,8 @@ configure_http_body_disk_offload(char *option) http_body_disk_offload = kore_strtonum(option, 10, 0, LONG_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad http_body_disk_offload value: %s\n", option); + kore_log(LOG_ERR, + "bad http_body_disk_offload value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1439,8 +1459,8 @@ configure_http_pretty_error(char *yesno) } else if (!strcmp(yesno, "yes")) { http_pretty_error = 1; } else { - printf("invalid '%s' for yes|no http_pretty_error option\n", - yesno); + kore_log(LOG_ERR, + "invalid '%s' for yes|no http_pretty_error option", yesno); return (KORE_RESULT_ERROR); } @@ -1454,7 +1474,7 @@ configure_http_hsts_enable(char *option) http_hsts_enable = kore_strtonum(option, 10, 0, LONG_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad http_hsts_enable value: %s\n", option); + kore_log(LOG_ERR, "bad http_hsts_enable value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1468,7 +1488,8 @@ configure_http_keepalive_time(char *option) http_keepalive_time = kore_strtonum(option, 10, 0, USHRT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad http_keepalive_time value: %s\n", option); + kore_log(LOG_ERR, + "bad http_keepalive_time value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1482,7 +1503,7 @@ configure_http_request_ms(char *option) http_request_ms = kore_strtonum(option, 10, 0, UINT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad http_request_ms value: %s\n", option); + kore_log(LOG_ERR, "bad http_request_ms value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1496,7 +1517,7 @@ configure_http_request_limit(char *option) http_request_limit = kore_strtonum(option, 10, 0, UINT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad http_request_limit value: %s\n", option); + kore_log(LOG_ERR, "bad http_request_limit value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1510,14 +1531,14 @@ configure_validator(char *name) char *tname, *value; if ((tname = strchr(name, ' ')) == NULL) { - printf("missing validator name\n"); + kore_log(LOG_ERR, "missing validator name"); return (KORE_RESULT_ERROR); } *(tname)++ = '\0'; tname = kore_text_trim(tname, strlen(tname)); if ((value = strchr(tname, ' ')) == NULL) { - printf("missing validator value\n"); + kore_log(LOG_ERR, "missing validator value"); return (KORE_RESULT_ERROR); } @@ -1529,12 +1550,13 @@ configure_validator(char *name) } else if (!strcmp(tname, "function")) { type = KORE_VALIDATOR_TYPE_FUNCTION; } else { - printf("bad type for validator %s\n", tname); + kore_log(LOG_ERR, + "bad type '%s' for validator '%s'", tname, name); return (KORE_RESULT_ERROR); } if (!kore_validator_add(name, type, value)) { - printf("bad validator specified: %s\n", tname); + kore_log(LOG_ERR, "bad validator specified for '%s'", name); return (KORE_RESULT_ERROR); } @@ -1542,101 +1564,75 @@ configure_validator(char *name) } static int -configure_params(char *options) +configure_validate(char *options) { - struct kore_module_handle *hdlr; - char *method, *argv[3]; + struct kore_validator *val; + struct kore_route_params *param; + char *method, *argv[4]; + int flags, http_method; - if (current_domain == NULL) { - printf("params not used in domain context\n"); + if (kore_split_string(options, " ", argv, 4) != 3) { + kore_log(LOG_ERR, + "validate keyword needs 3 args: method param validator"); return (KORE_RESULT_ERROR); } - if (current_handler != NULL) { - printf("previous params block not closed\n"); - return (KORE_RESULT_ERROR); - } - - kore_split_string(options, " ", argv, 3); - if (argv[1] == NULL) - return (KORE_RESULT_ERROR); + flags = 0; if ((method = strchr(argv[0], ':')) != NULL) { *(method)++ = '\0'; if (!strcasecmp(argv[0], "qs")) { - current_flags = KORE_PARAMS_QUERY_STRING; + flags = KORE_PARAMS_QUERY_STRING; } else { - printf("unknown prefix '%s' for '%s'\n", - argv[0], argv[1]); + kore_log(LOG_ERR, + "unknown validate method prefix '%s' for '%s'", + argv[0], current_route->path); return (KORE_RESULT_ERROR); } } else { method = argv[0]; } + if ((val = kore_validator_lookup(argv[2])) == NULL) { + kore_log(LOG_ERR, "unknown validator '%s'", argv[2]); + return (KORE_RESULT_ERROR); + } + if (!strcasecmp(method, "post")) { - current_method = HTTP_METHOD_POST; + http_method = HTTP_METHOD_POST; } else if (!strcasecmp(method, "get")) { - current_method = HTTP_METHOD_GET; + http_method = HTTP_METHOD_GET; /* Let params get /foo {} imply qs:get automatically. */ - current_flags |= KORE_PARAMS_QUERY_STRING; + flags |= KORE_PARAMS_QUERY_STRING; } else if (!strcasecmp(method, "put")) { - current_method = HTTP_METHOD_PUT; + http_method = HTTP_METHOD_PUT; } else if (!strcasecmp(method, "delete")) { - current_method = HTTP_METHOD_DELETE; + http_method = HTTP_METHOD_DELETE; } else if (!strcasecmp(method, "head")) { - current_method = HTTP_METHOD_HEAD; + http_method = HTTP_METHOD_HEAD; } else if (!strcasecmp(method, "patch")) { - current_method = HTTP_METHOD_PATCH; + http_method = HTTP_METHOD_PATCH; } else { - printf("unknown method: %s in params block for %s\n", - method, argv[1]); + kore_log(LOG_ERR, "unknown method: %s in validator for %s", + method, current_route->path); return (KORE_RESULT_ERROR); } - /* - * Find the handler ourselves, otherwise the regex is applied - * in case of a dynamic page. - */ - TAILQ_FOREACH(hdlr, &(current_domain->handlers), list) { - if (!strcmp(hdlr->path, argv[1])) { - current_handler = hdlr; - return (KORE_RESULT_OK); - } - } - - printf("params for unknown page handler: %s\n", argv[1]); - return (KORE_RESULT_ERROR); -} - -static int -configure_validate(char *options) -{ - struct kore_handler_params *p; - struct kore_validator *val; - char *argv[3]; - - if (current_handler == NULL) { - printf("validate not used in domain context\n"); + if (!(current_route->methods & http_method)) { + kore_log(LOG_ERR, "method '%s' not enabled for route '%s'", + method, current_route->path); return (KORE_RESULT_ERROR); } - kore_split_string(options, " ", argv, 3); - if (argv[1] == NULL) - return (KORE_RESULT_ERROR); + param = kore_calloc(1, sizeof(*param)); - if ((val = kore_validator_lookup(argv[1])) == NULL) { - printf("unknown validator %s for %s\n", argv[1], argv[0]); - return (KORE_RESULT_ERROR); - } + param->flags = flags; + param->validator = val; + param->method = http_method; + param->name = kore_strdup(argv[1]); - p = kore_malloc(sizeof(*p)); - p->validator = val; - p->flags = current_flags; - p->method = current_method; - p->name = kore_strdup(argv[0]); + TAILQ_INSERT_TAIL(¤t_route->params, param, list); - TAILQ_INSERT_TAIL(&(current_handler->params), p, list); return (KORE_RESULT_OK); } @@ -1646,18 +1642,18 @@ configure_authentication(char *options) char *argv[3]; if (current_auth != NULL) { - printf("previous authentication block not closed\n"); + kore_log(LOG_ERR, "previous authentication block not closed"); return (KORE_RESULT_ERROR); } kore_split_string(options, " ", argv, 3); if (argv[1] == NULL) { - printf("missing name for authentication block\n"); + kore_log(LOG_ERR, "missing name for authentication block"); return (KORE_RESULT_ERROR); } if (strcmp(argv[1], "{")) { - printf("missing { for authentication block\n"); + kore_log(LOG_ERR, "missing { for authentication block"); return (KORE_RESULT_ERROR); } @@ -1673,7 +1669,8 @@ static int configure_authentication_type(char *option) { if (current_auth == NULL) { - printf("authentication_type outside authentication context\n"); + kore_log(LOG_ERR, + "authentication_type keyword not in correct context"); return (KORE_RESULT_ERROR); } @@ -1684,7 +1681,7 @@ configure_authentication_type(char *option) } else if (!strcmp(option, "request")) { current_auth->type = KORE_AUTH_TYPE_REQUEST; } else { - printf("unknown authentication type '%s'\n", option); + kore_log(LOG_ERR, "unknown authentication type '%s'", option); return (KORE_RESULT_ERROR); } @@ -1695,7 +1692,8 @@ static int configure_authentication_value(char *option) { if (current_auth == NULL) { - printf("authentication_value outside authentication context\n"); + kore_log(LOG_ERR, + "authentication_value keyword not in correct context"); return (KORE_RESULT_ERROR); } @@ -1712,12 +1710,14 @@ configure_authentication_validator(char *validator) struct kore_validator *val; if (current_auth == NULL) { - printf("authentication_validator outside authentication\n"); + kore_log(LOG_ERR, + "authentication_validator not in correct context"); return (KORE_RESULT_ERROR); } if ((val = kore_validator_lookup(validator)) == NULL) { - printf("authentication validator '%s' not found\n", validator); + kore_log(LOG_ERR, + "authentication validator '%s' not found", validator); return (KORE_RESULT_ERROR); } @@ -1730,7 +1730,8 @@ static int configure_authentication_uri(char *uri) { if (current_auth == NULL) { - printf("authentication_uri outside authentication context\n"); + kore_log(LOG_ERR, + "authentication_uri keyword not in correct context"); return (KORE_RESULT_ERROR); } @@ -1748,7 +1749,8 @@ configure_websocket_maxframe(char *option) kore_websocket_maxframe = kore_strtonum64(option, 1, &err); if (err != KORE_RESULT_OK) { - printf("bad kore_websocket_maxframe value: %s\n", option); + kore_log(LOG_ERR, + "bad kore_websocket_maxframe value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1762,7 +1764,8 @@ configure_websocket_timeout(char *option) kore_websocket_timeout = kore_strtonum64(option, 1, &err); if (err != KORE_RESULT_OK) { - printf("bad kore_websocket_timeout value: %s\n", option); + kore_log(LOG_ERR, + "bad kore_websocket_timeout value '%s'", option); return (KORE_RESULT_ERROR); } @@ -1787,7 +1790,7 @@ configure_workers(char *option) worker_count = kore_strtonum(option, 10, 1, KORE_WORKER_MAX, &err); if (err != KORE_RESULT_OK) { - printf("%s is not a valid worker number\n", option); + kore_log(LOG_ERR, "bad value for worker '%s'", option); return (KORE_RESULT_ERROR); } @@ -1811,7 +1814,8 @@ configure_max_connections(char *option) worker_max_connections = kore_strtonum(option, 10, 1, UINT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad value for worker_max_connections: %s\n", option); + kore_log(LOG_ERR, + "bad value for worker_max_connections '%s'", option); return (KORE_RESULT_ERROR); } @@ -1825,7 +1829,8 @@ configure_rlimit_nofiles(char *option) worker_rlimit_nofiles = kore_strtonum(option, 10, 1, UINT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad value for worker_rlimit_nofiles: %s\n", option); + kore_log(LOG_ERR, + "bad value for worker_rlimit_nofiles '%s'", option); return (KORE_RESULT_ERROR); } @@ -1839,7 +1844,8 @@ configure_accept_threshold(char *option) worker_accept_threshold = kore_strtonum(option, 0, 1, UINT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad value for worker_accept_threshold: %s\n", option); + kore_log(LOG_ERR, + "bad value for worker_accept_threshold '%s'\n", option); return (KORE_RESULT_ERROR); } @@ -1854,7 +1860,8 @@ configure_death_policy(char *option) } else if (!strcmp(option, "terminate")) { worker_policy = KORE_WORKER_POLICY_TERMINATE; } else { - printf("bad value for worker_death_policy: %s\n", option); + kore_log(LOG_ERR, + "bad value for worker_death_policy '%s'\n", option); return (KORE_RESULT_ERROR); } @@ -1868,7 +1875,8 @@ configure_set_affinity(char *option) worker_set_affinity = kore_strtonum(option, 10, 0, 1, &err); if (err != KORE_RESULT_OK) { - printf("bad value for worker_set_affinity: %s\n", option); + kore_log(LOG_ERR, + "bad value for worker_set_affinity '%s'", option); return (KORE_RESULT_ERROR); } @@ -1882,7 +1890,7 @@ configure_socket_backlog(char *option) kore_socket_backlog = kore_strtonum(option, 10, 0, UINT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad socket_backlog value: %s\n", option); + kore_log(LOG_ERR, "bad socket_backlog value: '%s'", option); return (KORE_RESULT_ERROR); } @@ -1897,7 +1905,7 @@ configure_pgsql_conn_max(char *option) pgsql_conn_max = kore_strtonum(option, 10, 0, USHRT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad value for pgsql_conn_max: %s\n", option); + kore_log(LOG_ERR, "bad value for pgsql_conn_max '%s'", option); return (KORE_RESULT_ERROR); } @@ -1911,7 +1919,8 @@ configure_pgsql_queue_limit(char *option) pgsql_queue_limit = kore_strtonum(option, 10, 0, UINT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad value for pgsql_queue_limit: %s\n", option); + kore_log(LOG_ERR, + "bad value for pgsql_queue_limit '%s'", option); return (KORE_RESULT_ERROR); } @@ -1927,7 +1936,7 @@ configure_task_threads(char *option) kore_task_threads = kore_strtonum(option, 10, 0, UCHAR_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad value for task_threads: %s\n", option); + kore_log(LOG_ERR, "bad value for task_threads: '%s'", option); return (KORE_RESULT_ERROR); } @@ -1980,7 +1989,7 @@ configure_curl_recv_max(char *option) kore_curl_recv_max = kore_strtonum64(option, 1, &err); if (err != KORE_RESULT_OK) { - printf("bad curl_recv_max value: %s\n", option); + kore_log(LOG_ERR, "bad curl_recv_max value '%s'\n", option); return (KORE_RESULT_ERROR); } @@ -1994,7 +2003,7 @@ configure_curl_timeout(char *option) kore_curl_timeout = kore_strtonum(option, 10, 0, USHRT_MAX, &err); if (err != KORE_RESULT_OK) { - printf("bad kore_curl_timeout value: %s\n", option); + kore_log(LOG_ERR, "bad kore_curl_timeout value: '%s'", option); return (KORE_RESULT_ERROR); } @@ -2011,8 +2020,8 @@ configure_seccomp_tracing(char *opt) } else if (!strcmp(opt, "no")) { kore_seccomp_tracing = 0; } else { - printf("bad seccomp_tracing value: %s (expected yes|no)\n", - opt); + kore_log(LOG_ERR, + "bad seccomp_tracing value '%s' (expected yes|no)\n", opt); return (KORE_RESULT_ERROR); } diff --git a/src/domain.c b/src/domain.c index 96caf63..78fbb4b 100644 --- a/src/domain.c +++ b/src/domain.c @@ -131,8 +131,8 @@ kore_domain_new(const char *domain) dom->domain = kore_strdup(domain); #if !defined(KORE_NO_HTTP) - TAILQ_INIT(&(dom->handlers)); - TAILQ_INIT(&(dom->redirects)); + TAILQ_INIT(&dom->routes); + TAILQ_INIT(&dom->redirects); #endif if (dom->id < KORE_DOMAIN_CACHE) { @@ -174,8 +174,8 @@ void kore_domain_free(struct kore_domain *dom) { #if !defined(KORE_NO_HTTP) + struct kore_route *rt; struct http_redirect *rdr; - struct kore_module_handle *hdlr; #endif if (dom == NULL) return; @@ -190,20 +190,17 @@ kore_domain_free(struct kore_domain *dom) if (dom->ssl_ctx != NULL) SSL_CTX_free(dom->ssl_ctx); - if (dom->cafile != NULL) - kore_free(dom->cafile); - if (dom->certkey != NULL) - kore_free(dom->certkey); - if (dom->certfile != NULL) - kore_free(dom->certfile); - if (dom->crlfile != NULL) - kore_free(dom->crlfile); + + kore_free(dom->cafile); + kore_free(dom->certkey); + kore_free(dom->certfile); + kore_free(dom->crlfile); #if !defined(KORE_NO_HTTP) /* Drop all handlers associated with this domain */ - while ((hdlr = TAILQ_FIRST(&(dom->handlers))) != NULL) { - TAILQ_REMOVE(&(dom->handlers), hdlr, list); - kore_module_handler_free(hdlr); + while ((rt = TAILQ_FIRST(&dom->routes)) != NULL) { + TAILQ_REMOVE(&dom->routes, rt, list); + kore_route_free(rt); } while ((rdr = TAILQ_FIRST(&(dom->redirects))) != NULL) { diff --git a/src/filemap.c b/src/filemap.c index 89918d8..293a83d 100644 --- a/src/filemap.c +++ b/src/filemap.c @@ -58,7 +58,7 @@ kore_filemap_create(struct kore_domain *dom, const char *path, const char *root) size_t sz; struct stat st; int len; - struct kore_module_handle *hdlr; + struct kore_route *rt; struct filemap_entry *entry; char regex[1024], fpath[PATH_MAX]; @@ -86,20 +86,11 @@ kore_filemap_create(struct kore_domain *dom, const char *path, const char *root) if (len == -1 || (size_t)len >= sizeof(regex)) fatal("kore_filemap_create: buffer too small"); - if (!kore_module_handler_new(dom, regex, "filemap_resolve", - NULL, HANDLER_TYPE_DYNAMIC)) + if ((rt = kore_route_create(dom, regex, HANDLER_TYPE_DYNAMIC)) == NULL) return (KORE_RESULT_ERROR); - hdlr = NULL; - TAILQ_FOREACH(hdlr, &dom->handlers, list) { - if (!strcmp(hdlr->path, regex)) - break; - } - - if (hdlr == NULL) - fatal("couldn't find newly created handler for filemap"); - - hdlr->methods = HTTP_METHOD_GET | HTTP_METHOD_HEAD; + kore_route_callback(rt, "filemap_resolve"); + rt->methods = HTTP_METHOD_GET | HTTP_METHOD_HEAD; entry = kore_calloc(1, sizeof(*entry)); entry->domain = dom; @@ -151,7 +142,7 @@ filemap_resolve(struct http_request *req) best_len = 0; TAILQ_FOREACH(entry, &maps, list) { - if (entry->domain != req->hdlr->dom) + if (entry->domain != req->rt->dom) continue; if (!strncmp(entry->root, req->path, entry->root_len)) { diff --git a/src/http.c b/src/http.c index adb7a43..88e8181 100644 --- a/src/http.c +++ b/src/http.c @@ -345,18 +345,18 @@ http_process_request(struct http_request *req) kore_debug("http_process_request: %p->%p (%s)", req->owner, req, req->path); - if (req->flags & HTTP_REQUEST_DELETE || req->hdlr == NULL) + if (req->flags & HTTP_REQUEST_DELETE || req->rt == NULL) return; req->start = kore_time_ms(); - if (req->hdlr->auth != NULL && !(req->flags & HTTP_REQUEST_AUTHED)) - r = kore_auth_run(req, req->hdlr->auth); + if (req->rt->auth != NULL && !(req->flags & HTTP_REQUEST_AUTHED)) + r = kore_auth_run(req, req->rt->auth); else r = KORE_RESULT_OK; switch (r) { case KORE_RESULT_OK: - r = kore_runtime_http_request(req->hdlr->rcall, req); + r = kore_runtime_http_request(req->rt->rcall, req); break; case KORE_RESULT_RETRY: break; @@ -389,7 +389,7 @@ http_process_request(struct http_request *req) fatal("A page handler returned an unknown result: %d", r); } - if (req->hdlr->dom->accesslog) + if (req->rt->dom->accesslog) kore_accesslog(req); req->flags |= HTTP_REQUEST_DELETE; @@ -2103,7 +2103,7 @@ http_request_new(struct connection *c, const char *host, } /* Checked further down below if we need to 404. */ - exists = kore_module_handler_find(req, dom, m, &req->hdlr); + exists = kore_route_lookup(req, dom, m, &req->rt); TAILQ_INIT(&(req->resp_headers)); TAILQ_INIT(&(req->req_headers)); @@ -2135,7 +2135,7 @@ http_request_new(struct connection *c, const char *host, return (NULL); } - if (req->hdlr == NULL) { + if (req->rt == NULL) { http_request_free(req); http_error_response(c, HTTP_STATUS_METHOD_NOT_ALLOWED); return (NULL); @@ -2339,14 +2339,14 @@ http_argument_add(struct http_request *req, char *name, char *value, int qs, int decode) { struct http_arg *q; - struct kore_handler_params *p; + struct kore_route_params *p; if (decode) { if (!http_argument_urldecode(name)) return; } - TAILQ_FOREACH(p, &(req->hdlr->params), list) { + TAILQ_FOREACH(p, &req->rt->params, list) { if (qs == 1 && !(p->flags & KORE_PARAMS_QUERY_STRING)) continue; if (qs == 0 && (p->flags & KORE_PARAMS_QUERY_STRING)) diff --git a/src/module.c b/src/module.c index ff57a35..47b5ab5 100644 --- a/src/module.c +++ b/src/module.c @@ -135,11 +135,6 @@ kore_module_reload(int cbs) { struct stat st; int ret; -#if !defined(KORE_NO_HTTP) - struct kore_server *srv; - struct kore_domain *dom; - struct kore_module_handle *hdlr; -#endif struct kore_module *module; TAILQ_FOREACH(module, &modules, list) { @@ -183,22 +178,7 @@ kore_module_reload(int cbs) } #if !defined(KORE_NO_HTTP) - LIST_FOREACH(srv, &kore_servers, list) { - TAILQ_FOREACH(dom, &srv->domains, list) { - TAILQ_FOREACH(hdlr, &(dom->handlers), list) { - kore_free(hdlr->rcall); - hdlr->rcall = kore_runtime_getcall(hdlr->func); - if (hdlr->rcall == NULL) { - fatal("no function '%s' found", - hdlr->func); - } - hdlr->errors = 0; - } - } - } -#endif - -#if !defined(KORE_NO_HTTP) + kore_route_reload(); kore_validator_reload(); #endif } @@ -212,112 +192,6 @@ kore_module_loaded(void) return (1); } -#if !defined(KORE_NO_HTTP) -int -kore_module_handler_new(struct kore_domain *dom, const char *path, - const char *func, const char *auth, int type) -{ - struct kore_auth *ap; - struct kore_module_handle *hdlr; - - if (auth != NULL) { - if ((ap = kore_auth_lookup(auth)) == NULL) - fatal("no authentication block '%s' found", auth); - } else { - ap = NULL; - } - - hdlr = kore_malloc(sizeof(*hdlr)); - hdlr->auth = ap; - hdlr->dom = dom; - hdlr->errors = 0; - hdlr->type = type; - hdlr->path = kore_strdup(path); - hdlr->func = kore_strdup(func); - hdlr->methods = HTTP_METHOD_ALL; - - TAILQ_INIT(&(hdlr->params)); - - if ((hdlr->rcall = kore_runtime_getcall(func)) == NULL) { - kore_module_handler_free(hdlr); - kore_log(LOG_ERR, "function '%s' not found", func); - return (KORE_RESULT_ERROR); - } - - if (hdlr->type == HANDLER_TYPE_DYNAMIC) { - if (regcomp(&(hdlr->rctx), hdlr->path, - REG_EXTENDED | REG_NOSUB)) { - kore_module_handler_free(hdlr); - kore_debug("regcomp() on %s failed", path); - return (KORE_RESULT_ERROR); - } - } - - TAILQ_INSERT_TAIL(&(dom->handlers), hdlr, list); - return (KORE_RESULT_OK); -} - -void -kore_module_handler_free(struct kore_module_handle *hdlr) -{ - struct kore_handler_params *param; - - if (hdlr == NULL) - return; - - if (hdlr->func != NULL) - kore_free(hdlr->func); - if (hdlr->path != NULL) - kore_free(hdlr->path); - if (hdlr->type == HANDLER_TYPE_DYNAMIC) - regfree(&(hdlr->rctx)); - - /* Drop all validators associated with this handler */ - while ((param = TAILQ_FIRST(&(hdlr->params))) != NULL) { - TAILQ_REMOVE(&(hdlr->params), param, list); - if (param->name != NULL) - kore_free(param->name); - kore_free(param); - } - - kore_free(hdlr); -} - -int -kore_module_handler_find(struct http_request *req, struct kore_domain *dom, - int method, struct kore_module_handle **out) -{ - struct kore_module_handle *hdlr; - int exists; - - exists = 0; - *out = NULL; - - TAILQ_FOREACH(hdlr, &(dom->handlers), list) { - if (hdlr->type == HANDLER_TYPE_STATIC) { - if (!strcmp(hdlr->path, req->path)) { - if (hdlr->methods & method) { - *out = hdlr; - return (1); - } - exists++; - } - } else { - if (!regexec(&(hdlr->rctx), req->path, - HTTP_CAPTURE_GROUPS, req->cgroups, 0)) { - if (hdlr->methods & method) { - *out = hdlr; - return (1); - } - exists++; - } - } - } - - return (exists); -} -#endif /* !KORE_NO_HTTP */ - void * kore_module_getsym(const char *symbol, struct kore_runtime **runtime) { diff --git a/src/python.c b/src/python.c index 1d9439e..4006842 100644 --- a/src/python.c +++ b/src/python.c @@ -82,13 +82,11 @@ static struct python_coro *python_coro_create(PyObject *, static struct kore_domain *python_route_domain_resolve(struct pyroute *); static int python_route_install(struct pyroute *); -static int python_route_params(PyObject *, - struct kore_module_handle *, const char *, - int, int); +static int python_route_params(PyObject *, struct kore_route *, + const char *, int, int); static int python_route_methods(PyObject *, PyObject *, - struct kore_module_handle *); -static int python_route_auth(PyObject *, - struct kore_module_handle *); + struct kore_route *); +static int python_route_auth(PyObject *, struct kore_route *); static int python_coro_run(struct python_coro *); static void python_coro_wakeup(struct python_coro *); @@ -1250,7 +1248,7 @@ python_runtime_http_request(void *addr, struct http_request *req) callable = (PyObject *)addr; /* starts at 1 to skip the full path. */ - if (req->hdlr->type == HANDLER_TYPE_DYNAMIC) { + if (req->rt->type == HANDLER_TYPE_DYNAMIC) { for (idx = 1; idx < HTTP_CAPTURE_GROUPS - 1; idx++) { if (req->cgroups[idx].rm_so == -1 || req->cgroups[idx].rm_eo == -1) @@ -5211,7 +5209,7 @@ python_route_install(struct pyroute *route) { const char *val; struct kore_domain *domain; - struct kore_module_handle *hdlr, *entry; + struct kore_route *rt, *entry; PyObject *kwargs, *repr, *obj; if ((repr = PyObject_Repr(route->func)) == NULL) { @@ -5221,56 +5219,56 @@ python_route_install(struct pyroute *route) domain = python_route_domain_resolve(route); - hdlr = kore_calloc(1, sizeof(*hdlr)); - hdlr->dom = domain; - hdlr->methods = HTTP_METHOD_ALL; - hdlr->path = kore_strdup(route->path); + rt = kore_calloc(1, sizeof(*rt)); + rt->dom = domain; + rt->methods = HTTP_METHOD_ALL; + rt->path = kore_strdup(route->path); - TAILQ_INIT(&hdlr->params); + TAILQ_INIT(&rt->params); val = PyUnicode_AsUTF8(repr); - hdlr->func = kore_strdup(val); + rt->func = kore_strdup(val); kwargs = route->kwargs; - hdlr->rcall = kore_calloc(1, sizeof(struct kore_runtime_call)); - hdlr->rcall->addr = route->func; - hdlr->rcall->runtime = &kore_python_runtime; - Py_INCREF(hdlr->rcall->addr); + rt->rcall = kore_calloc(1, sizeof(struct kore_runtime_call)); + rt->rcall->addr = route->func; + rt->rcall->runtime = &kore_python_runtime; + Py_INCREF(rt->rcall->addr); if (kwargs != NULL) { if ((obj = PyDict_GetItemString(kwargs, "methods")) != NULL) { - if (!python_route_methods(obj, kwargs, hdlr)) { + if (!python_route_methods(obj, kwargs, rt)) { kore_python_log_error("python_route_install"); - kore_module_handler_free(hdlr); + kore_route_free(rt); return (KORE_RESULT_ERROR); } } if ((obj = PyDict_GetItemString(kwargs, "auth")) != NULL) { - if (!python_route_auth(obj, hdlr)) { + if (!python_route_auth(obj, rt)) { kore_python_log_error("python_route_install"); - kore_module_handler_free(hdlr); + kore_route_free(rt); return (KORE_RESULT_ERROR); } } } - if (hdlr->path[0] == '/') { - hdlr->type = HANDLER_TYPE_STATIC; + if (rt->path[0] == '/') { + rt->type = HANDLER_TYPE_STATIC; } else { - hdlr->type = HANDLER_TYPE_DYNAMIC; - if (regcomp(&hdlr->rctx, hdlr->path, REG_EXTENDED)) - fatal("failed to compile regex for '%s'", hdlr->path); + rt->type = HANDLER_TYPE_DYNAMIC; + if (regcomp(&rt->rctx, rt->path, REG_EXTENDED)) + fatal("failed to compile regex for '%s'", rt->path); } - TAILQ_FOREACH(entry, &domain->handlers, list) { - if (!strcmp(entry->path, hdlr->path) && - (entry->methods & hdlr->methods)) + TAILQ_FOREACH(entry, &domain->routes, list) { + if (!strcmp(entry->path, rt->path) && + (entry->methods & rt->methods)) fatal("duplicate route for '%s'", route->path); } - TAILQ_INSERT_TAIL(&domain->handlers, hdlr, list); + TAILQ_INSERT_TAIL(&domain->routes, rt, list); return (KORE_RESULT_OK); } @@ -5312,8 +5310,7 @@ python_route_domain_resolve(struct pyroute *route) } static int -python_route_methods(PyObject *obj, PyObject *kwargs, - struct kore_module_handle *hdlr) +python_route_methods(PyObject *obj, PyObject *kwargs, struct kore_route *rt) { const char *val; PyObject *item; @@ -5323,7 +5320,7 @@ python_route_methods(PyObject *obj, PyObject *kwargs, if (!PyList_CheckExact(obj)) return (KORE_RESULT_ERROR); - hdlr->methods = 0; + rt->methods = 0; list_len = PyList_Size(obj); for (idx = 0; idx < list_len; idx++) { @@ -5339,14 +5336,14 @@ python_route_methods(PyObject *obj, PyObject *kwargs, return (KORE_RESULT_ERROR); } - hdlr->methods |= method; + rt->methods |= method; if (method == HTTP_METHOD_GET) - hdlr->methods |= HTTP_METHOD_HEAD; + rt->methods |= HTTP_METHOD_HEAD; - if (!python_route_params(kwargs, hdlr, val, method, 0)) + if (!python_route_params(kwargs, rt, val, method, 0)) return (KORE_RESULT_ERROR); - if (!python_route_params(kwargs, hdlr, "qs", method, 1)) + if (!python_route_params(kwargs, rt, "qs", method, 1)) return (KORE_RESULT_ERROR); } @@ -5354,14 +5351,14 @@ python_route_methods(PyObject *obj, PyObject *kwargs, } static int -python_route_params(PyObject *kwargs, struct kore_module_handle *hdlr, +python_route_params(PyObject *kwargs, struct kore_route *rt, const char *method, int type, int qs) { Py_ssize_t idx; const char *val; int vtype; struct kore_validator *vldr; - struct kore_handler_params *param; + struct kore_route_params *param; PyObject *obj, *key, *item; if ((obj = PyDict_GetItemString(kwargs, method)) == NULL) @@ -5418,14 +5415,14 @@ python_route_params(PyObject *kwargs, struct kore_module_handle *hdlr, if (type == HTTP_METHOD_GET || qs == 1) param->flags = KORE_PARAMS_QUERY_STRING; - TAILQ_INSERT_TAIL(&hdlr->params, param, list); + TAILQ_INSERT_TAIL(&rt->params, param, list); } return (KORE_RESULT_OK); } static int -python_route_auth(PyObject *dict, struct kore_module_handle *hdlr) +python_route_auth(PyObject *dict, struct kore_route *rt) { int type; struct kore_auth *auth; @@ -5449,7 +5446,7 @@ python_route_auth(PyObject *dict, struct kore_module_handle *hdlr) } else { PyErr_Format(PyExc_RuntimeError, "invalid 'type' (%s) in auth dictionary for '%s'", - value, hdlr->path); + value, rt->path); return (KORE_RESULT_ERROR); } @@ -5464,7 +5461,7 @@ python_route_auth(PyObject *dict, struct kore_module_handle *hdlr) if ((obj = PyDict_GetItemString(dict, "verify")) == NULL || !PyCallable_Check(obj)) { PyErr_Format(PyExc_RuntimeError, - "missing 'verify' in auth dictionary for '%s'", hdlr->path); + "missing 'verify' in auth dictionary for '%s'", rt->path); return (KORE_RESULT_ERROR); } @@ -5495,7 +5492,7 @@ python_route_auth(PyObject *dict, struct kore_module_handle *hdlr) Py_DECREF(repr); auth->validator = vldr; - hdlr->auth = auth; + rt->auth = auth; return (KORE_RESULT_OK); } diff --git a/src/route.c b/src/route.c new file mode 100644 index 0000000..fe79934 --- /dev/null +++ b/src/route.c @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2021 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include + +#include + +#include "kore.h" +#include "http.h" + +struct kore_route * +kore_route_create(struct kore_domain *dom, const char *path, int type) +{ + struct kore_route *rt; + + rt = kore_calloc(1, sizeof(*rt)); + rt->dom = dom; + rt->type = type; + rt->path = kore_strdup(path); + rt->methods = HTTP_METHOD_ALL; + + TAILQ_INIT(&rt->params); + + if (rt->type == HANDLER_TYPE_DYNAMIC) { + if (regcomp(&rt->rctx, rt->path, REG_EXTENDED | REG_NOSUB)) { + kore_route_free(rt); + return (NULL); + } + } + + TAILQ_INSERT_TAIL(&dom->routes, rt, list); + + return (rt); +} + +void +kore_route_free(struct kore_route *rt) +{ + struct kore_route_params *param; + + if (rt == NULL) + return; + + kore_free(rt->func); + kore_free(rt->path); + + if (rt->type == HANDLER_TYPE_DYNAMIC) + regfree(&rt->rctx); + + /* Drop all validators associated with this handler */ + while ((param = TAILQ_FIRST(&rt->params)) != NULL) { + TAILQ_REMOVE(&rt->params, param, list); + kore_free(param->name); + kore_free(param); + } + + kore_free(rt); +} + +void +kore_route_callback(struct kore_route *rt, const char *func) +{ + if ((rt->rcall = kore_runtime_getcall(func)) == NULL) + fatal("callback '%s' for '%s' not found", func, rt->path); + + kore_free(rt->func); + rt->func = kore_strdup(func); +} + +int +kore_route_lookup(struct http_request *req, struct kore_domain *dom, + int method, struct kore_route **out) +{ + struct kore_route *rt; + int exists; + + exists = 0; + *out = NULL; + + TAILQ_FOREACH(rt, &dom->routes, list) { + if (rt->type == HANDLER_TYPE_STATIC) { + if (!strcmp(rt->path, req->path)) { + if (rt->methods & method) { + *out = rt; + return (1); + } + exists++; + } + } else { + if (!regexec(&rt->rctx, req->path, + HTTP_CAPTURE_GROUPS, req->cgroups, 0)) { + if (rt->methods & method) { + *out = rt; + return (1); + } + exists++; + } + } + } + + return (exists); +} + +void +kore_route_reload(void) +{ + struct kore_route *rt; + struct kore_server *srv; + struct kore_domain *dom; + + LIST_FOREACH(srv, &kore_servers, list) { + TAILQ_FOREACH(dom, &srv->domains, list) { + TAILQ_FOREACH(rt, &dom->routes, list) { + kore_free(rt->rcall); + rt->rcall = kore_runtime_getcall(rt->func); + if (rt->rcall == NULL) { + fatal("no function '%s' for route '%s'", + rt->func, rt->path); + } + rt->errors = 0; + } + } + } +} diff --git a/src/worker.c b/src/worker.c index 6c70073..3c13a8a 100644 --- a/src/worker.c +++ b/src/worker.c @@ -191,10 +191,11 @@ kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu) kw = WORKER(idx); kw->id = id; kw->cpu = cpu; - kw->has_lock = 0; - kw->active_hdlr = NULL; kw->running = 1; + kw->ready = 0; + kw->has_lock = 0; + kw->active_route = NULL; if (socketpair(AF_UNIX, SOCK_STREAM, 0, kw->pipe) == -1) fatal("socketpair(): %s", errno_s); @@ -743,7 +744,7 @@ kore_worker_started(void) if (!kore_quiet) { kore_log(LOG_NOTICE, - "process started (#%d %s=%s%s%s)", + "started (#%d %s=%s%s%s)", getpid(), chroot, worker->ps->root, worker->ps->skip_runas ? "" : " user=", worker->ps->skip_runas ? "" : worker->ps->runas); @@ -797,8 +798,8 @@ worker_reaper(pid_t pid, int status) func = "none"; #if !defined(KORE_NO_HTTP) - if (kw->active_hdlr != NULL) - func = kw->active_hdlr->func; + if (kw->active_route != NULL) + func = kw->active_route->func; #endif kore_log(LOG_NOTICE, "worker %d (pid: %d) (hdlr: %s) gone", @@ -828,12 +829,12 @@ worker_reaper(pid_t pid, int status) worker_unlock(); #if !defined(KORE_NO_HTTP) - if (kw->active_hdlr != NULL) { - kw->active_hdlr->errors++; + if (kw->active_route != NULL) { + kw->active_route->errors++; kore_log(LOG_NOTICE, "hdlr %s has caused %d error(s)", - kw->active_hdlr->func, - kw->active_hdlr->errors); + kw->active_route->func, + kw->active_route->errors); } #endif From f5a58368b7ea9bf99e95ae24fcb235babc80dafc Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 15 Sep 2021 22:16:22 +0200 Subject: [PATCH 073/121] HTTP improvements. Introduce an on_headers callback for routes, allowing one to inspect the headers before the request is processed further. Additionall, Add a new way of obtaining HTTP headers. Much like http_argument_get_*() functions, these new APIs allow you to fetch the data of an HTTP header as a specified C type. The new APIs are: * http_request_header_int16() * http_request_header_uint16() * http_request_header_int32() * http_request_header_uint32() * http_request_header_int64() * http_request_header_uint64() * http_request_header_float() * http_request_header_double() Should make it easier to operate in HTTP header data in a safe way. No need to always roll your own string to int conversion functions. --- include/kore/http.h | 66 ++++++++++++++++------- include/kore/kore.h | 7 ++- src/accesslog.c | 3 +- src/config.c | 44 ++++++++++++++- src/http.c | 127 ++++++++++++++++++++++++++------------------ 5 files changed, 172 insertions(+), 75 deletions(-) diff --git a/include/kore/http.h b/include/kore/http.h index 9b3cf6c..e40b686 100644 --- a/include/kore/http.h +++ b/include/kore/http.h @@ -119,7 +119,7 @@ struct http_arg { do { \ int err; \ type nval; \ - nval = (type)kore_strtonum64(q->s_value, sign, &err); \ + nval = (type)kore_strtonum64(data, sign, &err); \ if (err != KORE_RESULT_OK) \ return (KORE_RESULT_ERROR); \ COPY_ARG_TYPE(nval, type); \ @@ -129,7 +129,7 @@ struct http_arg { do { \ int err; \ type nval; \ - nval = kore_strtodouble(q->s_value, min, max, &err); \ + nval = kore_strtodouble(data, min, max, &err); \ if (err != KORE_RESULT_OK) \ return (KORE_RESULT_ERROR); \ COPY_ARG_TYPE(nval, type); \ @@ -139,7 +139,7 @@ struct http_arg { do { \ int err; \ int64_t nval; \ - nval = kore_strtonum(q->s_value, 10, min, max, &err); \ + nval = kore_strtonum(data, 10, min, max, &err); \ if (err != KORE_RESULT_OK) \ return (KORE_RESULT_ERROR); \ COPY_ARG_TYPE(nval, type); \ @@ -159,38 +159,62 @@ struct http_arg { COPY_ARG_INT(min, max, type); \ } while (0) -#define http_argument_type(r, n, so, no, t) \ - http_argument_get(r, n, so, no, t) - #define http_argument_get_string(r, n, o) \ - http_argument_type(r, n, (void **)o, NULL, HTTP_ARG_TYPE_STRING) + http_argument_get(r, n, (void **)o, NULL, HTTP_ARG_TYPE_STRING) -#define http_argument_get_byte(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_BYTE) +#define http_argument_get_byte(r, n, o) \ + http_argument_get(r, n, NULL, o, HTTP_ARG_TYPE_BYTE) #define http_argument_get_uint16(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT16) + http_argument_get(r, n, NULL, o, HTTP_ARG_TYPE_UINT16) #define http_argument_get_int16(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT16) + http_argument_get(r, n, NULL, o, HTTP_ARG_TYPE_INT16) #define http_argument_get_uint32(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT32) + http_argument_get(r, n, NULL, o, HTTP_ARG_TYPE_UINT32) #define http_argument_get_int32(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT32) + http_argument_get(r, n, NULL, o, HTTP_ARG_TYPE_INT32) #define http_argument_get_uint64(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT64) + http_argument_get(r, n, NULL, o, HTTP_ARG_TYPE_UINT64) #define http_argument_get_int64(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT64) + http_argument_get(r, n, NULL, o, HTTP_ARG_TYPE_INT64) #define http_argument_get_float(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_FLOAT) + http_argument_get(r, n, NULL, o, HTTP_ARG_TYPE_FLOAT) #define http_argument_get_double(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_DOUBLE) + http_argument_get(r, n, NULL, o, HTTP_ARG_TYPE_DOUBLE) + +#define http_request_header_byte(r, n, o) \ + http_request_header_get(r, n, NULL, o, HTTP_ARG_TYPE_BYTE) + +#define http_request_header_uint16(r, n, o) \ + http_request_header_get(r, n, NULL, o, HTTP_ARG_TYPE_UINT16) + +#define http_request_header_int16(r, n, o) \ + http_request_header_get(r, n, NULL, o, HTTP_ARG_TYPE_INT16) + +#define http_request_header_uint32(r, n, o) \ + http_request_header_get(r, n, NULL, o, HTTP_ARG_TYPE_UINT32) + +#define http_request_header_int32(r, n, o) \ + http_request_header_get(r, n, NULL, o, HTTP_ARG_TYPE_INT32) + +#define http_request_header_uint64(r, n, o) \ + http_request_header_get(r, n, NULL, o, HTTP_ARG_TYPE_UINT64) + +#define http_request_header_int64(r, n, o) \ + http_request_header_get(r, n, NULL, o, HTTP_ARG_TYPE_INT64) + +#define http_request_header_float(r, n, o) \ + http_request_header_get(r, n, NULL, o, HTTP_ARG_TYPE_FLOAT) + +#define http_request_header_double(r, n, o) \ + http_request_header_get(r, n, NULL, o, HTTP_ARG_TYPE_DOUBLE) struct http_file { char *name; @@ -262,9 +286,9 @@ struct http_request { struct kore_buf *http_body; int http_body_fd; char *http_body_path; - size_t http_body_length; - size_t http_body_offset; - size_t content_length; + u_int64_t http_body_length; + u_int64_t http_body_offset; + u_int64_t content_length; void *hdlr_extra; size_t state_len; char *query_string; @@ -400,6 +424,8 @@ void http_populate_multipart_form(struct http_request *); void http_populate_cookies(struct http_request *); int http_argument_get(struct http_request *, const char *, void **, void *, int); +int http_request_header_get(struct http_request *, + const char *, void **, void *, int); void http_file_rewind(struct http_file *); ssize_t http_file_read(struct http_file *, void *, size_t); diff --git a/include/kore/kore.h b/include/kore/kore.h index ecb0749..80ff563 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -320,11 +320,14 @@ struct kore_route { char *func; int type; int errors; + int methods; regex_t rctx; struct kore_domain *dom; - struct kore_runtime_call *rcall; struct kore_auth *auth; - int methods; + struct kore_runtime_call *rcall; + struct kore_runtime_call *on_headers; + struct kore_runtime_call *on_body_chunk; + TAILQ_HEAD(, kore_route_params) params; TAILQ_ENTRY(kore_route) list; }; diff --git a/src/accesslog.c b/src/accesslog.c index 84af979..26f4195 100644 --- a/src/accesslog.c +++ b/src/accesslog.c @@ -18,6 +18,7 @@ #include #include +#include #include #include "kore.h" @@ -174,7 +175,7 @@ kore_accesslog(struct http_request *req) worker->lb.offset += sizeof(*hdr); len = snprintf(worker->lb.buf + worker->lb.offset, avail, - "%s - %s [%s] \"%s %s %s\" %d %zu \"%s\" \"%s\"\n", + "%s - %s [%s] \"%s %s %s\" %d %" PRIu64" \"%s\" \"%s\"\n", addr, cn, tbuf, method, req->path, http_version, req->status, req->content_length, referer, req->agent); if (len == -1) diff --git a/src/config.c b/src/config.c index 9d1d421..2cd66ac 100644 --- a/src/config.c +++ b/src/config.c @@ -109,8 +109,10 @@ static int configure_client_verify_depth(char *); #if !defined(KORE_NO_HTTP) static int configure_route(char *); -static int configure_route_handler(char *); static int configure_route_methods(char *); +static int configure_route_handler(char *); +static int configure_route_on_headers(char *); +static int configure_route_on_body_chunk(char *); static int configure_filemap(char *); static int configure_return(char *); static int configure_redirect(char *); @@ -191,6 +193,8 @@ static struct { #if !defined(KORE_NO_HTTP) { "route", configure_route }, { "handler", configure_route_handler }, + { "on_headers", configure_route_on_headers }, + { "on_body_chunk", configure_route_on_body_chunk }, { "methods", configure_route_methods }, { "filemap", configure_filemap }, { "redirect", configure_redirect }, @@ -1155,6 +1159,44 @@ configure_route_handler(char *name) return (KORE_RESULT_OK); } +static int +configure_route_on_headers(char *name) +{ + if (current_route == NULL) { + kore_log(LOG_ERR, + "on_header keyword not inside of route context"); + return (KORE_RESULT_ERROR); + } + + if ((current_route->on_headers = kore_runtime_getcall(name)) == NULL) { + kore_log(LOG_ERR, "on_headers callback '%s' for '%s' not found", + name, current_route->path); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int +configure_route_on_body_chunk(char *name) +{ + if (current_route == NULL) { + kore_log(LOG_ERR, + "on_body_chunk keyword not inside of route context"); + return (KORE_RESULT_ERROR); + } + + current_route->on_body_chunk = kore_runtime_getcall(name); + if (current_route->on_body_chunk == NULL) { + kore_log(LOG_ERR, + "on_body_chunk callback '%s' for '%s' not found", + name, current_route->path); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + static int configure_route_methods(char *options) { diff --git a/src/http.c b/src/http.c index 88e8181..2c9152c 100644 --- a/src/http.c +++ b/src/http.c @@ -130,6 +130,7 @@ static const char *pretty_error_fmt = static int http_body_recv(struct netbuf *); static int http_release_buffer(struct netbuf *); static void http_error_response(struct connection *, int); +static int http_data_convert(void *, void **, void *, int); static void http_write_response_cookie(struct http_cookie *); static void http_argument_add(struct http_request *, char *, char *, int, int); @@ -763,6 +764,28 @@ http_request_header(struct http_request *req, const char *header, return (KORE_RESULT_ERROR); } +int +http_request_header_get(struct http_request *req, const char *header, + void **out, void *nout, int type) +{ + struct http_header *hdr; + + if (type == HTTP_ARG_TYPE_STRING) + fatal("%s: cannot be called with type string", __func__); + + TAILQ_FOREACH(hdr, &req->req_headers, list) { + if (strcasecmp(hdr->header, header)) + continue; + + if (http_data_convert(hdr->value, out, nout, type)) + return (KORE_RESULT_OK); + + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_ERROR); +} + int http_request_cookie(struct http_request *req, const char *cookie, char **out) { @@ -786,7 +809,6 @@ http_header_recv(struct netbuf *nb) ssize_t ret; struct http_header *hdr; struct http_request *req; - const char *clp; u_int64_t bytes_left; u_int8_t *end_headers; int h, i, v, skip, l; @@ -897,17 +919,8 @@ http_header_recv(struct netbuf *nb) return (KORE_RESULT_OK); } - if (!http_request_header(req, "content-length", &clp)) { - kore_debug("expected body but no content-length"); - req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, - HTTP_STATUS_LENGTH_REQUIRED); - return (KORE_RESULT_OK); - } - - req->content_length = kore_strtonum(clp, 10, 0, LONG_MAX, &v); - if (v == KORE_RESULT_ERROR) { - kore_debug("content-length invalid: %s", clp); + if (!http_request_header_uint64(req, "content-length", + &req->content_length)) { req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, HTTP_STATUS_LENGTH_REQUIRED); @@ -921,8 +934,6 @@ http_header_recv(struct netbuf *nb) } if (req->content_length > http_body_max) { - kore_log(LOG_NOTICE, "body too large (%zu > %zu)", - req->content_length, http_body_max); req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE); @@ -997,6 +1008,13 @@ http_header_recv(struct netbuf *nb) c->http_timeout = 0; } + if (req->rt->on_headers != NULL) { + if (!kore_runtime_http_request(req->rt->on_headers, req)) { + req->flags |= HTTP_REQUEST_DELETE; + return (KORE_RESULT_OK); + } + } + return (KORE_RESULT_OK); } @@ -1010,45 +1028,10 @@ http_argument_get(struct http_request *req, const char *name, if (strcmp(q->name, name)) continue; - switch (type) { - case HTTP_ARG_TYPE_RAW: - *out = q->s_value; + if (http_data_convert(q->s_value, out, nout, type)) return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_BYTE: - COPY_ARG_TYPE(*(u_int8_t *)q->s_value, u_int8_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_INT16: - COPY_AS_INTTYPE(SHRT_MIN, SHRT_MAX, int16_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_UINT16: - COPY_AS_INTTYPE(0, USHRT_MAX, u_int16_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_INT32: - COPY_AS_INTTYPE(INT_MIN, INT_MAX, int32_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_UINT32: - COPY_AS_INTTYPE(0, UINT_MAX, u_int32_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_INT64: - COPY_AS_INTTYPE_64(int64_t, 1); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_UINT64: - COPY_AS_INTTYPE_64(u_int64_t, 0); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_FLOAT: - COPY_ARG_DOUBLE(-FLT_MAX, FLT_MAX, float); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_DOUBLE: - COPY_ARG_DOUBLE(-DBL_MAX, DBL_MAX, double); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_STRING: - *out = q->s_value; - return (KORE_RESULT_OK); - default: - break; - } - return (KORE_RESULT_ERROR); + break; } return (KORE_RESULT_ERROR); @@ -2598,3 +2581,45 @@ http_write_response_cookie(struct http_cookie *ck) kore_buf_appendf(header_buf, "set-cookie: %s\r\n", kore_buf_stringify(ckhdr_buf, NULL)); } + +static int +http_data_convert(void *data, void **out, void *nout, int type) +{ + switch (type) { + case HTTP_ARG_TYPE_RAW: + case HTTP_ARG_TYPE_STRING: + *out = data; + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_BYTE: + COPY_ARG_TYPE(*(u_int8_t *)data, u_int8_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_INT16: + COPY_AS_INTTYPE(SHRT_MIN, SHRT_MAX, int16_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_UINT16: + COPY_AS_INTTYPE(0, USHRT_MAX, u_int16_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_INT32: + COPY_AS_INTTYPE(INT_MIN, INT_MAX, int32_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_UINT32: + COPY_AS_INTTYPE(0, UINT_MAX, u_int32_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_INT64: + COPY_AS_INTTYPE_64(int64_t, 1); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_UINT64: + COPY_AS_INTTYPE_64(u_int64_t, 0); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_FLOAT: + COPY_ARG_DOUBLE(-FLT_MAX, FLT_MAX, float); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_DOUBLE: + COPY_ARG_DOUBLE(-DBL_MAX, DBL_MAX, double); + return (KORE_RESULT_OK); + default: + break; + } + + return (KORE_RESULT_ERROR); +} From 351eec7eb44998093ffd90f2e0aa0ecdbacf2330 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 17 Sep 2021 19:28:06 +0200 Subject: [PATCH 074/121] Add the on_body_chunk handler for routes. If set, will call a given handler with the prototype of `void body_chunk(struct http_request *req, const void *data, size_t len);` for each chunk of the received HTTP body, allowing a developer to handle it in their own way. The incoming body is still being handled and retained in the same way as before (in a kore_buf or temporary file). While here, allow HTTP_STATUS_CONTINUE to work via http_response() and make the handling of incoming HTTP header data a bit better. --- include/kore/kore.h | 4 ++ src/http.c | 91 +++++++++++++++++++++------------------------ src/net.c | 3 ++ src/runtime.c | 21 +++++++++++ 4 files changed, 71 insertions(+), 48 deletions(-) diff --git a/include/kore/kore.h b/include/kore/kore.h index 80ff563..b22becd 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -287,6 +287,8 @@ struct kore_runtime { int type; #if !defined(KORE_NO_HTTP) int (*http_request)(void *, struct http_request *); + void (*http_body_chunk)(void *, + struct http_request *, const void *, size_t); int (*validator)(void *, struct http_request *, const void *); void (*wsconnect)(void *, struct connection *); void (*wsdisconnect)(void *, struct connection *); @@ -986,6 +988,8 @@ void kore_runtime_connect(struct kore_runtime_call *, struct connection *); #if !defined(KORE_NO_HTTP) int kore_runtime_http_request(struct kore_runtime_call *, struct http_request *); +void kore_runtime_http_body_chunk(struct kore_runtime_call *, + struct http_request *, const void *, size_t); int kore_runtime_validator(struct kore_runtime_call *, struct http_request *, const void *); void kore_runtime_wsconnect(struct kore_runtime_call *, struct connection *); diff --git a/src/http.c b/src/http.c index 2c9152c..6be26cb 100644 --- a/src/http.c +++ b/src/http.c @@ -132,6 +132,7 @@ static int http_release_buffer(struct netbuf *); static void http_error_response(struct connection *, int); static int http_data_convert(void *, void **, void *, int); static void http_write_response_cookie(struct http_cookie *); +static int http_body_update(struct http_request *, const void *, size_t); static void http_argument_add(struct http_request *, char *, char *, int, int); static int http_check_redirect(struct http_request *, @@ -806,10 +807,8 @@ http_header_recv(struct netbuf *nb) { struct connection *c; size_t len; - ssize_t ret; struct http_header *hdr; struct http_request *req; - u_int64_t bytes_left; u_int8_t *end_headers; int h, i, v, skip, l; char *headers[HTTP_REQ_HEADER_MAX]; @@ -821,6 +820,11 @@ http_header_recv(struct netbuf *nb) if (nb->b_len < 4) return (KORE_RESULT_OK); + if (!isalpha(nb->buf[0])) { + http_error_response(c, HTTP_STATUS_BAD_REQUEST); + return (KORE_RESULT_ERROR); + } + skip = 4; end_headers = kore_mem_find(nb->buf, nb->s_off, "\r\n\r\n", 4); if (end_headers == NULL) { @@ -962,47 +966,19 @@ http_header_recv(struct netbuf *nb) HTTP_STATUS_INTERNAL_ERROR); return (KORE_RESULT_OK); } - - ret = write(req->http_body_fd, - end_headers, (nb->s_off - len)); - if (ret == -1 || (size_t)ret != (nb->s_off - len)) { - req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, - HTTP_STATUS_INTERNAL_ERROR); - return (KORE_RESULT_OK); - } } else { req->http_body_fd = -1; req->http_body = kore_buf_alloc(req->content_length); - kore_buf_append(req->http_body, end_headers, - (nb->s_off - len)); } SHA256_Init(&req->hashctx); - SHA256_Update(&req->hashctx, end_headers, (nb->s_off - len)); + c->http_timeout = http_body_timeout * 1000; - bytes_left = req->content_length - (nb->s_off - len); - if (bytes_left > 0) { - kore_debug("%ld/%ld (%ld - %ld) more bytes for body", - bytes_left, req->content_length, nb->s_off, len); - net_recv_reset(c, - MIN(bytes_left, NETBUF_SEND_PAYLOAD_MAX), - http_body_recv); - c->rnb->extra = req; - http_request_sleep(req); - req->content_length = bytes_left; - c->http_timeout = http_body_timeout * 1000; - } else { - c->http_timeout = 0; - req->flags |= HTTP_REQUEST_COMPLETE; - req->flags &= ~HTTP_REQUEST_EXPECT_BODY; - SHA256_Final(req->http_body_digest, &req->hashctx); - if (!http_body_rewind(req)) { - req->flags |= HTTP_REQUEST_DELETE; - http_error_response(req->owner, - HTTP_STATUS_INTERNAL_ERROR); - return (KORE_RESULT_OK); - } + if (!http_body_update(req, end_headers, nb->s_off - len)) { + req->flags |= HTTP_REQUEST_DELETE; + http_error_response(req->owner, + HTTP_STATUS_INTERNAL_ERROR); + return (KORE_RESULT_OK); } } else { c->http_timeout = 0; @@ -1505,7 +1481,6 @@ http_start_recv(struct connection *c) c->http_start = kore_time_ms(); c->http_timeout = http_header_timeout * 1000; net_recv_reset(c, http_header_max, http_header_recv); - (void)net_recv_flush(c); } void @@ -2360,22 +2335,29 @@ http_argument_add(struct http_request *req, char *name, char *value, int qs, static int http_body_recv(struct netbuf *nb) { - ssize_t ret; - u_int64_t bytes_left; struct http_request *req = (struct http_request *)nb->extra; - SHA256_Update(&req->hashctx, nb->buf, nb->s_off); + return (http_body_update(req, nb->buf, nb->s_off)); +} + +static int +http_body_update(struct http_request *req, const void *data, size_t len) +{ + ssize_t ret; + u_int64_t bytes_left; + + SHA256_Update(&req->hashctx, data, len); if (req->http_body_fd != -1) { - ret = write(req->http_body_fd, nb->buf, nb->s_off); - if (ret == -1 || (size_t)ret != nb->s_off) { + ret = write(req->http_body_fd, data, len); + if (ret == -1 || (size_t)ret != len) { req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, HTTP_STATUS_INTERNAL_ERROR); return (KORE_RESULT_ERROR); } } else if (req->http_body != NULL) { - kore_buf_append(req->http_body, nb->buf, nb->s_off); + kore_buf_append(req->http_body, data, len); } else { req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, @@ -2383,10 +2365,10 @@ http_body_recv(struct netbuf *nb) return (KORE_RESULT_ERROR); } - req->content_length -= nb->s_off; + req->content_length -= len; if (req->content_length == 0) { - nb->extra = NULL; + req->owner->rnb->extra = NULL; http_request_wakeup(req); req->flags |= HTTP_REQUEST_COMPLETE; req->flags &= ~HTTP_REQUEST_EXPECT_BODY; @@ -2398,12 +2380,17 @@ http_body_recv(struct netbuf *nb) return (KORE_RESULT_ERROR); } SHA256_Final(req->http_body_digest, &req->hashctx); - net_recv_reset(nb->owner, http_header_max, http_header_recv); } else { bytes_left = req->content_length; - net_recv_reset(nb->owner, + net_recv_reset(req->owner, MIN(bytes_left, NETBUF_SEND_PAYLOAD_MAX), http_body_recv); + req->owner->rnb->extra = req; + } + + if (req->rt->on_body_chunk != NULL && len > 0) { + kore_runtime_http_body_chunk(req->rt->on_body_chunk, + req, data, len); } return (KORE_RESULT_OK); @@ -2442,7 +2429,6 @@ http_response_normal(struct http_request *req, struct connection *c, send_body = 1; text = http_status_text(status); - kore_buf_init(&buf, 1024); kore_buf_reset(header_buf); if (req != NULL) { @@ -2456,6 +2442,13 @@ http_response_normal(struct http_request *req, struct connection *c, kore_buf_appendf(header_buf, "HTTP/1.%c %d %s\r\n", version, status, text); + + if (status == 100) { + kore_buf_append(header_buf, "\r\n", 2); + net_send_queue(c, header_buf->data, header_buf->offset); + return; + } + kore_buf_append(header_buf, http_version, http_version_len); if ((c->flags & CONN_CLOSE_EMPTY) || @@ -2474,6 +2467,8 @@ http_response_normal(struct http_request *req, struct connection *c, } } + kore_buf_init(&buf, 1024); + /* Note that req CAN be NULL. */ if (req == NULL || req->owner->proto != CONN_PROTO_WEBSOCKET) { if (http_keepalive_time && connection_close == 0) { diff --git a/src/net.c b/src/net.c index 992e099..5771c82 100644 --- a/src/net.c +++ b/src/net.c @@ -292,6 +292,9 @@ net_recv_flush(struct connection *c) if (c->rnb->buf == NULL) return (KORE_RESULT_OK); + if ((c->rnb->b_len - c->rnb->s_off) == 0) + return (KORE_RESULT_OK); + if (!c->read(c, &r)) return (KORE_RESULT_ERROR); if (!(c->evt.flags & KORE_EVENT_READ)) diff --git a/src/runtime.c b/src/runtime.c index 3d1e2fa..1edd8de 100644 --- a/src/runtime.c +++ b/src/runtime.c @@ -33,6 +33,8 @@ static void native_runtime_connect(void *, struct connection *); static void native_runtime_configure(void *, int, char **); #if !defined(KORE_NO_HTTP) static int native_runtime_http_request(void *, struct http_request *); +static void native_runtime_http_body_chunk(void *, struct http_request *, + const void *, size_t); static int native_runtime_validator(void *, struct http_request *, const void *); @@ -44,6 +46,7 @@ struct kore_runtime kore_native_runtime = { KORE_RUNTIME_NATIVE, #if !defined(KORE_NO_HTTP) .http_request = native_runtime_http_request, + .http_body_chunk= native_runtime_http_body_chunk, .validator = native_runtime_validator, .wsconnect = native_runtime_connect, .wsmessage = native_runtime_wsmessage, @@ -105,6 +108,13 @@ kore_runtime_http_request(struct kore_runtime_call *rcall, return (rcall->runtime->http_request(rcall->addr, req)); } +void +kore_runtime_http_body_chunk(struct kore_runtime_call *rcall, + struct http_request *req, const void *data, size_t len) +{ + rcall->runtime->http_body_chunk(rcall->addr, req, data, len); +} + int kore_runtime_validator(struct kore_runtime_call *rcall, struct http_request *req, const void *data) @@ -178,6 +188,17 @@ native_runtime_http_request(void *addr, struct http_request *req) return (cb(req)); } +static void +native_runtime_http_body_chunk(void *addr, struct http_request *req, + const void *data, size_t len) +{ + void (*cb)(struct http_request *, const void *, size_t); + + *(void **)&(cb) = addr; + + cb(req, data, len); +} + static int native_runtime_validator(void *addr, struct http_request *req, const void *data) { From bcfb79a389b6bc7689fc6e3d1ec2f3035231870a Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 17 Sep 2021 19:49:32 +0200 Subject: [PATCH 075/121] Remove dead assignment and unused vars. --- src/http.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/http.c b/src/http.c index 6be26cb..e8a9031 100644 --- a/src/http.c +++ b/src/http.c @@ -1906,9 +1906,9 @@ http_request_new(struct connection *c, const char *host, { struct kore_domain *dom; struct http_request *req; + size_t qsoff; char *p, *hp; int m, flags, exists; - size_t hostlen, pathlen, qsoff; if (http_request_count >= http_request_limit) { http_error_response(c, HTTP_STATUS_SERVICE_UNAVAILABLE); @@ -1918,12 +1918,12 @@ http_request_new(struct connection *c, const char *host, kore_debug("http_request_new(%p, %s, %s, %s, %s)", c, host, method, path, version); - if ((hostlen = strlen(host)) >= KORE_DOMAINNAME_LEN - 1) { + if (strlen(host) >= KORE_DOMAINNAME_LEN - 1) { http_error_response(c, HTTP_STATUS_BAD_REQUEST); return (NULL); } - if ((pathlen = strlen(path)) >= HTTP_URI_LEN - 1) { + if (strlen(path) >= HTTP_URI_LEN - 1) { http_error_response(c, HTTP_STATUS_REQUEST_URI_TOO_LARGE); return (NULL); } From 9f6043bbdea03f4236c28e05b37ce7b9a4453d76 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 17 Sep 2021 19:52:52 +0200 Subject: [PATCH 076/121] Remove dead assignment. --- src/msg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/msg.c b/src/msg.c index a01f6ac..bc259c1 100644 --- a/src/msg.c +++ b/src/msg.c @@ -141,7 +141,7 @@ kore_msg_register(u_int8_t id, void (*cb)(struct kore_msg *, const void *)) { struct msg_type *type; - if ((type = msg_type_lookup(id)) != NULL) + if (msg_type_lookup(id) != NULL) return (KORE_RESULT_ERROR); type = kore_malloc(sizeof(*type)); From 6b1f02e6b06e8a9f1beb3a1673cfb2cf2ffd7c0d Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 17 Sep 2021 19:53:31 +0200 Subject: [PATCH 077/121] Remove another dead assignment. --- src/auth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auth.c b/src/auth.c index 4ac836e..7f04bb9 100644 --- a/src/auth.c +++ b/src/auth.c @@ -35,7 +35,7 @@ kore_auth_new(const char *name) { struct kore_auth *auth; - if ((auth = kore_auth_lookup(name)) != NULL) + if (kore_auth_lookup(name) != NULL) return (KORE_RESULT_ERROR); auth = kore_malloc(sizeof(*auth)); From 63bbc1fa0fb5855953b476692bae60192a31c441 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 17 Sep 2021 19:56:35 +0200 Subject: [PATCH 078/121] Be sure content_length is 0, just in case. --- src/http.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/http.c b/src/http.c index e8a9031..7e56092 100644 --- a/src/http.c +++ b/src/http.c @@ -2038,6 +2038,7 @@ http_request_new(struct connection *c, const char *host, req->http_body = NULL; req->http_body_fd = -1; req->hdlr_extra = NULL; + req->content_length = 0; req->query_string = NULL; req->http_body_length = 0; req->http_body_offset = 0; From 31aaf128a11473591ab993f4592a7c056f5783f1 Mon Sep 17 00:00:00 2001 From: Joel Arbring Date: Fri, 17 Sep 2021 15:22:46 +0200 Subject: [PATCH 079/121] Check for .so file where we create it --- src/cli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli.c b/src/cli.c index 5ed97b1..114548e 100644 --- a/src/cli.c +++ b/src/cli.c @@ -718,9 +718,9 @@ cli_build(int argc, char **argv) if (bopt->single_binary) { requires_relink++; - (void)cli_vasprintf(&sofile, "%s", appl); + (void)cli_vasprintf(&sofile, "%s/%s", out_dir, appl); } else { - (void)cli_vasprintf(&sofile, "%s.so", appl); + (void)cli_vasprintf(&sofile, "%s/%s.so", out_dir, appl); } if (!cli_file_exists(sofile) && source_files_count > 0) From 46b7a962bfeb0e70c3032dfac4cbc2c9f4035384 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 21 Sep 2021 14:01:20 +0200 Subject: [PATCH 080/121] be verbose about filemap resolving. --- src/filemap.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/filemap.c b/src/filemap.c index 293a83d..9c5b7ef 100644 --- a/src/filemap.c +++ b/src/filemap.c @@ -79,8 +79,11 @@ kore_filemap_create(struct kore_domain *dom, const char *path, const char *root) fatal("kore_filemap_create: failed to copy path"); } - if (stat(fpath, &st) == -1) + if (stat(fpath, &st) == -1) { + kore_log(LOG_ERR, "%s: failed to stat '%s': %s", __func__, + fpath, errno_s); return (KORE_RESULT_ERROR); + } len = snprintf(regex, sizeof(regex), "^%s.*$", root); if (len == -1 || (size_t)len >= sizeof(regex)) From 17ceb32e23f02f20ae30f012530aae0c745e5818 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 17 Sep 2021 20:23:20 +0200 Subject: [PATCH 081/121] Cleanup single/dso files properly with kodev clean. Obey the out_dir too, in case its set differently. --- src/cli.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/cli.c b/src/cli.c index 114548e..8e1dd98 100644 --- a/src/cli.c +++ b/src/cli.c @@ -748,7 +748,8 @@ cli_source(int argc, char **argv) static void cli_clean(int argc, char **argv) { - char pwd[PATH_MAX], *sofile; + struct buildopt *bopt; + char pwd[PATH_MAX], *bin; if (cli_dir_exists(object_dir)) cli_cleanup_files(object_dir); @@ -757,11 +758,23 @@ cli_clean(int argc, char **argv) fatal("could not get cwd: %s", errno_s); appl = basename(pwd); - (void)cli_vasprintf(&sofile, "%s.so", appl); - if (unlink(sofile) == -1 && errno != ENOENT) - printf("couldn't unlink %s: %s", sofile, errno_s); - free(sofile); + TAILQ_INIT(&mime_types); + TAILQ_INIT(&build_options); + + cli_flavor_load(); + bopt = cli_buildopt_new("_default"); + cli_buildopt_parse("conf/build.conf"); + + if (bopt->single_binary) + (void)cli_vasprintf(&bin, "%s/%s", out_dir, appl); + else + (void)cli_vasprintf(&bin, "%s/%s.so", out_dir, appl); + + if (unlink(bin) == -1 && errno != ENOENT) + printf("couldn't unlink %s: %s", bin, errno_s); + + free(bin); } static void From 01370c262d0902374f690578bb62bb7e6ea64a49 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 21 Sep 2021 20:47:16 +0200 Subject: [PATCH 082/121] fix builds with DEBUG. --- src/http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http.c b/src/http.c index 7e56092..17b7699 100644 --- a/src/http.c +++ b/src/http.c @@ -647,7 +647,7 @@ http_response_json(struct http_request *req, int status, if (req->owner == NULL) return; - kore_debug("%s(%p, %d)", __func__, req, code); + kore_debug("%s(%p, %d)", __func__, req, status); buf = kore_buf_alloc(1024); kore_json_item_tobuf(json, buf); From af45284641878da4a407e6f8ee2cdf0acbf5f717 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 21 Sep 2021 20:47:23 +0200 Subject: [PATCH 083/121] count acme domains when configured with Python --- src/python.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python.c b/src/python.c index 4006842..f9b1c1a 100644 --- a/src/python.c +++ b/src/python.c @@ -2142,6 +2142,7 @@ python_kore_domain(PyObject *self, PyObject *args, PyObject *kwargs) if (acme) { kore_acme_get_paths(name, &akey, &acert); + acme_domains++; key = akey; cert = acert; } From 3e85d36532f60bfd31970dd5ea52ccd001dda7ae Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 22 Sep 2021 16:48:21 +0200 Subject: [PATCH 084/121] The *_CheckExact() family sets no exceptions. So set a runtime exception if the objects passed mismatch. --- src/python.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/python.c b/src/python.c index f9b1c1a..08f0225 100644 --- a/src/python.c +++ b/src/python.c @@ -5151,13 +5151,17 @@ pydomain_filemaps(struct pydomain *domain, PyObject *args) if (!PyArg_ParseTuple(args, "O", &dict)) return (NULL); - if (!PyDict_CheckExact(dict)) + if (!PyDict_CheckExact(dict)) { + PyErr_SetString(PyExc_RuntimeError, "filemaps not a dict"); return (NULL); + } idx = 0; while (PyDict_Next(dict, &idx, &key, &value)) { if (!PyUnicode_CheckExact(key) || !PyUnicode_CheckExact(value)) { + PyErr_SetString(PyExc_RuntimeError, + "filemap entries not strings"); return (NULL); } @@ -5318,8 +5322,10 @@ python_route_methods(PyObject *obj, PyObject *kwargs, struct kore_route *rt) int method; Py_ssize_t list_len, idx; - if (!PyList_CheckExact(obj)) + if (!PyList_CheckExact(obj)) { + PyErr_SetString(PyExc_RuntimeError, "methods not a list"); return (KORE_RESULT_ERROR); + } rt->methods = 0; list_len = PyList_Size(obj); From 1fcc9345a682579694fca84532d7802b153e1aa3 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 22 Sep 2021 20:50:09 +0200 Subject: [PATCH 085/121] add cflags/ldflags commands to kodev. These will spew out the required CFLAGS and LDFLAGS respectively when compiling source code for use in Kore applications. This should make it easier to integrate this into existing build systems where using kodev may be a bit annoying. Eg: gcc -Wall -std=c99 `kodev cflags` koreapp.c `kodev ldflags` -o koreapp.so --- Makefile | 4 ++- src/cli.c | 79 +++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index fd8cb8b..f162ced 100644 --- a/Makefile +++ b/Makefile @@ -184,6 +184,7 @@ $(KODEV): src/cli.c $(KORE): $(OBJDIR) $(S_OBJS) $(CC) $(S_OBJS) $(LDFLAGS) -o $(KORE) + @echo $(LDFLAGS) > kore.linker @echo $(FEATURES) $(FEATURES_INC) > kore.features objects: $(OBJDIR) $(PLATFORM) $(GENERATED) $(S_OBJS) @@ -201,6 +202,7 @@ install: install -m 644 share/man/kodev.1 $(DESTDIR)$(MAN_DIR)/man1/kodev.1 install -m 555 $(KORE) $(DESTDIR)$(INSTALL_DIR)/$(KORE) install -m 644 kore.features $(DESTDIR)$(SHARE_DIR)/features + install -m 644 kore.linker $(DESTDIR)$(SHARE_DIR)/linker install -m 644 include/kore/*.h $(DESTDIR)$(INCLUDE_DIR) install -m 644 misc/ffdhe4096.pem $(DESTDIR)$(SHARE_DIR)/ffdhe4096.pem $(MAKE) -C kodev install @@ -267,7 +269,7 @@ src/seccomp.c: $(PLATFORM) clean: rm -f $(VERSION) find . -type f -name \*.o -exec rm {} \; - rm -rf $(KORE) $(OBJDIR) kore.features + rm -rf $(KORE) $(OBJDIR) kore.features kore.linker $(MAKE) -C kodev clean releng-build-examples: diff --git a/src/cli.c b/src/cli.c index 8e1dd98..d7a80f0 100644 --- a/src/cli.c +++ b/src/cli.c @@ -187,9 +187,12 @@ static void cli_buildopt_kore_flavor(struct buildopt *, const char *); static void cli_buildopt_mime(struct buildopt *, const char *); +static void cli_build_flags_common(struct buildopt *, + struct cli_buf *); + static void cli_flavor_load(void); static void cli_flavor_change(const char *); -static void cli_kore_features(struct buildopt *, +static void cli_kore_load_file(const char *, struct buildopt *, char **, size_t *); static void cli_run(int, char **); @@ -200,6 +203,8 @@ static void cli_clean(int, char **); static void cli_source(int, char **); static void cli_reload(int, char **); static void cli_flavor(int, char **); +static void cli_cflags(int, char **); +static void cli_ldflags(int, char **); #if !defined(KODEV_MINIMAL) static void cli_create(int, char **); @@ -226,6 +231,8 @@ static struct cmd cmds[] = { { "create", "create a new application skeleton", cli_create }, #endif { "flavor", "switch between build flavors", cli_flavor }, + { "cflags", "show kore CFLAGS", cli_cflags }, + { "ldflags", "show kore LDFLAGS", cli_ldflags }, { NULL, NULL, NULL } }; @@ -668,14 +675,12 @@ cli_build(int argc, char **argv) if (cli_dir_exists(assets_path)) cli_find_files(assets_path, cli_build_asset); - if (bopt->single_binary) { - memset(&dp, 0, sizeof(dp)); - dp.d_type = DT_REG; - printf("adding config %s\n", config); - (void)snprintf(dp.d_name, - sizeof(dp.d_name), "builtin_kore.conf"); - cli_build_asset(config, &dp); - } + memset(&dp, 0, sizeof(dp)); + dp.d_type = DT_REG; + printf("adding config %s\n", config); + (void)snprintf(dp.d_name, + sizeof(dp.d_name), "builtin_kore.conf"); + cli_build_asset(config, &dp); cli_file_writef(s_fd, "\n#endif\n"); cli_file_close(s_fd); @@ -842,13 +847,43 @@ cli_info(int argc, char **argv) printf("kore features\t %s\n", bopt->kore_flavor); printf("kore source \t %s\n", bopt->kore_source); } else { - cli_kore_features(bopt, &features, &len); + cli_kore_load_file("features", bopt, &features, &len); printf("kore binary \t %s/bin/kore\n", prefix); printf("kore features\t %.*s\n", (int)len, features); free(features); } } +static void +cli_cflags(int argc, char **argv) +{ + struct cli_buf *buf; + + buf = cli_buf_alloc(128); + cli_build_flags_common(NULL, buf); + printf("%.*s\n", (int)buf->offset, buf->data); + cli_buf_free(buf); +} + +static void +cli_ldflags(int argc, char **argv) +{ + char *p; + size_t len; + + cli_kore_load_file("linker", NULL, &p, &len); + printf("%.*s ", (int)len, p); + +#if defined(__MACH__) + printf("-dynamiclib -undefined suppress -flat_namespace "); +#else + printf("-shared "); +#endif + printf("\n"); + + free(p); +} + #if !defined(KODEV_MINIMAL) static void file_create_python_src(void) @@ -1906,9 +1941,12 @@ cli_build_flags_common(struct buildopt *bopt, struct cli_buf *buf) size_t len; char *data; - cli_buf_appendf(buf, "-fPIC -Isrc -Isrc/includes "); + cli_buf_appendf(buf, "-fPIC "); - if (bopt->single_binary == 0) + if (bopt != NULL) + cli_buf_appendf(buf, "-Isrc -Isrc/includes "); + + if (bopt == NULL || bopt->single_binary == 0) cli_buf_appendf(buf, "-I%s/include ", prefix); else cli_buf_appendf(buf, "-I%s/include ", bopt->kore_source); @@ -1919,8 +1957,8 @@ cli_build_flags_common(struct buildopt *bopt, struct cli_buf *buf) cli_buf_appendf(buf, "-I/usr/local/opt/openssl/include "); cli_buf_appendf(buf, "-I/opt/homebrew/opt/openssl/include "); #endif - if (bopt->single_binary == 0) { - cli_kore_features(bopt, &data, &len); + if (bopt == NULL || bopt->single_binary == 0) { + cli_kore_load_file("features", bopt, &data, &len); cli_buf_append(buf, data, len); cli_buf_appendf(buf, " "); free(data); @@ -1948,7 +1986,7 @@ cli_build_cflags(struct buildopt *bopt) } if (bopt->single_binary) { - cli_kore_features(bopt, &buf, &len); + cli_kore_load_file("features", bopt, &buf, &len); cli_buf_append(bopt->cflags, buf, len); cli_buf_appendf(bopt->cflags, " "); free(buf); @@ -2078,16 +2116,17 @@ cli_flavor_load(void) } static void -cli_kore_features(struct buildopt *bopt, char **out, size_t *outlen) +cli_kore_load_file(const char *name, struct buildopt *bopt, + char **out, size_t *outlen) { int fd; size_t len; char *path, *data; - if (bopt->single_binary) { - (void)cli_vasprintf(&path, "%s/features", object_dir); + if (bopt != NULL && bopt->single_binary) { + (void)cli_vasprintf(&path, "%s/%s", object_dir, name); } else { - (void)cli_vasprintf(&path, "%s/share/kore/features", prefix); + (void)cli_vasprintf(&path, "%s/share/kore/%s", prefix, name); } cli_file_open(path, O_RDONLY, &fd); @@ -2096,7 +2135,7 @@ cli_kore_features(struct buildopt *bopt, char **out, size_t *outlen) free(path); if (len == 0) - fatal("features is empty"); + fatal("%s is empty", name); len--; From d078bdfb95d1b1a18ff83ba713897d05eadb353c Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 22 Sep 2021 22:39:42 +0200 Subject: [PATCH 086/121] Add a gen command to kodev. This will generate an asset file for Kore based on the source file or directory given. This allows other build systems to more easily generate asset files if their compilation steps are different. --- src/cli.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/src/cli.c b/src/cli.c index d7a80f0..f77cb1a 100644 --- a/src/cli.c +++ b/src/cli.c @@ -205,6 +205,8 @@ static void cli_reload(int, char **); static void cli_flavor(int, char **); static void cli_cflags(int, char **); static void cli_ldflags(int, char **); +static void cli_genasset(int, char **); +static void cli_genasset_help(void); #if !defined(KODEV_MINIMAL) static void cli_create(int, char **); @@ -222,6 +224,7 @@ static void cli_file_create(const char *, const char *, size_t); static struct cmd cmds[] = { { "help", "this help text", cli_help }, { "run", "run an application (-fnr implied)", cli_run }, + { "gen", "generate asset file for compilation", cli_genasset }, { "reload", "reload the application (SIGHUP)", cli_reload }, { "info", "show info on kore on this system", cli_info }, { "build", "build an application", cli_build }, @@ -384,6 +387,7 @@ static int source_files_count; static int cxx_files_count; static struct cmd *command = NULL; static int cflags_count = 0; +static int genasset_cmd = 0; static int cxxflags_count = 0; static int ldflags_count = 0; static char *flavor = NULL; @@ -884,6 +888,64 @@ cli_ldflags(int argc, char **argv) free(p); } +static void +cli_genasset(int argc, char **argv) +{ + struct stat st; + struct dirent dp; + char *hdr; + + genasset_cmd = 1; + TAILQ_INIT(&build_options); + (void)cli_buildopt_new("_default"); + + if (getenv("KORE_OBJDIR") == NULL) + object_dir = out_dir; + + if (argv[0] == NULL) + cli_genasset_help(); + + (void)cli_vasprintf(&hdr, "%s/assets.h", out_dir); + (void)unlink(hdr); + + cli_file_open(hdr, O_CREAT | O_TRUNC | O_WRONLY, &s_fd); + cli_file_writef(s_fd, "#ifndef __H_KORE_ASSETS_H\n"); + cli_file_writef(s_fd, "#define __H_KORE_ASSETS_H\n"); + + if (stat(argv[0], &st) == -1) + fatal("%s: %s", argv[0], errno_s); + + if (S_ISDIR(st.st_mode)) { + if (cli_dir_exists(argv[0])) + cli_find_files(argv[0], cli_build_asset); + } else if (S_ISREG(st.st_mode)) { + memset(&dp, 0, sizeof(dp)); + dp.d_type = DT_REG; + (void)snprintf(dp.d_name, sizeof(dp.d_name), "%s", + basename(argv[0])); + cli_build_asset(argv[0], &dp); + } else { + fatal("%s is not a directory or regular file", argv[0]); + } + + cli_file_writef(s_fd, "\n#endif\n"); + cli_file_close(s_fd); +} + +static void +cli_genasset_help(void) +{ + printf("Usage: kodev genasset [source]\n"); + printf("Synopsis:\n"); + printf(" Generates asset file(s) to be used for compilation.\n"); + printf(" The source can be a single file or directory.\n"); + printf("\n"); + printf("This command honors the KODEV_OUTPUT environment variable.\n"); + printf("This command honors the KORE_OBJDIR environment variable.\n"); + + exit(1); +} + #if !defined(KODEV_MINIMAL) static void file_create_python_src(void) @@ -1268,7 +1330,9 @@ cli_build_asset(char *fpath, struct dirent *dp) *--ext = '.'; /* Register the .c file now (cpath is free'd later). */ - cli_add_source_file(name, cpath, opath, &st, BUILD_C); + if (genasset_cmd == 0) + cli_add_source_file(name, cpath, opath, &st, BUILD_C); + free(name); } From 0af7258c302a28276cac067b860bd08f0c64c106 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 4 Oct 2021 19:18:15 +0200 Subject: [PATCH 087/121] Don't include kore config in all builds just yet. --- src/cli.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/cli.c b/src/cli.c index f77cb1a..eda9a04 100644 --- a/src/cli.c +++ b/src/cli.c @@ -679,12 +679,14 @@ cli_build(int argc, char **argv) if (cli_dir_exists(assets_path)) cli_find_files(assets_path, cli_build_asset); - memset(&dp, 0, sizeof(dp)); - dp.d_type = DT_REG; - printf("adding config %s\n", config); - (void)snprintf(dp.d_name, - sizeof(dp.d_name), "builtin_kore.conf"); - cli_build_asset(config, &dp); + if (bopt->single_binary) { + memset(&dp, 0, sizeof(dp)); + dp.d_type = DT_REG; + printf("adding config %s\n", config); + (void)snprintf(dp.d_name, + sizeof(dp.d_name), "builtin_kore.conf"); + cli_build_asset(config, &dp); + } cli_file_writef(s_fd, "\n#endif\n"); cli_file_close(s_fd); From 23b95448cc614aef03946666f2531541941e2e90 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 5 Oct 2021 12:29:50 +0200 Subject: [PATCH 088/121] Hide worker logs behind kore_quiet. --- src/worker.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/worker.c b/src/worker.c index 3c13a8a..1f32f33 100644 --- a/src/worker.c +++ b/src/worker.c @@ -146,7 +146,8 @@ kore_worker_init(void) kw->lb.offset = 0; } - kore_log(LOG_INFO, "starting worker processes"); + if (!kore_quiet) + kore_log(LOG_INFO, "starting worker processes"); /* Now start all the workers. */ id = 1; @@ -174,7 +175,8 @@ kore_worker_init(void) return (KORE_RESULT_ERROR); } - kore_log(LOG_INFO, "all worker processes started"); + if (!kore_quiet) + kore_log(LOG_INFO, "all worker processes started"); return (KORE_RESULT_OK); } From 98d5909b7d57aa68e8514d2fdc2c67d120fc3261 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 20 Oct 2021 11:20:25 +0200 Subject: [PATCH 089/121] bring back python_import and python_path. These were mistakingly removed a while ago. --- src/config.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/config.c b/src/config.c index 2cd66ac..af23f60 100644 --- a/src/config.c +++ b/src/config.c @@ -156,6 +156,8 @@ static int configure_task_threads(char *); #if defined(KORE_USE_PYTHON) static int configure_deployment(char *); +static int configure_python_path(char *); +static int configure_python_import(char *); #endif #if defined(KORE_USE_CURL) @@ -190,6 +192,10 @@ static struct { { "runas", configure_privsep_runas }, { "client_verify", configure_client_verify }, { "client_verify_depth", configure_client_verify_depth }, +#if defined(KORE_USE_PYTHON) + { "python_path", configure_python_path }, + { "python_import", configure_python_import }, +#endif #if !defined(KORE_NO_HTTP) { "route", configure_route }, { "handler", configure_route_handler }, @@ -2011,6 +2017,26 @@ configure_deployment(char *value) return (KORE_RESULT_OK); } +static int +configure_python_path(char *path) +{ + kore_python_path(path); + + return (KORE_RESULT_OK); +} + +static int +configure_python_import(char *module) +{ + char *argv[3]; + + kore_split_string(module, " ", argv, 3); + if (argv[0] == NULL) + return (KORE_RESULT_ERROR); + + kore_module_load(argv[0], argv[1], KORE_MODULE_PYTHON); + return (KORE_RESULT_OK); +} #endif #if defined(KORE_USE_PLATFORM_PLEDGE) From 995b6b8586a6ec44884f6690355baf1b79a1ab71 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 27 Oct 2021 14:34:06 +0200 Subject: [PATCH 090/121] On macos use pkg-config for openssl. --- Makefile | 6 ++---- kodev/Makefile | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index f162ced..e662aa9 100644 --- a/Makefile +++ b/Makefile @@ -132,10 +132,8 @@ ifneq ("$(SANITIZE)", "") endif ifeq ("$(OSNAME)", "darwin") - CFLAGS+=-I/opt/local/include/ -I/usr/local/opt/openssl/include - LDFLAGS+=-L/opt/local/lib -L/usr/local/opt/openssl/lib - CFLAGS+=-I/opt/homebrew/opt/openssl/include - LDFLAGS+=-L/opt/homebrew/opt/openssl/lib + CFLAGS+=$(shell pkg-config openssl --cflags) + LDFLAGS=$(shell pkg-config openssl --libs) S_SRC+=src/bsd.c else ifeq ("$(OSNAME)", "linux") CFLAGS+=-D_GNU_SOURCE=1 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 diff --git a/kodev/Makefile b/kodev/Makefile index b21a6d6..c518cf6 100644 --- a/kodev/Makefile +++ b/kodev/Makefile @@ -28,10 +28,8 @@ endif OSNAME=$(shell uname -s | sed -e 's/[-_].*//g' | tr A-Z a-z) ifeq ("$(OSNAME)", "darwin") - CFLAGS+=-I/opt/local/include/ -I/usr/local/opt/openssl/include - LDFLAGS+=-L/opt/local/lib -L/usr/local/opt/openssl/lib - CFLAGS+=-I/opt/homebrew/opt/openssl/include - LDFLAGS+=-L/opt/homebrew/opt/openssl/lib + CFLAGS+=$(shell pkg-config openssl --cflags) + LDFLAGS=$(shell pkg-config openssl --libs) else ifeq ("$(OSNAME)", "linux") CFLAGS+=-D_GNU_SOURCE=1 endif From c68eb0c7055f36e0ab97d8bb8192447b64f01685 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 27 Oct 2021 17:59:14 +0200 Subject: [PATCH 091/121] make sure we add to LDFLAGS --- Makefile | 2 +- kodev/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e662aa9..9f65b8b 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,7 @@ endif ifeq ("$(OSNAME)", "darwin") CFLAGS+=$(shell pkg-config openssl --cflags) - LDFLAGS=$(shell pkg-config openssl --libs) + LDFLAGS+=$(shell pkg-config openssl --libs) S_SRC+=src/bsd.c else ifeq ("$(OSNAME)", "linux") CFLAGS+=-D_GNU_SOURCE=1 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 diff --git a/kodev/Makefile b/kodev/Makefile index c518cf6..8fb6027 100644 --- a/kodev/Makefile +++ b/kodev/Makefile @@ -29,7 +29,7 @@ endif OSNAME=$(shell uname -s | sed -e 's/[-_].*//g' | tr A-Z a-z) ifeq ("$(OSNAME)", "darwin") CFLAGS+=$(shell pkg-config openssl --cflags) - LDFLAGS=$(shell pkg-config openssl --libs) + LDFLAGS+=$(shell pkg-config openssl --libs) else ifeq ("$(OSNAME)", "linux") CFLAGS+=-D_GNU_SOURCE=1 endif From fa97544f01e9c9bf0a3f708334ba777e51c80ddc Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 27 Oct 2021 22:27:42 +0200 Subject: [PATCH 092/121] Handle PGRES_PIPELINE_* for PQResult() --- src/pgsql.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pgsql.c b/src/pgsql.c index e58ea09..9177528 100644 --- a/src/pgsql.c +++ b/src/pgsql.c @@ -772,6 +772,8 @@ pgsql_read_result(struct kore_pgsql *pgsql) } switch (PQresultStatus(pgsql->result)) { + case PGRES_PIPELINE_SYNC: + case PGRES_PIPELINE_ABORTED: case PGRES_COPY_OUT: case PGRES_COPY_IN: case PGRES_NONFATAL_ERROR: From 960730a062c23f30321956169fa6c1b97ff669c4 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 27 Oct 2021 22:28:08 +0200 Subject: [PATCH 093/121] On MacOS put the OpenSSL flags under FEATURES_INC. Use this to pick them up automatically for kodev. --- Makefile | 4 +++- src/cli.c | 6 ------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 9f65b8b..a69c90b 100644 --- a/Makefile +++ b/Makefile @@ -132,8 +132,10 @@ ifneq ("$(SANITIZE)", "") endif ifeq ("$(OSNAME)", "darwin") - CFLAGS+=$(shell pkg-config openssl --cflags) + OSSL_INCL=$(shell pkg-config openssl --cflags) + CFLAGS+=$(OSSL_INCL) LDFLAGS+=$(shell pkg-config openssl --libs) + FEATURES_INC+=$(OSSL_INCL) S_SRC+=src/bsd.c else ifeq ("$(OSNAME)", "linux") CFLAGS+=-D_GNU_SOURCE=1 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 diff --git a/src/cli.c b/src/cli.c index eda9a04..01bccad 100644 --- a/src/cli.c +++ b/src/cli.c @@ -2017,12 +2017,6 @@ cli_build_flags_common(struct buildopt *bopt, struct cli_buf *buf) else cli_buf_appendf(buf, "-I%s/include ", bopt->kore_source); -#if defined(__MACH__) - /* Add default openssl include path from homebrew / ports under OSX. */ - cli_buf_appendf(buf, "-I/opt/local/include "); - cli_buf_appendf(buf, "-I/usr/local/opt/openssl/include "); - cli_buf_appendf(buf, "-I/opt/homebrew/opt/openssl/include "); -#endif if (bopt == NULL || bopt->single_binary == 0) { cli_kore_load_file("features", bopt, &data, &len); cli_buf_append(buf, data, len); From 5962a945046c37d9594be8c7aefd6dba20e32c33 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 27 Oct 2021 22:39:29 +0200 Subject: [PATCH 094/121] wrap pipeline in PG_VERSION_NUM >= 140000 --- src/pgsql.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pgsql.c b/src/pgsql.c index 9177528..55c7fbf 100644 --- a/src/pgsql.c +++ b/src/pgsql.c @@ -772,8 +772,10 @@ pgsql_read_result(struct kore_pgsql *pgsql) } switch (PQresultStatus(pgsql->result)) { +#if PG_VERSION_NUM >= 140000 case PGRES_PIPELINE_SYNC: case PGRES_PIPELINE_ABORTED: +#endif case PGRES_COPY_OUT: case PGRES_COPY_IN: case PGRES_NONFATAL_ERROR: From 01e85fd717e492b39958445495a18a9adfbb6ac6 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 3 Nov 2021 15:19:43 +0100 Subject: [PATCH 095/121] Two small python improvements. - Decrement bytes count when python_cmsg_to_list() fails. - Use correct define for PYSOCKET_TYPE_RECVFROM. --- src/python.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python.c b/src/python.c index 08f0225..0c84d42 100644 --- a/src/python.c +++ b/src/python.c @@ -3545,8 +3545,10 @@ pysocket_async_recv(struct pysocket_op *op) return (NULL); case PYSOCKET_TYPE_RECVMSG: socklen = msg.msg_namelen; - if ((list = python_cmsg_to_list(&msg)) == NULL) + if ((list = python_cmsg_to_list(&msg)) == NULL) { + Py_DECREF(bytes); return (NULL); + } break; case PYSOCKET_TYPE_RECVFROM: break; @@ -3559,7 +3561,7 @@ pysocket_async_recv(struct pysocket_op *op) port = ntohs(op->sendaddr.ipv4.sin_port); ip = inet_ntoa(op->sendaddr.ipv4.sin_addr); - if (op->type == PYSOCKET_TYPE_RECV) + if (op->type == PYSOCKET_TYPE_RECVFROM) tuple = Py_BuildValue("(sHN)", ip, port, bytes); else tuple = Py_BuildValue("(sHNN)", ip, port, bytes, list); From a7e1d1d22aa5576207358bbdff37ccb1127ff461 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 3 Nov 2021 17:16:34 +0100 Subject: [PATCH 096/121] Remove keymgr cleaning keys logs. its such a dumb log message, obviously its going to cleanup keys. --- src/keymgr.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/keymgr.c b/src/keymgr.c index 535ef81..a8c9258 100644 --- a/src/keymgr.c +++ b/src/keymgr.c @@ -364,9 +364,6 @@ kore_keymgr_cleanup(int final) { struct key *key, *next; - if (final && !kore_quiet) - kore_log(LOG_INFO, "cleaning up keys"); - if (initialized == 0) return; From 155c7dfbde713a244a68707c680d117660eb9c0d Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 3 Nov 2021 17:22:53 +0100 Subject: [PATCH 097/121] prefix fatal messages with FATAL --- src/utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.c b/src/utils.c index eddb754..bf19eeb 100644 --- a/src/utils.c +++ b/src/utils.c @@ -651,7 +651,7 @@ fatal_log(const char *fmt, va_list args) char buf[2048]; (void)vsnprintf(buf, sizeof(buf), fmt, args); - kore_log(LOG_ERR, "%s", buf); + kore_log(LOG_ERR, "FATAL: %s", buf); if (worker != NULL && worker->id == KORE_WORKER_KEYMGR) kore_keymgr_cleanup(1); From efc7b3d9a63845bf5fd8451c49346f935efaa030 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 3 Nov 2021 17:23:05 +0100 Subject: [PATCH 098/121] Improve how the parent handles workers. - Make sure we drain the worker log channel if it dies so we can flush out any lingering log messages. - Get rid of the raise() in the parent to signal ourselves we should terminate. Instead depend on the new kore_quit. - Always attempt to reap children one way or the other. --- include/kore/kore.h | 1 + src/kore.c | 27 ++++++++++++------- src/msg.c | 16 ++--------- src/worker.c | 66 ++++++++++++++++++++++----------------------- 4 files changed, 54 insertions(+), 56 deletions(-) diff --git a/include/kore/kore.h b/include/kore/kore.h index b22becd..807e134 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -709,6 +709,7 @@ extern char *config_file; #endif extern pid_t kore_pid; +extern int kore_quit; extern int kore_quiet; extern int kore_debug; extern int skip_chroot; diff --git a/src/kore.c b/src/kore.c index b692752..18ab5d0 100644 --- a/src/kore.c +++ b/src/kore.c @@ -54,6 +54,7 @@ volatile sig_atomic_t sig_recv; struct kore_server_list kore_servers; u_int8_t nlisteners; int kore_argc = 0; +int kore_quit = 0; pid_t kore_pid = -1; u_int16_t cpu_count = 1; int kore_debug = 0; @@ -80,6 +81,7 @@ static void version(void); static void kore_write_kore_pid(void); static void kore_proctitle_setup(void); static void kore_server_sslstart(void); +static void kore_server_shutdown(void); static void kore_server_start(int, char *[]); static void kore_call_parent_configure(int, char **); @@ -271,11 +273,7 @@ main(int argc, char *argv[]) kore_signal_setup(); kore_server_start(argc, argv); - - if (!kore_quiet) - kore_log(LOG_INFO, "server shutting down"); - - kore_worker_shutdown(); + kore_server_shutdown(); rcall = kore_runtime_getcall(parent_teardown_hook); if (rcall != NULL) { @@ -858,7 +856,7 @@ kore_server_start(int argc, char *argv[]) u_int32_t tmp; struct kore_server *srv; u_int64_t netwait; - int quit, last_sig; + int last_sig; #if defined(KORE_SINGLE_BINARY) struct kore_runtime_call *rcall; #endif @@ -951,7 +949,6 @@ kore_server_start(int argc, char *argv[]) kore_platform_event_init(); kore_msg_parent_init(); - quit = 0; worker_max_connections = tmp; kore_timer_init(); @@ -963,7 +960,7 @@ kore_server_start(int argc, char *argv[]) kore_msg_unregister(KORE_PYTHON_SEND_OBJ); #endif - while (quit != 1) { + while (kore_quit != 1) { if (sig_recv != 0) { last_sig = sig_recv; @@ -975,7 +972,7 @@ kore_server_start(int argc, char *argv[]) case SIGINT: case SIGQUIT: case SIGTERM: - quit = 1; + kore_quit = 1; kore_worker_dispatch_signal(sig_recv); continue; case SIGUSR1: @@ -998,8 +995,20 @@ kore_server_start(int argc, char *argv[]) kore_platform_event_wait(netwait); kore_connection_prune(KORE_CONNECTION_PRUNE_DISCONNECT); kore_timer_run(kore_time_ms()); + kore_worker_reap(); } + kore_worker_dispatch_signal(SIGQUIT); +} + +static void +kore_server_shutdown(void) +{ + if (!kore_quiet) + kore_log(LOG_INFO, "server shutting down"); + + kore_worker_shutdown(); + #if !defined(KORE_NO_HTTP) kore_accesslog_gather(NULL, kore_time_ms(), 1); #endif diff --git a/src/msg.c b/src/msg.c index bc259c1..69f6404 100644 --- a/src/msg.c +++ b/src/msg.c @@ -33,9 +33,8 @@ struct msg_type { }; static struct msg_type *msg_type_lookup(u_int8_t); -static int msg_recv_packet(struct netbuf *); static int msg_recv_data(struct netbuf *); -static void msg_disconnected_parent(struct connection *); +static int msg_recv_packet(struct netbuf *); static void msg_disconnected_worker(struct connection *); static void msg_type_shutdown(struct kore_msg *, const void *); @@ -113,7 +112,6 @@ kore_msg_worker_init(void) worker->msg[1]->write = net_write; worker->msg[1]->proto = CONN_PROTO_MSG; worker->msg[1]->state = CONN_STATE_ESTABLISHED; - worker->msg[1]->disconnect = msg_disconnected_parent; worker->msg[1]->handle = kore_connection_handle; worker->msg[1]->evt.flags = KORE_EVENT_WRITE; @@ -245,16 +243,6 @@ msg_recv_data(struct netbuf *nb) return (KORE_RESULT_OK); } -static void -msg_disconnected_parent(struct connection *c) -{ - if (!kore_quiet) - kore_log(LOG_ERR, "parent gone, shutting down"); - - if (kill(worker->pid, SIGQUIT) == -1) - kore_log(LOG_ERR, "failed to send SIGQUIT: %s", errno_s); -} - static void msg_disconnected_worker(struct connection *c) { @@ -269,7 +257,7 @@ msg_type_shutdown(struct kore_msg *msg, const void *data) "shutdown requested by worker %u, going down", msg->src); } - (void)raise(SIGQUIT); + kore_quit = 1; } #if !defined(KORE_NO_HTTP) diff --git a/src/worker.c b/src/worker.c index 1f32f33..65e8642 100644 --- a/src/worker.c +++ b/src/worker.c @@ -297,9 +297,13 @@ kore_worker_shutdown(void) kw->pid = 0; kw->running = 0; + kw->msg[0]->evt.flags |= KORE_EVENT_READ; + net_recv_flush(kw->msg[0]); + if (!kore_quiet) { - kore_log(LOG_NOTICE, "worker %s exited", - kore_worker_name(kw->id)); + kore_log(LOG_NOTICE, + "worker %s exited (%d)", + kore_worker_name(kw->id), status); } } } @@ -610,7 +614,6 @@ kore_worker_entry(struct kore_worker *kw) kore_free(rcall); } - kore_msg_send(KORE_MSG_PARENT, KORE_MSG_SHUTDOWN, NULL, 0); kore_server_cleanup(); kore_platform_event_cleanup(); @@ -642,24 +645,19 @@ kore_worker_reap(void) pid_t pid; int status; - for (;;) { - pid = waitpid(WAIT_ANY, &status, WNOHANG); + pid = waitpid(WAIT_ANY, &status, WNOHANG); - if (pid == -1) { - if (errno == ECHILD) - return; - if (errno == EINTR) - continue; - kore_log(LOG_ERR, - "failed to wait for children: %s", errno_s); + if (pid == -1) { + if (errno == ECHILD || errno == EINTR) return; - } - - if (pid == 0) - return; - - worker_reaper(pid, status); + kore_log(LOG_ERR, "%s: waitpid(): %s", __func__, errno_s); + return; } + + if (pid == 0) + return; + + worker_reaper(pid, status); } void @@ -785,6 +783,9 @@ worker_reaper(pid_t pid, int status) if (kw->pid != pid) continue; + kw->msg[0]->evt.flags |= KORE_EVENT_READ; + net_recv_flush(kw->msg[0]); + if (!kore_quiet) { kore_log(LOG_NOTICE, "worker %s (%d) exited with status %d", @@ -819,10 +820,7 @@ worker_reaper(pid_t pid, int status) kore_log(LOG_CRIT, "keymgr or acme process gone, stopping"); kw->pid = 0; - if (raise(SIGTERM) != 0) { - kore_log(LOG_WARNING, - "failed to raise SIGTERM signal"); - } + kore_quit = 1; break; } @@ -844,22 +842,24 @@ worker_reaper(pid_t pid, int status) kw->pid = 0; kore_log(LOG_NOTICE, "worker policy is 'terminate', stopping"); - if (raise(SIGTERM) != 0) { - kore_log(LOG_WARNING, - "failed to raise SIGTERM signal"); - } + kore_quit = 1; break; } - kore_log(LOG_NOTICE, "restarting worker %d", kw->id); - kw->restarted = 1; - kore_msg_parent_remove(kw); + if (kore_quit == 0) { + kore_log(LOG_NOTICE, "restarting worker %d", kw->id); + kw->restarted = 1; + kore_msg_parent_remove(kw); - if (!kore_worker_spawn(idx, kw->id, kw->cpu)) - (void)raise(SIGQUIT); + if (!kore_worker_spawn(idx, kw->id, kw->cpu)) { + kore_quit = 1; + kore_log(LOG_ERR, "failed to restart worker"); + } else { + kore_msg_parent_add(kw); + } - kore_msg_parent_add(kw); - break; + break; + } } } From 0d39e5c62a4eb11ac476966b870475d4074195bc Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 29 Nov 2021 15:43:43 +0100 Subject: [PATCH 099/121] Allow setting of TARGET_PLATFORM. This overrides the linux seccomp building, useful for cross compiling Kore to other linux architectures. --- misc/linux-platform.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/misc/linux-platform.sh b/misc/linux-platform.sh index 818e53b..01569e6 100755 --- a/misc/linux-platform.sh +++ b/misc/linux-platform.sh @@ -2,7 +2,12 @@ # # Linux specific defines and system call maps. -PLATFORM=$(uname -m) +if [ -z "$TARGET_PLATFORM" ]; then + PLATFORM=$(uname -m) +else + PLATFORM=$TARGET_PLATFORM +fi + BASE=$(dirname $0) case "$PLATFORM" in From 86ecb85f035ccbe06f3a3a38985eba7a0f494912 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Thu, 2 Dec 2021 19:33:20 +0100 Subject: [PATCH 100/121] use correct privsep name for acme --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index af23f60..41f8795 100644 --- a/src/config.c +++ b/src/config.c @@ -973,7 +973,7 @@ configure_privsep(char *options) } else if (!strcmp(argv[0], "keymgr")) { current_privsep = &keymgr_privsep; #if defined(KORE_USE_ACME) - } else if (!strcmp(argv[0], "keymgr")) { + } else if (!strcmp(argv[0], "acme")) { current_privsep = &acme_privsep; #endif } else { From 5ac62b17bce8dae24c6cd9ee2c89a4e7b32484b6 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Thu, 2 Dec 2021 21:58:13 +0100 Subject: [PATCH 101/121] Python coro under-the-hood improvements. - Change python coroutine id to a uint64_t. - Add kore.task_id() to return active coro its id. --- include/kore/python_methods.h | 4 +++- src/python.c | 21 +++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index 8ca9c62..1160130 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -18,7 +18,7 @@ #define CORO_STATE_SUSPENDED 2 struct python_coro { - u_int32_t id; + u_int64_t id; int state; int killed; PyObject *obj; @@ -45,6 +45,7 @@ static PyObject *python_kore_queue(PyObject *, PyObject *); static PyObject *python_kore_worker(PyObject *, PyObject *); static PyObject *python_kore_tracer(PyObject *, PyObject *); static PyObject *python_kore_fatalx(PyObject *, PyObject *); +static PyObject *python_kore_task_id(PyObject *, PyObject *); static PyObject *python_kore_setname(PyObject *, PyObject *); static PyObject *python_kore_suspend(PyObject *, PyObject *); static PyObject *python_kore_shutdown(PyObject *, PyObject *); @@ -96,6 +97,7 @@ static struct PyMethodDef pykore_methods[] = { METHOD("tracer", python_kore_tracer, METH_VARARGS), METHOD("fatal", python_kore_fatal, METH_VARARGS), METHOD("fatalx", python_kore_fatalx, METH_VARARGS), + METHOD("task_id", python_kore_task_id, METH_NOARGS), METHOD("setname", python_kore_setname, METH_VARARGS), METHOD("suspend", python_kore_suspend, METH_VARARGS), METHOD("shutdown", python_kore_shutdown, METH_NOARGS), diff --git a/src/python.c b/src/python.c index 0c84d42..55ec7ab 100644 --- a/src/python.c +++ b/src/python.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -1173,7 +1174,7 @@ python_coro_trace(const char *label, struct python_coro *coro) kore_log(LOG_NOTICE, "coro '%s' %s <%s> @ [%s:%d]", coro->name, label, func, fname, line); } else { - kore_log(LOG_NOTICE, "coro %u %s <%s> @ [%s:%d]", + kore_log(LOG_NOTICE, "coro %" PRIu64 " %s <%s> @ [%s:%d]", coro->id, label, func, fname, line); } } @@ -1945,16 +1946,28 @@ python_kore_task_create(PyObject *self, PyObject *args) coro = python_coro_create(obj, NULL); Py_INCREF(obj); - return (PyLong_FromUnsignedLong(coro->id)); + return (PyLong_FromUnsignedLongLong(coro->id)); +} + +static PyObject * +python_kore_task_id(PyObject *self, PyObject *args) +{ + if (coro_running == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "no coroutine active"); + return (NULL); + } + + return (PyLong_FromUnsignedLongLong(coro_running->id)); } static PyObject * python_kore_task_kill(PyObject *self, PyObject *args) { - u_int32_t id; + u_int64_t id; struct python_coro *coro, *active; - if (!PyArg_ParseTuple(args, "I", &id)) + if (!PyArg_ParseTuple(args, "K", &id)) return (NULL); if (coro_running != NULL && coro_running->id == id) { From a9ee15bff6f5c154647ec994771645c34345fb4c Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Thu, 2 Dec 2021 22:47:17 +0100 Subject: [PATCH 102/121] Improve closing of a kore.socket() in Python API. When a kore.socket() is closed from any coroutine, make sure any other coroutines waiting on events on the socket are awoken so they properly can return errors. --- src/python.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python.c b/src/python.c index 55ec7ab..25ad9b7 100644 --- a/src/python.c +++ b/src/python.c @@ -3197,6 +3197,7 @@ pysocket_close(struct pysocket *sock, PyObject *args) } sock->fd = -1; + sock->event.evt.handle(&sock->event, 1); Py_RETURN_TRUE; } From 6b2609c2b861b9e0a99390dd8acc4dca1dc515f1 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Dec 2021 14:16:58 +0100 Subject: [PATCH 103/121] Allow DELETE for kore.httpclient() to have body. The DELETE method could have an HTTP body, so allow it in the kore.httpclient() python call. --- src/python.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/python.c b/src/python.c index 25ad9b7..99e1c59 100644 --- a/src/python.c +++ b/src/python.c @@ -6292,15 +6292,20 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) switch (m) { case HTTP_METHOD_GET: case HTTP_METHOD_HEAD: - case HTTP_METHOD_DELETE: case HTTP_METHOD_OPTIONS: break; case HTTP_METHOD_PUT: case HTTP_METHOD_POST: case HTTP_METHOD_PATCH: + case HTTP_METHOD_DELETE: length = -1; if (kwargs == NULL) { + if (m == HTTP_METHOD_DELETE) { + length = 0; + break; + } + PyErr_Format(PyExc_RuntimeError, "no keyword arguments given, but body expected ", http_method_text(m)); From 480e589dd5f36243dd06fbcf0e2852682bd21802 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Dec 2021 14:43:52 +0100 Subject: [PATCH 104/121] The DELETE method may have a request body. --- src/http.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/http.c b/src/http.c index 17b7699..6c21990 100644 --- a/src/http.c +++ b/src/http.c @@ -925,6 +925,11 @@ http_header_recv(struct netbuf *nb) if (!http_request_header_uint64(req, "content-length", &req->content_length)) { + if (req->method == HTTP_METHOD_DELETE) { + req->flags |= HTTP_REQUEST_COMPLETE; + return (KORE_RESULT_OK); + } + req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, HTTP_STATUS_LENGTH_REQUIRED); @@ -1992,7 +1997,7 @@ http_request_new(struct connection *c, const char *host, flags |= HTTP_REQUEST_COMPLETE; } else if (!strcasecmp(method, "delete")) { m = HTTP_METHOD_DELETE; - flags |= HTTP_REQUEST_COMPLETE; + flags |= HTTP_REQUEST_EXPECT_BODY; } else if (!strcasecmp(method, "post")) { m = HTTP_METHOD_POST; flags |= HTTP_REQUEST_EXPECT_BODY; From 93bf18be81aa9e2a0353416b63b276d74ffdc49c Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Dec 2021 14:44:07 +0100 Subject: [PATCH 105/121] Handle DELETE maybe having a request body. --- src/curl.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/curl.c b/src/curl.c index ed9b396..9d73a71 100644 --- a/src/curl.c +++ b/src/curl.c @@ -407,18 +407,17 @@ kore_curl_http_setup(struct kore_curl *client, int method, const void *data, switch (method) { case HTTP_METHOD_GET: + case HTTP_METHOD_OPTIONS: break; case HTTP_METHOD_HEAD: curl_easy_setopt(client->handle, CURLOPT_NOBODY, 1); break; - case HTTP_METHOD_DELETE: - case HTTP_METHOD_OPTIONS: - break; case HTTP_METHOD_PUT: has_body = 1; curl_easy_setopt(client->handle, CURLOPT_UPLOAD, 1); break; case HTTP_METHOD_PATCH: + case HTTP_METHOD_DELETE: mname = http_method_text(method); /* fallthrough */ case HTTP_METHOD_POST: From 06803e25925486ecef35bab09d091d68b38b2720 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Dec 2021 21:21:21 +0100 Subject: [PATCH 106/121] Get kore to at least build with openssl 3 --- include/kore/kore.h | 5 +++++ src/cli.c | 6 ++++++ src/keymgr.c | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/include/kore/kore.h b/include/kore/kore.h index 807e134..943ea8f 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -79,6 +79,11 @@ extern int daemon(int, int); #endif #endif +/* Ignore deprecation warnings for OpenSSL 3 for now. */ +#if defined(OPENSSL_VERSION_MAJOR) +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + #if defined(__OpenBSD__) #define KORE_USE_PLATFORM_PLEDGE 1 #endif diff --git a/src/cli.c b/src/cli.c index 01bccad..d45680f 100644 --- a/src/cli.c +++ b/src/cli.c @@ -26,6 +26,12 @@ #include #include #include + +/* Ignore deprecation warnings for OpenSSL 3 for now. */ +#if defined(OPENSSL_VERSION_MAJOR) +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + #endif #include diff --git a/src/keymgr.c b/src/keymgr.c index a8c9258..1c8d860 100644 --- a/src/keymgr.c +++ b/src/keymgr.c @@ -689,7 +689,12 @@ keymgr_rsa_encrypt(struct kore_msg *msg, const void *data, struct key *key) u_int8_t buf[1024]; req = (const struct kore_keyreq *)data; + +#if defined(OPENSSL_VERSION_MAJOR) + rsa = EVP_PKEY_get0(key->pkey); +#else rsa = EVP_PKEY_get0_RSA(key->pkey); +#endif keylen = RSA_size(rsa); if (req->data_len > keylen || keylen > sizeof(buf)) @@ -713,7 +718,12 @@ keymgr_ecdsa_sign(struct kore_msg *msg, const void *data, struct key *key) u_int8_t sig[1024]; req = (const struct kore_keyreq *)data; + +#if defined(OPENSSL_VERSION_MAJOR) + ec = EVP_PKEY_get0(key->pkey); +#else ec = EVP_PKEY_get0_EC_KEY(key->pkey); +#endif len = ECDSA_size(ec); if (req->data_len > len || len > sizeof(sig)) From 0337af20670ae1001bf0d0945c8df9d4127487b7 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Dec 2021 21:27:11 +0100 Subject: [PATCH 107/121] note on openssl 3 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ed18dc6..2e4e038 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Clone this repository or get the latest release at [https://kore.io/releases/4.1 Requirements * openssl 1.1.1 or libressl 3.x +(note: openssl 3.0.0 is currently *not* supported) Requirement for asynchronous curl (optional) * libcurl (7.64.0 or higher) From a54f806978df92890b6d5d0ea328bcd3382a37be Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 6 Dec 2021 23:58:13 +0100 Subject: [PATCH 108/121] Don't let kore build with openssl 3 again. The whole privilege separation breaks with OpenSSL 3, even if it builds. I guess it is somewhat time to start on donutls. --- include/kore/kore.h | 5 ----- src/cli.c | 6 ------ src/keymgr.c | 10 ---------- 3 files changed, 21 deletions(-) diff --git a/include/kore/kore.h b/include/kore/kore.h index 943ea8f..807e134 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -79,11 +79,6 @@ extern int daemon(int, int); #endif #endif -/* Ignore deprecation warnings for OpenSSL 3 for now. */ -#if defined(OPENSSL_VERSION_MAJOR) -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - #if defined(__OpenBSD__) #define KORE_USE_PLATFORM_PLEDGE 1 #endif diff --git a/src/cli.c b/src/cli.c index d45680f..01bccad 100644 --- a/src/cli.c +++ b/src/cli.c @@ -26,12 +26,6 @@ #include #include #include - -/* Ignore deprecation warnings for OpenSSL 3 for now. */ -#if defined(OPENSSL_VERSION_MAJOR) -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - #endif #include diff --git a/src/keymgr.c b/src/keymgr.c index 1c8d860..a8c9258 100644 --- a/src/keymgr.c +++ b/src/keymgr.c @@ -689,12 +689,7 @@ keymgr_rsa_encrypt(struct kore_msg *msg, const void *data, struct key *key) u_int8_t buf[1024]; req = (const struct kore_keyreq *)data; - -#if defined(OPENSSL_VERSION_MAJOR) - rsa = EVP_PKEY_get0(key->pkey); -#else rsa = EVP_PKEY_get0_RSA(key->pkey); -#endif keylen = RSA_size(rsa); if (req->data_len > keylen || keylen > sizeof(buf)) @@ -718,12 +713,7 @@ keymgr_ecdsa_sign(struct kore_msg *msg, const void *data, struct key *key) u_int8_t sig[1024]; req = (const struct kore_keyreq *)data; - -#if defined(OPENSSL_VERSION_MAJOR) - ec = EVP_PKEY_get0(key->pkey); -#else ec = EVP_PKEY_get0_EC_KEY(key->pkey); -#endif len = ECDSA_size(ec); if (req->data_len > len || len > sizeof(sig)) From a641c293017e22679a43699670d934ffe833add2 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sat, 11 Dec 2021 20:34:44 +0100 Subject: [PATCH 109/121] Add SYS_newfstatat to keymgr seccomp rules. From Aurelien Jarno via patches@ --- src/keymgr.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/keymgr.c b/src/keymgr.c index a8c9258..257b5e4 100644 --- a/src/keymgr.c +++ b/src/keymgr.c @@ -87,6 +87,9 @@ static struct sock_filter filter_keymgr[] = { KORE_SYSCALL_ALLOW(fstat), #if defined(SYS_fstat64) KORE_SYSCALL_ALLOW(fstat64), +#endif +#if defined(SYS_newfstatat) + KORE_SYSCALL_ALLOW(newfstatat), #endif KORE_SYSCALL_ALLOW(futex), KORE_SYSCALL_ALLOW(writev), From 685f50456483fc9b3dde26a73a44c809fb02df5b Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sat, 11 Dec 2021 21:59:41 +0100 Subject: [PATCH 110/121] Log if no TLS server was configured --- src/connection.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/connection.c b/src/connection.c index 0950e29..9532d0b 100644 --- a/src/connection.c +++ b/src/connection.c @@ -258,6 +258,12 @@ kore_connection_handle(struct connection *c) switch (c->state) { case CONN_STATE_TLS_SHAKE: + if (primary_dom == NULL) { + kore_log(LOG_NOTICE, + "TLS handshake but no TLS configured on server"); + return (KORE_RESULT_ERROR); + } + if (primary_dom->ssl_ctx == NULL) { kore_log(LOG_NOTICE, "TLS configuration for %s not yet complete", From cf6cf5f820e4531c548bca18806570730207a429 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sat, 11 Dec 2021 22:35:37 +0100 Subject: [PATCH 111/121] Always align pool element lengths properly. Enforce 8-byte alignment on the pool element lengths. --- src/pool.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pool.c b/src/pool.c index ea2edf2..07383bb 100644 --- a/src/pool.c +++ b/src/pool.c @@ -47,6 +47,8 @@ kore_pool_init(struct kore_pool *pool, const char *name, if ((pool->name = strdup(name)) == NULL) fatal("kore_pool_init: strdup %s", errno_s); + len = (len + (8 - 1)) & ~(8 - 1); + pool->lock = 0; pool->elms = 0; pool->inuse = 0; From ca4ffa457c0f93d20e70de92b2f64f4e291207aa Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sat, 11 Dec 2021 22:36:31 +0100 Subject: [PATCH 112/121] Add a kore_x509_issuer_name() function. Rework the underlying X509_NAME juggling into a kore_x509name_foreach() so that it can be called for multiple locations. --- include/kore/kore.h | 4 ++ src/utils.c | 110 ++++++++++++++++++++++++++++++++------------ 2 files changed, 84 insertions(+), 30 deletions(-) diff --git a/include/kore/kore.h b/include/kore/kore.h index 807e134..a4ce42c 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -910,7 +910,11 @@ char *kore_read_line(FILE *, char *, size_t); EVP_PKEY *kore_rsakey_load(const char *); EVP_PKEY *kore_rsakey_generate(const char *); +int kore_x509_issuer_name(struct connection *, char **, int); int kore_x509_subject_name(struct connection *, char **, int); +int kore_x509name_foreach(X509_NAME *, int, void *, + int (*)(void *, int, int, const char *, + const void *, size_t, int)); #if !defined(KORE_NO_HTTP) void kore_websocket_handshake(struct http_request *, diff --git a/src/utils.c b/src/utils.c index bf19eeb..756cf5e 100644 --- a/src/utils.c +++ b/src/utils.c @@ -54,6 +54,9 @@ static int utils_base64_encode(const void *, size_t, char **, const char *, int); static int utils_base64_decode(const char *, u_int8_t **, size_t *, const char *, int); +static int utils_x509name_tobuf(void *, int, int, const char *, + const void *, size_t, int); + static char b64_table[] = \ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -545,35 +548,72 @@ kore_worker_name(int id) return (buf); } +int +kore_x509_issuer_name(struct connection *c, char **out, int flags) +{ + struct kore_buf buf; + X509_NAME *name; + + if ((name = X509_get_issuer_name(c->cert)) == NULL) + return (KORE_RESULT_ERROR); + + kore_buf_init(&buf, 1024); + + if (!kore_x509name_foreach(name, flags, &buf, utils_x509name_tobuf)) { + kore_buf_cleanup(&buf); + return (KORE_RESULT_ERROR); + } + + *out = kore_buf_stringify(&buf, NULL); + + buf.offset = 0; + buf.data = NULL; + + return (KORE_RESULT_OK); +} + int kore_x509_subject_name(struct connection *c, char **out, int flags) { struct kore_buf buf; + X509_NAME *name; + + if ((name = X509_get_subject_name(c->cert)) == NULL) + return (KORE_RESULT_ERROR); + + kore_buf_init(&buf, 1024); + + if (!kore_x509name_foreach(name, flags, &buf, utils_x509name_tobuf)) { + kore_buf_cleanup(&buf); + return (KORE_RESULT_ERROR); + } + + *out = kore_buf_stringify(&buf, NULL); + + buf.offset = 0; + buf.data = NULL; + + return (KORE_RESULT_OK); +} + +int +kore_x509name_foreach(X509_NAME *name, int flags, void *udata, + int (*cb)(void *, int, int, const char *, const void *, size_t, int)) +{ u_int8_t *data; ASN1_STRING *astr; - X509_NAME *name; X509_NAME_ENTRY *entry; const char *field; - int ret, idx, namelen, nid, len; + int islast, ret, idx, namelen, nid, len; data = NULL; ret = KORE_RESULT_ERROR; - kore_buf_init(&buf, 1024); - - if (c->cert == NULL) - goto cleanup; - - if ((name = X509_get_subject_name(c->cert)) == NULL) - goto cleanup; - - namelen = X509_NAME_entry_count(name); - if (namelen == 0) + if ((namelen = X509_NAME_entry_count(name)) == 0) goto cleanup; for (idx = 0; idx < namelen; idx++) { - entry = X509_NAME_get_entry(name, idx); - if (entry == NULL) + if ((entry = X509_NAME_get_entry(name, idx)) == NULL) goto cleanup; nid = OBJ_obj2nid(X509_NAME_ENTRY_get_object(entry)); @@ -587,33 +627,24 @@ kore_x509_subject_name(struct connection *c, char **out, int flags) if ((len = ASN1_STRING_to_UTF8(&data, astr)) < 0) goto cleanup; - if (flags & KORE_X509_COMMON_NAME_ONLY) { - if (nid == NID_commonName) { - kore_buf_append(&buf, data, len); - break; - } - } else { - kore_buf_appendf(&buf, "%s=", field); - kore_buf_append(&buf, data, len); - if (idx != (namelen - 1)) - kore_buf_appendf(&buf, " "); - } + if (idx != (namelen - 1)) + islast = 0; + else + islast = 1; + + if (!cb(udata, islast, nid, field, data, len, flags)) + goto cleanup; OPENSSL_free(data); data = NULL; } ret = KORE_RESULT_OK; - *out = kore_buf_stringify(&buf, NULL); - - buf.offset = 0; - buf.data = NULL; cleanup: if (data != NULL) OPENSSL_free(data); - kore_buf_cleanup(&buf); return (ret); } @@ -657,6 +688,25 @@ fatal_log(const char *fmt, va_list args) kore_keymgr_cleanup(1); } +static int +utils_x509name_tobuf(void *udata, int islast, int nid, const char *field, + const void *data, size_t len, int flags) +{ + struct kore_buf *buf = udata; + + if (flags & KORE_X509_COMMON_NAME_ONLY) { + if (nid == NID_commonName) + kore_buf_append(buf, data, len); + } else { + kore_buf_appendf(buf, "%s=", field); + kore_buf_append(buf, data, len); + if (!islast) + kore_buf_appendf(buf, " "); + } + + return (KORE_RESULT_OK); +} + static int utils_base64_encode(const void *data, size_t len, char **out, const char *table, int flags) From 774cc56ed257aed6f744768dd8d35ac4fa5bdb04 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sat, 11 Dec 2021 22:37:15 +0100 Subject: [PATCH 113/121] Python: Add an req.connection.x509dict This dictionary for now only contains the subject and issuer names from the client certificate (if one was provided) with their X509_NAME components. Eg: { "issuer": { "C": "SE", "O": "kore autogen: x509name-test", "CN": "localhost" }, "subject": { "C": "SE", "O": "kore autogen: x509name-test", "CN": "localhost" } } --- include/kore/python_methods.h | 2 + src/python.c | 93 +++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index 1160130..72112d0 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -696,11 +696,13 @@ static PyObject *pyconnection_get_fd(struct pyconnection *, void *); static PyObject *pyconnection_get_addr(struct pyconnection *, void *); static PyObject *pyconnection_get_peer_x509(struct pyconnection *, void *); +static PyObject *pyconnection_get_peer_x509dict(struct pyconnection *, void *); static PyGetSetDef pyconnection_getset[] = { GETTER("fd", pyconnection_get_fd), GETTER("addr", pyconnection_get_addr), GETTER("x509", pyconnection_get_peer_x509), + GETTER("x509dict", pyconnection_get_peer_x509dict), GETTER(NULL, NULL), }; diff --git a/src/python.c b/src/python.c index 99e1c59..5a3259d 100644 --- a/src/python.c +++ b/src/python.c @@ -119,6 +119,9 @@ static int pyhttp_iterobj_chunk_sent(struct netbuf *); static int pyhttp_iterobj_next(struct pyhttp_iterobj *); static void pyhttp_iterobj_disconnect(struct connection *); +static int pyconnection_x509_cb(void *, int, int, const char *, + const void *, size_t, int); + #if defined(KORE_USE_PGSQL) static int pykore_pgsql_result(struct pykore_pgsql *); static void pykore_pgsql_callback(struct kore_pgsql *, void *); @@ -2845,6 +2848,96 @@ pyconnection_get_peer_x509(struct pyconnection *pyc, void *closure) return (bytes); } +static PyObject * +pyconnection_get_peer_x509dict(struct pyconnection *pyc, void *closure) +{ + X509_NAME *name; + PyObject *dict, *issuer, *subject, *ret; + + ret = NULL; + issuer = NULL; + subject = NULL; + + if (pyc->c->cert == NULL) { + Py_RETURN_NONE; + } + + if ((dict = PyDict_New()) == NULL) + goto out; + + if ((issuer = PyDict_New()) == NULL) + goto out; + + if (PyDict_SetItemString(dict, "issuer", issuer) == -1) + goto out; + + if ((subject = PyDict_New()) == NULL) + goto out; + + if (PyDict_SetItemString(dict, "subject", subject) == -1) + goto out; + + PyErr_Clear(); + + if ((name = X509_get_issuer_name(pyc->c->cert)) == NULL) { + PyErr_Format(PyExc_RuntimeError, + "X509_get_issuer_name: %s", ssl_errno_s); + goto out; + } + + if (!kore_x509name_foreach(name, 0, issuer, pyconnection_x509_cb)) { + if (PyErr_Occurred() == NULL) { + PyErr_Format(PyExc_RuntimeError, + "failed to add issuer name to dictionary"); + } + goto out; + } + + if ((name = X509_get_subject_name(pyc->c->cert)) == NULL) { + PyErr_Format(PyExc_RuntimeError, + "X509_get_subject_name: %s", ssl_errno_s); + goto out; + } + + if (!kore_x509name_foreach(name, 0, subject, pyconnection_x509_cb)) { + if (PyErr_Occurred() == NULL) { + PyErr_Format(PyExc_RuntimeError, + "failed to add subject name to dictionary"); + } + goto out; + } + + ret = dict; + dict = NULL; + +out: + Py_XDECREF(dict); + Py_XDECREF(issuer); + Py_XDECREF(subject); + + return (ret); +} + +static int +pyconnection_x509_cb(void *udata, int islast, int nid, const char *field, + const void *data, size_t len, int flags) +{ + PyObject *dict, *obj; + + dict = udata; + + if ((obj = PyUnicode_FromStringAndSize(data, len)) == NULL) + return (KORE_RESULT_ERROR); + + if (PyDict_SetItemString(dict, field, obj) == -1) { + Py_DECREF(obj); + return (KORE_RESULT_ERROR); + } + + Py_DECREF(obj); + return (KORE_RESULT_OK); +} + static void pytimer_run(void *arg, u_int64_t now) { From 9845c8bbe1e68867fb06e826536bfd661d1a59df Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 13 Dec 2021 10:45:00 +0100 Subject: [PATCH 114/121] Python: Add req.body_digest. Returns the SHA256 digest of the uploaded body as a bytes object. --- include/kore/python_methods.h | 2 ++ src/python.c | 34 +++++++++++++--------------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index 72112d0..ff44d96 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -782,6 +782,7 @@ static PyObject *pyhttp_get_agent(struct pyhttp_request *, void *); static PyObject *pyhttp_get_method(struct pyhttp_request *, void *); static PyObject *pyhttp_get_body_path(struct pyhttp_request *, void *); static PyObject *pyhttp_get_connection(struct pyhttp_request *, void *); +static PyObject *pyhttp_get_body_digest(struct pyhttp_request *, void *); static PyGetSetDef pyhttp_request_getset[] = { GETTER("host", pyhttp_get_host), @@ -790,6 +791,7 @@ static PyGetSetDef pyhttp_request_getset[] = { GETTER("agent", pyhttp_get_agent), GETTER("method", pyhttp_get_method), GETTER("body_path", pyhttp_get_body_path), + GETTER("body_digest", pyhttp_get_body_digest), GETTER("connection", pyhttp_get_connection), GETTER(NULL, NULL) }; diff --git a/src/python.c b/src/python.c index 5a3259d..b38bf66 100644 --- a/src/python.c +++ b/src/python.c @@ -5109,42 +5109,34 @@ pyhttp_get_body(struct pyhttp_request *pyreq, void *closure) static PyObject * pyhttp_get_agent(struct pyhttp_request *pyreq, void *closure) { - PyObject *agent; - - if (pyreq->req->agent == NULL) { - Py_RETURN_NONE; - } - - if ((agent = PyUnicode_FromString(pyreq->req->path)) == NULL) - return (PyErr_NoMemory()); - - return (agent); + return (PyUnicode_FromString(pyreq->req->path)); } static PyObject * pyhttp_get_method(struct pyhttp_request *pyreq, void *closure) { - PyObject *method; - - if ((method = PyLong_FromUnsignedLong(pyreq->req->method)) == NULL) - return (PyErr_NoMemory()); - - return (method); + return (PyLong_FromUnsignedLong(pyreq->req->method)); } static PyObject * pyhttp_get_body_path(struct pyhttp_request *pyreq, void *closure) { - PyObject *path; - if (pyreq->req->http_body_path == NULL) { Py_RETURN_NONE; } - if ((path = PyUnicode_FromString(pyreq->req->http_body_path)) == NULL) - return (PyErr_NoMemory()); + return (PyUnicode_FromString(pyreq->req->http_body_path)); +} - return (path); +static PyObject * +pyhttp_get_body_digest(struct pyhttp_request *pyreq, void *closure) +{ + PyObject *digest; + + digest = PyBytes_FromStringAndSize((char *)pyreq->req->http_body_digest, + sizeof(pyreq->req->http_body_digest)); + + return (digest); } static PyObject * From 97ef486d2264f17d50e916c09458da5fbe12371c Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Mon, 13 Dec 2021 10:48:29 +0100 Subject: [PATCH 115/121] Fix indentation --- src/runtime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime.c b/src/runtime.c index 1edd8de..5be55b9 100644 --- a/src/runtime.c +++ b/src/runtime.c @@ -46,7 +46,7 @@ struct kore_runtime kore_native_runtime = { KORE_RUNTIME_NATIVE, #if !defined(KORE_NO_HTTP) .http_request = native_runtime_http_request, - .http_body_chunk= native_runtime_http_body_chunk, + .http_body_chunk = native_runtime_http_body_chunk, .validator = native_runtime_validator, .wsconnect = native_runtime_connect, .wsmessage = native_runtime_wsmessage, From e8e01980fc45cddf62f6eca3fffc6d2c9e9ae2ae Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Tue, 14 Dec 2021 23:15:21 +0100 Subject: [PATCH 116/121] Python: allow route hooks via kore.route(). Adding the hooks keyword with a dictionary attached to specify the relevant hooks will hook them for the given route. Eg: domain.route("/", self.index, methods=["get"], hooks={ "on_free": self.request_free } ) These are the same hooks available via a normal Kore route configuration. --- examples/async-curl/src/ftp.c | 2 +- examples/async-curl/src/http.c | 2 +- examples/pgsql/src/pgsql.c | 2 +- include/kore/http.h | 3 +- include/kore/kore.h | 4 ++ src/config.c | 20 +++++++ src/http.c | 9 +-- src/python.c | 104 +++++++++++++++++++++++++++++++++ src/runtime.c | 18 ++++++ 9 files changed, 153 insertions(+), 11 deletions(-) diff --git a/examples/async-curl/src/ftp.c b/examples/async-curl/src/ftp.c index ab5736e..de3a9f6 100644 --- a/examples/async-curl/src/ftp.c +++ b/examples/async-curl/src/ftp.c @@ -44,7 +44,7 @@ state_setup(struct http_request *req) { struct kore_curl *client; - client = http_state_create(req, sizeof(*client), NULL); + client = http_state_create(req, sizeof(*client)); if (!kore_curl_init(client, "http://ftp.eu.openbsd.org/pub/OpenBSD/README", KORE_CURL_ASYNC)) { diff --git a/examples/async-curl/src/http.c b/examples/async-curl/src/http.c index 2cc3688..f7148e7 100644 --- a/examples/async-curl/src/http.c +++ b/examples/async-curl/src/http.c @@ -59,7 +59,7 @@ state_setup(struct http_request *req) { struct kore_curl *client; - client = http_state_create(req, sizeof(*client), NULL); + client = http_state_create(req, sizeof(*client)); /* Initialize curl. */ if (!kore_curl_init(client, "https://kore.io", KORE_CURL_ASYNC)) { diff --git a/examples/pgsql/src/pgsql.c b/examples/pgsql/src/pgsql.c index 38e8c8b..8ddcd5e 100644 --- a/examples/pgsql/src/pgsql.c +++ b/examples/pgsql/src/pgsql.c @@ -85,7 +85,7 @@ request_perform_init(struct http_request *req) /* Setup our state context (if not yet set). */ if (!http_state_exists(req)) { - state = http_state_create(req, sizeof(*state), NULL); + state = http_state_create(req, sizeof(*state)); /* * Initialize the kore_pgsql data structure and bind it diff --git a/include/kore/http.h b/include/kore/http.h index e40b686..dc5f519 100644 --- a/include/kore/http.h +++ b/include/kore/http.h @@ -413,8 +413,7 @@ const char *http_media_type(const char *); void *http_state_get(struct http_request *); int http_state_exists(struct http_request *); void http_state_cleanup(struct http_request *); -void *http_state_create(struct http_request *, size_t, - void (*onfree)(struct http_request *)); +void *http_state_create(struct http_request *, size_t); int http_argument_urldecode(char *); int http_header_recv(struct netbuf *); diff --git a/include/kore/kore.h b/include/kore/kore.h index a4ce42c..be92eae 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -287,6 +287,7 @@ struct kore_runtime { int type; #if !defined(KORE_NO_HTTP) int (*http_request)(void *, struct http_request *); + void (*http_request_free)(void *, struct http_request *); void (*http_body_chunk)(void *, struct http_request *, const void *, size_t); int (*validator)(void *, struct http_request *, const void *); @@ -327,6 +328,7 @@ struct kore_route { struct kore_domain *dom; struct kore_auth *auth; struct kore_runtime_call *rcall; + struct kore_runtime_call *on_free; struct kore_runtime_call *on_headers; struct kore_runtime_call *on_body_chunk; @@ -993,6 +995,8 @@ void kore_runtime_connect(struct kore_runtime_call *, struct connection *); #if !defined(KORE_NO_HTTP) int kore_runtime_http_request(struct kore_runtime_call *, struct http_request *); +void kore_runtime_http_request_free(struct kore_runtime_call *, + struct http_request *); void kore_runtime_http_body_chunk(struct kore_runtime_call *, struct http_request *, const void *, size_t); int kore_runtime_validator(struct kore_runtime_call *, diff --git a/src/config.c b/src/config.c index 41f8795..6062b9f 100644 --- a/src/config.c +++ b/src/config.c @@ -111,6 +111,7 @@ static int configure_client_verify_depth(char *); static int configure_route(char *); static int configure_route_methods(char *); static int configure_route_handler(char *); +static int configure_route_on_free(char *); static int configure_route_on_headers(char *); static int configure_route_on_body_chunk(char *); static int configure_filemap(char *); @@ -201,6 +202,7 @@ static struct { { "handler", configure_route_handler }, { "on_headers", configure_route_on_headers }, { "on_body_chunk", configure_route_on_body_chunk }, + { "on_free", configure_route_on_free }, { "methods", configure_route_methods }, { "filemap", configure_filemap }, { "redirect", configure_redirect }, @@ -1203,6 +1205,24 @@ configure_route_on_body_chunk(char *name) return (KORE_RESULT_OK); } +static int +configure_route_on_free(char *name) +{ + if (current_route == NULL) { + kore_log(LOG_ERR, + "on_free keyword not inside of route context"); + return (KORE_RESULT_ERROR); + } + + if ((current_route->on_free = kore_runtime_getcall(name)) == NULL) { + kore_log(LOG_ERR, "on_free callback '%s' for '%s' not found", + name, current_route->path); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + static int configure_route_methods(char *options) { diff --git a/src/http.c b/src/http.c index 6c21990..acd2884 100644 --- a/src/http.c +++ b/src/http.c @@ -442,8 +442,8 @@ http_request_free(struct http_request *req) struct http_header *hdr, *next; struct http_cookie *ck, *cknext; - if (req->onfree != NULL) - req->onfree(req); + if (req->rt->on_free != NULL) + kore_runtime_http_request_free(req->rt->on_free, req); if (req->runlock != NULL) { LIST_REMOVE(req->runlock, list); @@ -1454,14 +1454,12 @@ http_state_exists(struct http_request *req) } void * -http_state_create(struct http_request *req, size_t len, - void (*onfree)(struct http_request *)) +http_state_create(struct http_request *req, size_t len) { if (req->hdlr_extra != NULL) fatal("http_state_create: state already exists"); req->state_len = len; - req->onfree = onfree; req->hdlr_extra = kore_calloc(1, len); return (req->hdlr_extra); @@ -2035,7 +2033,6 @@ http_request_new(struct connection *c, const char *host, req->status = 0; req->method = m; req->agent = NULL; - req->onfree = NULL; req->referer = NULL; req->runlock = NULL; req->flags = flags; diff --git a/src/python.c b/src/python.c index b38bf66..03ee0c1 100644 --- a/src/python.c +++ b/src/python.c @@ -88,6 +88,9 @@ static int python_route_params(PyObject *, struct kore_route *, static int python_route_methods(PyObject *, PyObject *, struct kore_route *); static int python_route_auth(PyObject *, struct kore_route *); +static int python_route_hooks(PyObject *, struct kore_route *); +static int python_route_hook_set(PyObject *, const char *, + struct kore_runtime_call **); static int python_coro_run(struct python_coro *); static void python_coro_wakeup(struct python_coro *); @@ -146,6 +149,9 @@ static void python_push_type(const char *, PyObject *, PyTypeObject *); static int python_validator_check(PyObject *); static int python_runtime_http_request(void *, struct http_request *); +static void python_runtime_http_request_free(void *, struct http_request *); +static void python_runtime_http_body_chunk(void *, struct http_request *, + const void *, size_t); static int python_runtime_validator(void *, struct http_request *, const void *); static void python_runtime_wsmessage(void *, struct connection *, @@ -175,6 +181,8 @@ struct kore_module_functions kore_python_module = { struct kore_runtime kore_python_runtime = { KORE_RUNTIME_PYTHON, .http_request = python_runtime_http_request, + .http_body_chunk = python_runtime_http_body_chunk, + .http_request_free = python_runtime_http_request_free, .validator = python_runtime_validator, .wsconnect = python_runtime_connect, .wsmessage = python_runtime_wsmessage, @@ -1318,6 +1326,49 @@ python_runtime_http_request(void *addr, struct http_request *req) return (KORE_RESULT_OK); } +static void +python_runtime_http_request_free(void *addr, struct http_request *req) +{ + PyObject *ret; + + if (req->py_req == NULL) + fatal("%s: py_req is NULL", __func__); + + PyErr_Clear(); + ret = PyObject_CallFunctionObjArgs(addr, req->py_req, NULL); + + if (ret == NULL) + kore_python_log_error("python_runtime_http_request_free"); + + Py_XDECREF(ret); +} + +static void +python_runtime_http_body_chunk(void *addr, struct http_request *req, + const void *data, size_t len) +{ + PyObject *args, *ret; + + if (req->py_req == NULL) { + if ((req->py_req = pyhttp_request_alloc(req)) == NULL) + fatal("%s: pyreq alloc failed", __func__); + } + + if ((args = Py_BuildValue("(Oy#)", req->py_req, data, len)) == NULL) { + kore_python_log_error("python_runtime_http_body_chunk"); + return; + } + + PyErr_Clear(); + ret = PyObject_Call(addr, args, NULL); + + if (ret == NULL) + kore_python_log_error("python_runtime_http_body_chunk"); + + Py_XDECREF(ret); + Py_DECREF(args); +} + static int python_runtime_validator(void *addr, struct http_request *req, const void *data) { @@ -5358,6 +5409,14 @@ python_route_install(struct pyroute *route) return (KORE_RESULT_ERROR); } } + + if ((obj = PyDict_GetItemString(kwargs, "hooks")) != NULL) { + if (!python_route_hooks(obj, rt)) { + kore_python_log_error("python_route_install"); + kore_route_free(rt); + return (KORE_RESULT_ERROR); + } + } } if (rt->path[0] == '/') { @@ -5605,6 +5664,51 @@ python_route_auth(PyObject *dict, struct kore_route *rt) return (KORE_RESULT_OK); } +static int +python_route_hooks(PyObject *dict, struct kore_route *rt) +{ + if (!PyDict_CheckExact(dict)) + return (KORE_RESULT_ERROR); + + if (!python_route_hook_set(dict, "on_free", &rt->on_free)) + return (KORE_RESULT_ERROR); + + if (!python_route_hook_set(dict, "on_headers", &rt->on_headers)) + return (KORE_RESULT_ERROR); + + if (!python_route_hook_set(dict, "on_body_chunk", &rt->on_body_chunk)) + return (KORE_RESULT_ERROR); + + return (KORE_RESULT_OK); +} + +static int +python_route_hook_set(PyObject *dict, const char *name, + struct kore_runtime_call **out) +{ + PyObject *obj; + struct kore_runtime_call *rcall; + + if ((obj = PyDict_GetItemString(dict, name)) == NULL) + return (KORE_RESULT_OK); + + if (!PyCallable_Check(obj)) { + PyErr_Format(PyExc_RuntimeError, + "%s for a route not callable", name); + Py_DECREF(obj); + return (KORE_RESULT_ERROR); + } + + rcall = kore_calloc(1, sizeof(struct kore_runtime_call)); + rcall->addr = obj; + rcall->runtime = &kore_python_runtime; + + Py_INCREF(rcall->addr); + *out = rcall; + + return (KORE_RESULT_OK); +} + #if defined(KORE_USE_PGSQL) static PyObject * python_kore_pgsql_query(PyObject *self, PyObject *args, PyObject *kwargs) diff --git a/src/runtime.c b/src/runtime.c index 5be55b9..fa19a1d 100644 --- a/src/runtime.c +++ b/src/runtime.c @@ -33,6 +33,7 @@ static void native_runtime_connect(void *, struct connection *); static void native_runtime_configure(void *, int, char **); #if !defined(KORE_NO_HTTP) static int native_runtime_http_request(void *, struct http_request *); +static void native_runtime_http_request_free(void *, struct http_request *); static void native_runtime_http_body_chunk(void *, struct http_request *, const void *, size_t); static int native_runtime_validator(void *, struct http_request *, @@ -46,6 +47,7 @@ struct kore_runtime kore_native_runtime = { KORE_RUNTIME_NATIVE, #if !defined(KORE_NO_HTTP) .http_request = native_runtime_http_request, + .http_request_free = native_runtime_http_request_free, .http_body_chunk = native_runtime_http_body_chunk, .validator = native_runtime_validator, .wsconnect = native_runtime_connect, @@ -108,6 +110,13 @@ kore_runtime_http_request(struct kore_runtime_call *rcall, return (rcall->runtime->http_request(rcall->addr, req)); } +void +kore_runtime_http_request_free(struct kore_runtime_call *rcall, + struct http_request *req) +{ + rcall->runtime->http_request_free(rcall->addr, req); +} + void kore_runtime_http_body_chunk(struct kore_runtime_call *rcall, struct http_request *req, const void *data, size_t len) @@ -188,6 +197,15 @@ native_runtime_http_request(void *addr, struct http_request *req) return (cb(req)); } +static void +native_runtime_http_request_free(void *addr, struct http_request *req) +{ + int (*cb)(struct http_request *); + + *(void **)&(cb) = addr; + cb(req); +} + static void native_runtime_http_body_chunk(void *addr, struct http_request *req, const void *data, size_t len) From e545657023f7640fb3307ddb90b16235ed28f5e8 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 15 Dec 2021 12:16:37 +0100 Subject: [PATCH 117/121] make sure we only call rt->on_free if req has route --- src/http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http.c b/src/http.c index acd2884..4747484 100644 --- a/src/http.c +++ b/src/http.c @@ -442,7 +442,7 @@ http_request_free(struct http_request *req) struct http_header *hdr, *next; struct http_cookie *ck, *cknext; - if (req->rt->on_free != NULL) + if (req->rt != NULL && req->rt->on_free != NULL) kore_runtime_http_request_free(req->rt->on_free, req); if (req->runlock != NULL) { From a3800fa57e9cfb98809ae0d53a5af7f0b27e343f Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 15 Dec 2021 12:19:04 +0100 Subject: [PATCH 118/121] Python: allocate py_req in the http_request_free hook if needed. Makes it possible to mix on_free with other runtimes. --- src/python.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python.c b/src/python.c index 03ee0c1..196e134 100644 --- a/src/python.c +++ b/src/python.c @@ -1331,8 +1331,10 @@ python_runtime_http_request_free(void *addr, struct http_request *req) { PyObject *ret; - if (req->py_req == NULL) - fatal("%s: py_req is NULL", __func__); + if (req->py_req == NULL) { + if ((req->py_req = pyhttp_request_alloc(req)) == NULL) + fatal("%s: pyreq alloc failed", __func__); + } PyErr_Clear(); ret = PyObject_CallFunctionObjArgs(addr, req->py_req, NULL); From ff19ce76521ef34ba5a674f0a51c1092ab4eea69 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Fri, 17 Dec 2021 16:52:13 +0100 Subject: [PATCH 119/121] Python: add a protocol member to kore.httprequest This returns a string depending on the protocol used (https / http) for the HTTP request. --- include/kore/python_methods.h | 2 ++ src/python.c | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index ff44d96..20928bd 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -780,6 +780,7 @@ static PyObject *pyhttp_get_path(struct pyhttp_request *, void *); static PyObject *pyhttp_get_body(struct pyhttp_request *, void *); static PyObject *pyhttp_get_agent(struct pyhttp_request *, void *); static PyObject *pyhttp_get_method(struct pyhttp_request *, void *); +static PyObject *pyhttp_get_protocol(struct pyhttp_request *, void *); static PyObject *pyhttp_get_body_path(struct pyhttp_request *, void *); static PyObject *pyhttp_get_connection(struct pyhttp_request *, void *); static PyObject *pyhttp_get_body_digest(struct pyhttp_request *, void *); @@ -790,6 +791,7 @@ static PyGetSetDef pyhttp_request_getset[] = { GETTER("body", pyhttp_get_body), GETTER("agent", pyhttp_get_agent), GETTER("method", pyhttp_get_method), + GETTER("protocol", pyhttp_get_protocol), GETTER("body_path", pyhttp_get_body_path), GETTER("body_digest", pyhttp_get_body_digest), GETTER("connection", pyhttp_get_connection), diff --git a/src/python.c b/src/python.c index 196e134..1e8157f 100644 --- a/src/python.c +++ b/src/python.c @@ -5171,6 +5171,22 @@ pyhttp_get_method(struct pyhttp_request *pyreq, void *closure) return (PyLong_FromUnsignedLong(pyreq->req->method)); } +static PyObject * +pyhttp_get_protocol(struct pyhttp_request *pyreq, void *closure) +{ + struct connection *c; + const char *proto; + + c = pyreq->req->owner; + + if (c->owner->server->tls) + proto = "https"; + else + proto = "http"; + + return (PyUnicode_FromString(proto)); +} + static PyObject * pyhttp_get_body_path(struct pyhttp_request *pyreq, void *closure) { From d8113e4545c9ddf6cb092548df6a49306aba4ea4 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Sun, 19 Dec 2021 00:14:33 +0100 Subject: [PATCH 120/121] Reset dom->acme_cert upon clearing. --- src/worker.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/worker.c b/src/worker.c index 65e8642..465ddcf 100644 --- a/src/worker.c +++ b/src/worker.c @@ -1000,7 +1000,10 @@ worker_keymgr_response(struct kore_msg *msg, const void *data) case KORE_ACME_CHALLENGE_CLEAR_CERT: dom->acme_cert_len = 0; dom->acme_challenge = 0; + kore_free(dom->acme_cert); + dom->acme_cert = NULL; + kore_log(LOG_NOTICE, "[%s] tls-alpn-01 challenge disabled", dom->domain); break; From 93a4fe2a15738d2ecd2f97b219b3ac698bb348e4 Mon Sep 17 00:00:00 2001 From: Joris Vink Date: Wed, 22 Dec 2021 09:50:26 +0100 Subject: [PATCH 121/121] Worker hook rework. This commit adds improved hooks for Python and a new signal delivery hook. For the Python API kore_worker_configure() and kore_worker_teardown() had to be implemented before this commit. Now one can create a workerstart and workerend method in their koreapp as those will be called when they exist. The new signal hook is either kore_worker_signal() or koreapp.signal. This new hook is called after the worker event code handles the received signal itself first. With this commit there is also a new kore_signal_trap() API call allowing you to more easily trap new signals. This API also also exported to the Python part of the code under kore.sigtrap() --- include/kore/hooks.h | 1 + include/kore/kore.h | 3 ++ include/kore/python_api.h | 3 ++ include/kore/python_methods.h | 2 + src/kore.c | 48 +++++++++--------- src/python.c | 31 ++++++++++++ src/runtime.c | 17 +++++++ src/worker.c | 91 +++++++++++++++++++++++++++++------ 8 files changed, 156 insertions(+), 40 deletions(-) diff --git a/include/kore/hooks.h b/include/kore/hooks.h index 2127160..aef85aa 100644 --- a/include/kore/hooks.h +++ b/include/kore/hooks.h @@ -22,6 +22,7 @@ #define KORE_DAEMONIZED_HOOK "kore_parent_daemonized" void kore_seccomp_hook(void); +void kore_worker_signal(int); void kore_worker_teardown(void); void kore_parent_teardown(void); void kore_worker_configure(void); diff --git a/include/kore/kore.h b/include/kore/kore.h index be92eae..8844c0b 100644 --- a/include/kore/kore.h +++ b/include/kore/kore.h @@ -298,6 +298,7 @@ struct kore_runtime { #endif void (*execute)(void *); int (*onload)(void *, int); + void (*signal)(void *, int); void (*connect)(void *, struct connection *); void (*configure)(void *, int, char **); }; @@ -754,6 +755,7 @@ extern struct kore_server_list kore_servers; void kore_signal(int); void kore_shutdown(void); +void kore_signal_trap(int); void kore_signal_setup(void); void kore_proctitle(const char *); void kore_default_getopt(int, char **); @@ -990,6 +992,7 @@ struct kore_module *kore_module_load(const char *, void kore_runtime_execute(struct kore_runtime_call *); int kore_runtime_onload(struct kore_runtime_call *, int); +void kore_runtime_signal(struct kore_runtime_call *, int); void kore_runtime_configure(struct kore_runtime_call *, int, char **); void kore_runtime_connect(struct kore_runtime_call *, struct connection *); #if !defined(KORE_NO_HTTP) diff --git a/include/kore/python_api.h b/include/kore/python_api.h index d8052aa..0aefc76 100644 --- a/include/kore/python_api.h +++ b/include/kore/python_api.h @@ -51,8 +51,11 @@ extern const char *kore_pymodule; extern struct kore_module_functions kore_python_module; extern struct kore_runtime kore_python_runtime; +#define KORE_PYTHON_SIGNAL_HOOK "koreapp.signal" #define KORE_PYTHON_TEARDOWN_HOOK "koreapp.cleanup" #define KORE_PYTHON_CONFIG_HOOK "koreapp.configure" #define KORE_PYTHON_DAEMONIZED_HOOK "koreapp.daemonized" +#define KORE_PYTHON_WORKER_STOP_HOOK "koreapp.workerstop" +#define KORE_PYTHON_WORKER_START_HOOK "koreapp.workerstart" #endif diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h index 20928bd..4a6c6ef 100644 --- a/include/kore/python_methods.h +++ b/include/kore/python_methods.h @@ -46,6 +46,7 @@ static PyObject *python_kore_worker(PyObject *, PyObject *); static PyObject *python_kore_tracer(PyObject *, PyObject *); static PyObject *python_kore_fatalx(PyObject *, PyObject *); static PyObject *python_kore_task_id(PyObject *, PyObject *); +static PyObject *python_kore_sigtrap(PyObject *, PyObject *); static PyObject *python_kore_setname(PyObject *, PyObject *); static PyObject *python_kore_suspend(PyObject *, PyObject *); static PyObject *python_kore_shutdown(PyObject *, PyObject *); @@ -98,6 +99,7 @@ static struct PyMethodDef pykore_methods[] = { METHOD("fatal", python_kore_fatal, METH_VARARGS), METHOD("fatalx", python_kore_fatalx, METH_VARARGS), METHOD("task_id", python_kore_task_id, METH_NOARGS), + METHOD("sigtrap", python_kore_sigtrap, METH_VARARGS), METHOD("setname", python_kore_setname, METH_VARARGS), METHOD("suspend", python_kore_suspend, METH_VARARGS), METHOD("shutdown", python_kore_shutdown, METH_NOARGS), diff --git a/src/kore.c b/src/kore.c index 18ab5d0..b66e043 100644 --- a/src/kore.c +++ b/src/kore.c @@ -740,6 +740,23 @@ kore_sockopt(int fd, int what, int opt) void kore_signal_setup(void) +{ + kore_signal_trap(SIGHUP); + kore_signal_trap(SIGQUIT); + kore_signal_trap(SIGTERM); + kore_signal_trap(SIGUSR1); + kore_signal_trap(SIGCHLD); + + if (kore_foreground) + kore_signal_trap(SIGINT); + else + (void)signal(SIGINT, SIG_IGN); + + (void)signal(SIGPIPE, SIG_IGN); +} + +void +kore_signal_trap(int sig) { struct sigaction sa; @@ -750,25 +767,8 @@ kore_signal_setup(void) if (sigfillset(&sa.sa_mask) == -1) fatal("sigfillset: %s", errno_s); - if (sigaction(SIGHUP, &sa, NULL) == -1) + if (sigaction(sig, &sa, NULL) == -1) fatal("sigaction: %s", errno_s); - if (sigaction(SIGQUIT, &sa, NULL) == -1) - fatal("sigaction: %s", errno_s); - if (sigaction(SIGTERM, &sa, NULL) == -1) - fatal("sigaction: %s", errno_s); - if (sigaction(SIGUSR1, &sa, NULL) == -1) - fatal("sigaction: %s", errno_s); - if (sigaction(SIGCHLD, &sa, NULL) == -1) - fatal("sigaction: %s", errno_s); - - if (kore_foreground) { - if (sigaction(SIGINT, &sa, NULL) == -1) - fatal("sigaction: %s", errno_s); - } else { - (void)signal(SIGINT, SIG_IGN); - } - - (void)signal(SIGPIPE, SIG_IGN); } void @@ -961,22 +961,22 @@ kore_server_start(int argc, char *argv[]) #endif while (kore_quit != 1) { - if (sig_recv != 0) { - last_sig = sig_recv; + last_sig = sig_recv; - switch (sig_recv) { + if (last_sig != 0) { + switch (last_sig) { case SIGHUP: - kore_worker_dispatch_signal(sig_recv); + kore_worker_dispatch_signal(last_sig); kore_module_reload(0); break; case SIGINT: case SIGQUIT: case SIGTERM: kore_quit = 1; - kore_worker_dispatch_signal(sig_recv); + kore_worker_dispatch_signal(last_sig); continue; case SIGUSR1: - kore_worker_dispatch_signal(sig_recv); + kore_worker_dispatch_signal(last_sig); break; case SIGCHLD: kore_worker_reap(); diff --git a/src/python.c b/src/python.c index 1e8157f..77a3c7d 100644 --- a/src/python.c +++ b/src/python.c @@ -158,6 +158,7 @@ static void python_runtime_wsmessage(void *, struct connection *, u_int8_t, const void *, size_t); static void python_runtime_execute(void *); static int python_runtime_onload(void *, int); +static void python_runtime_signal(void *, int); static void python_runtime_configure(void *, int, char **); static void python_runtime_connect(void *, struct connection *); @@ -188,6 +189,7 @@ struct kore_runtime kore_python_runtime = { .wsmessage = python_runtime_wsmessage, .wsdisconnect = python_runtime_connect, .onload = python_runtime_onload, + .signal = python_runtime_signal, .connect = python_runtime_connect, .execute = python_runtime_execute, .configure = python_runtime_configure, @@ -1643,6 +1645,22 @@ python_runtime_connect(void *addr, struct connection *c) Py_DECREF(pyret); } +static void +python_runtime_signal(void *addr, int sig) +{ + PyObject *obj, *ret; + + if ((obj = Py_BuildValue("i", sig)) == NULL) { + kore_python_log_error("python_runtime_signal"); + return; + } + + ret = PyObject_CallFunctionObjArgs(addr, obj, NULL); + + Py_DECREF(obj); + Py_XDECREF(ret); +} + PyMODINIT_FUNC python_module_init(void) { @@ -2441,6 +2459,19 @@ python_kore_setname(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +python_kore_sigtrap(PyObject *self, PyObject *args) +{ + int sig; + + if (!PyArg_ParseTuple(args, "i", &sig)) + return (NULL); + + kore_signal_trap(sig); + + Py_RETURN_NONE; +} + static PyObject * python_kore_sendobj(PyObject *self, PyObject *args, PyObject *kwargs) { diff --git a/src/runtime.c b/src/runtime.c index fa19a1d..8cdf4c8 100644 --- a/src/runtime.c +++ b/src/runtime.c @@ -29,6 +29,7 @@ static void native_runtime_execute(void *); static int native_runtime_onload(void *, int); +static void native_runtime_signal(void *, int); static void native_runtime_connect(void *, struct connection *); static void native_runtime_configure(void *, int, char **); #if !defined(KORE_NO_HTTP) @@ -55,6 +56,7 @@ struct kore_runtime kore_native_runtime = { .wsdisconnect = native_runtime_connect, #endif .onload = native_runtime_onload, + .signal = native_runtime_signal, .connect = native_runtime_connect, .execute = native_runtime_execute, .configure = native_runtime_configure @@ -102,6 +104,12 @@ kore_runtime_connect(struct kore_runtime_call *rcall, struct connection *c) rcall->runtime->connect(rcall->addr, c); } +void +kore_runtime_signal(struct kore_runtime_call *rcall, int sig) +{ + rcall->runtime->signal(rcall->addr, sig); +} + #if !defined(KORE_NO_HTTP) int kore_runtime_http_request(struct kore_runtime_call *rcall, @@ -187,6 +195,15 @@ native_runtime_onload(void *addr, int action) return (cb(action)); } +static void +native_runtime_signal(void *addr, int sig) +{ + void (*cb)(int); + + *(void **)&(cb) = addr; + cb(sig); +} + #if !defined(KORE_NO_HTTP) static int native_runtime_http_request(void *addr, struct http_request *req) diff --git a/src/worker.c b/src/worker.c index 465ddcf..bea317b 100644 --- a/src/worker.c +++ b/src/worker.c @@ -79,8 +79,12 @@ struct wlock { static int worker_trylock(void); static void worker_unlock(void); static void worker_reaper(pid_t, int); +static void worker_runtime_teardown(void); +static void worker_runtime_configure(void); static void worker_domain_check(struct kore_domain *); +static struct kore_runtime_call *worker_runtime_signal(void); + static inline int worker_acceptlock_obtain(void); static inline void worker_acceptlock_release(void); static void worker_accept_avail(struct kore_msg *, const void *); @@ -420,9 +424,9 @@ kore_worker_privsep(void) void kore_worker_entry(struct kore_worker *kw) { - struct kore_runtime_call *rcall; + struct kore_runtime_call *sigcall; u_int64_t last_seed; - int quit, had_lock; + int quit, had_lock, sig; u_int64_t netwait, now, next_timeo; worker = kw; @@ -504,11 +508,7 @@ kore_worker_entry(struct kore_worker *kw) if (nlisteners == 0) worker_no_lock = 1; - rcall = kore_runtime_getcall("kore_worker_configure"); - if (rcall != NULL) { - kore_runtime_execute(rcall); - kore_free(rcall); - } + worker_runtime_configure(); kore_module_onload(); kore_domain_callback(worker_domain_check); @@ -516,6 +516,8 @@ kore_worker_entry(struct kore_worker *kw) kore_worker_started(); worker->restarted = 0; + sigcall = worker_runtime_signal(); + for (;;) { now = kore_time_ms(); @@ -564,8 +566,9 @@ kore_worker_entry(struct kore_worker *kw) } } - if (sig_recv != 0) { - switch (sig_recv) { + sig = sig_recv; + if (sig != 0) { + switch (sig) { case SIGHUP: kore_module_reload(1); break; @@ -583,7 +586,11 @@ kore_worker_entry(struct kore_worker *kw) break; } - sig_recv = 0; + if (sigcall != NULL) + kore_runtime_signal(sigcall, sig); + + if (sig == sig_recv) + sig_recv = 0; } if (quit) @@ -608,12 +615,7 @@ kore_worker_entry(struct kore_worker *kw) kore_connection_prune(KORE_CONNECTION_PRUNE_DISCONNECT); } - rcall = kore_runtime_getcall("kore_worker_teardown"); - if (rcall != NULL) { - kore_runtime_execute(rcall); - kore_free(rcall); - } - + worker_runtime_teardown(); kore_server_cleanup(); kore_platform_event_cleanup(); @@ -753,6 +755,63 @@ kore_worker_started(void) worker->ready = 1; } +static void +worker_runtime_configure(void) +{ + struct kore_runtime_call *rcall; + + rcall = NULL; + +#if defined(KORE_USE_PYTHON) + rcall = kore_runtime_getcall(KORE_PYTHON_WORKER_START_HOOK); +#endif + + if (rcall == NULL) + rcall = kore_runtime_getcall("kore_worker_configure"); + + if (rcall != NULL) { + kore_runtime_execute(rcall); + kore_free(rcall); + } +} + +static struct kore_runtime_call * +worker_runtime_signal(void) +{ + struct kore_runtime_call *rcall; + + rcall = NULL; + +#if defined(KORE_USE_PYTHON) + rcall = kore_runtime_getcall(KORE_PYTHON_SIGNAL_HOOK); +#endif + + if (rcall == NULL) + rcall = kore_runtime_getcall("kore_worker_signal"); + + return (rcall); +} + +static void +worker_runtime_teardown(void) +{ + struct kore_runtime_call *rcall; + + rcall = NULL; + +#if defined(KORE_USE_PYTHON) + rcall = kore_runtime_getcall(KORE_PYTHON_WORKER_STOP_HOOK); +#endif + + if (rcall == NULL) + rcall = kore_runtime_getcall("kore_worker_teardown"); + + if (rcall != NULL) { + kore_runtime_execute(rcall); + kore_free(rcall); + } +} + static void worker_domain_check(struct kore_domain *dom) {