diff -Nru kore-2.0.0/conf/kore.conf.example kore-3.3.1/conf/kore.conf.example --- kore-2.0.0/conf/kore.conf.example 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/conf/kore.conf.example 2019-06-03 13:29:24.000000000 +0000 @@ -11,11 +11,17 @@ # Server configuration. bind 127.0.0.1 443 +#bind_unix /var/run/kore.sock -# The path worker processes will chroot into after starting. -chroot /home/joris/src/kore +# The worker process root directory. If chrooting was not disabled +# at startup the worker processes will chroot into this directory. +# +# If this configuration option is not set, Kore will take the current +# working directory as the root. +root /home/joris/src/kore -# Worker processes will run as the specified user. +# Worker processes will run as the specified user. If this option is +# missing Kore will run as the current user. runas joris # How many worker processes Kore will spawn. If the directive @@ -28,10 +34,10 @@ # The number of active connections each worker can handle. # You might have to tweak this number based on your hardware. -#worker_max_connections 250 +#worker_max_connections 512 # Limit of maximum open files per worker. -#worker_rlimit_nofiles 1024 +#worker_rlimit_nofiles 768 # Limit the number of new connections a worker can accept # in a single event loop. By default Kore will accept as @@ -42,11 +48,18 @@ # then worker_max_connections) or you have an actual reason # to not spend too much time in the accept loop this setting # will make a HUGE positive difference. -# -# This is disabled by default. If you wish to enable this -# specify the number of connections a worker will accept -# before returning from the accept loop. -#worker_accept_threshold 0 + +# Number of accept() calls a worker will do at most in one go +# before releasing the lock to others. +#worker_accept_threshold 16 + +# What should the Kore parent process do if a worker +# process unexpectedly exits. The default policy is that +# the worker process is automatically restarted. +# +# If you want the kore server to exit if a worker dies +# you can swap the policy to "terminate". +#worker_death_policy restart # Workers bind themselves to a single CPU by default. # Turn this off by setting this option to 0 @@ -55,13 +68,57 @@ # Store the pid of the main process in this file. #pidfile kore.pid +# If TLS is enabled you can specify a file where Kore will read +# initial entropy from and save entropy towards when exiting. +# +# Note that if you enable this you must provide the first iteration +# of this file by generating 1024 cryptographically safe random bytes +# and writing them to the file specified. +# +# Kore will refuse to start if the specified file does not exist, +# is of the wrong size or cannot be opened in anyway. +# +# NOTE: This file location must be inside your chrooted environment. +#rand_file random.data + +# Key manager specific options. +# If TLS is enabled you will need to specify paths to the domain +# certificate and key that Kore will load. This loading is done +# from the keymgr (separate process) and all paths must be relative +# to the keymgr_root configuration option. +# +# keymgr_root The root path the keymgr will chdir into. +# If chroot was not disable at startup time +# the keymgr process will chroot into here. +# +# keymgr_runas The user to run the keymgr as. +# +# If privsep and chrooting is enabled at startup time but these +# configuration options were not set, they will take over the +# values from the 'root' and 'runas' configuration options. +# +#keymgr_root +#keymgr_runas + +# Filemap settings +# filemap_index Name of the file to be used as the directory +# index for a filemap. +#filemap_index index.html + # HTTP specific settings. # http_header_max Maximum size of HTTP headers (in bytes). # +# http_header_timeout Timeout in seconds for receiving the +# HTTP headers before the connection is closed. +# # http_body_max Maximum size of an HTTP body (in bytes). # If set to 0 disallows requests with a body # all together. # +# http_body_timeout Timeout in seconds for receiving the +# HTTP body in full before the connection +# is closed with an 408. +# # http_body_disk_offload Number of bytes after which Kore will use # a temporary file to hold the HTTP body # instead of holding it in memory. If set to @@ -79,13 +136,20 @@ # all responses. Parameter is the age. # (Set to 0 to disable sending this header). # -# http_request_limit Limit the number of requests Kore processes -# in a single event loop. +# http_request_limit Limit the number of HTTP requests workers +# can queue up. +# +# http_request_ms The number of milliseconds workers can max +# spend inside the HTTP processing loop. +# #http_header_max 4096 +#http_header_timeout 10 #http_body_max 1024000 +#http_body_timeout 60 #http_keepalive_time 0 #http_hsts_enable 31536000 #http_request_limit 1000 +#http_request_ms 10 #http_body_disk_offload 0 #http_body_disk_path tmp_files @@ -104,6 +168,9 @@ # which Kore will call when the module is loaded/reloaded. load contrib/examples/generic/example.module example_load +# Load a python file (if built with PYTHON=1) +#python_import src/index.py example_load + # Validators # validator name type regex|function # @@ -125,6 +192,13 @@ # Required DH parameters for TLS. #tls_dhparam dh2048.pem +# OpenBSD specific settings. +# Add more pledges if your application requires more privileges. +# All worker processes call pledge(2) after dropping privileges +# (even if -rn was specified). +# By default Kore will use the following promises: "stdio rpath inet error" +#pledge dns wpath + # Authentication configuration # # Using authentication blocks you can define a standard way for @@ -177,9 +251,16 @@ # # accesslog # - File where all requests are logged. -# client_certificates [CA] [optional CRL] -# - Require client certificates to be sent for the given -# CA with an optional CRL file. +# +# NOTE: due to current limitations the client_verify CA path +# MUST be in the 'root' of the Kore workers, not the keymgr. +# +# client_verify [CA] [optional CRL] +# - Turns on client verification, requiring the client to +# send a certificate that will be verified by the given CA. +# client_verify_depth [depth] +# - Configure the depth for x509 chain validation. +# By default 1. # # Handlers # @@ -210,9 +291,20 @@ static /params-test serve_params_test static /private serve_private + # Restrict some URIs to certain methods + restrict /private post + restrict /validator post get head + # Page handlers with authentication. static /private/test serve_private_test auth_example + # Allow access to files from the directory static_files via + # the /files/ URI. + # + # Note the directory given must be relative to the root configuration + # option. + filemap /files/ static_files + # Configure /params-test POST to only accept the following parameters. # They are automatically tested against the validator listed. # If the validator would fail Kore will automatically remove the @@ -230,13 +322,24 @@ validate arg1 v_example validate id v_number } + + # Configure a params block for allowed parameters in the + # querystring when performing a POST against /params-test. + # You do this by prefixing the method with the qs: marker. + # In the param blocks below we allow the parameter "post_id" + # in the querystring validated by v_number when a POST is + # done against the supplied URL. + params qs:post /params-test { + validate post_id v_number + } } #domain domain.com { # certfile cert/other/server.crt # certkey cert/other/server.key # accesslog /var/log/other_kore_access.log -# client_certificates cert/other/ca.crt +# client_verify /other/ca.crt +# client_verify_depth 1 # static /css/style.css serve_style_css # static / serve_index diff -Nru kore-2.0.0/debian/changelog kore-3.3.1/debian/changelog --- kore-2.0.0/debian/changelog 2018-06-23 11:40:32.000000000 +0000 +++ kore-3.3.1/debian/changelog 2018-10-01 07:40:24.000000000 +0000 @@ -1,3 +1,20 @@ +kore (3.3.1-1) unstable; urgency=medium + + [ Ondřej Nový ] + * d/copyright: Use https protocol in Format field + * d/rules: Remove trailing whitespaces + + [ Shih-Yuan Lee (FourDollars) ] + * Bump to 3.3.1. (Closes: #945098) + * d/p/debian.patch: Adjust the patch for 3.3.1. + * d/p/maybe-uninitialized.patch, + d/p/Add-support-for-openssl-1.1.0-release-line.patch, + d/p/Deal-with-the-Host-header-in-a-way-IPv6-hosts-work.patch: + Drop patches from upstream + * d/control: Update Standards-Version to 4.4.1. + + -- Shih-Yuan Lee (FourDollars) Mon, 01 Oct 2018 09:40:24 +0200 + kore (2.0.0-4) unstable; urgency=medium * debian/patches/Add-support-for-openssl-1.1.0-release-line.patch: Add the diff -Nru kore-2.0.0/debian/control kore-3.3.1/debian/control --- kore-2.0.0/debian/control 2018-06-23 11:29:25.000000000 +0000 +++ kore-3.3.1/debian/control 2018-10-01 07:40:24.000000000 +0000 @@ -3,7 +3,7 @@ Priority: optional Maintainer: Shih-Yuan Lee (FourDollars) Build-Depends: debhelper (>= 9.0.0), libssl-dev, libpq-dev -Standards-Version: 4.1.4.2 +Standards-Version: 4.4.1 Homepage: https://kore.io Vcs-Git: https://salsa.debian.org/debian/kore.git Vcs-Browser: https://salsa.debian.org/debian/kore diff -Nru kore-2.0.0/debian/copyright kore-3.3.1/debian/copyright --- kore-2.0.0/debian/copyright 2018-06-22 14:59:37.000000000 +0000 +++ kore-3.3.1/debian/copyright 2018-10-01 07:40:24.000000000 +0000 @@ -1,4 +1,4 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: kore Source: https://kore.io/downloads diff -Nru kore-2.0.0/debian/patches/Add-support-for-openssl-1.1.0-release-line.patch kore-3.3.1/debian/patches/Add-support-for-openssl-1.1.0-release-line.patch --- kore-2.0.0/debian/patches/Add-support-for-openssl-1.1.0-release-line.patch 2018-06-23 11:11:47.000000000 +0000 +++ kore-3.3.1/debian/patches/Add-support-for-openssl-1.1.0-release-line.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,303 +0,0 @@ -Description: Add support for openssl 1.1.0 release line. -Author: Joris Vink -Origin: upstream, https://github.com/jorisvink/kore/commit/95daf3a62bef0b9c84101eecbb464fd474566f45 -Bug-Debian: https://bugs.debian.org/858939 ---- -This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ - -From 95daf3a62bef0b9c84101eecbb464fd474566f45 Mon Sep 17 00:00:00 2001 -From: Joris Vink -Date: Mon, 22 May 2017 14:31:38 +0200 -Subject: [PATCH] Add support for openssl 1.1.0 release line. - -Eventually I will phase out 1.0.2 down the line to get rid of the -nightmare that is the 2 different APIs. - -This commit adds full support for building kore with 1.1.0e while -retaining the privsep keymanager support. - -based on excellent work done by @hiwk. ---- - src/domain.c | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---- - src/keymgr.c | 25 ++++++++++---- - 2 files changed, 119 insertions(+), 14 deletions(-) - -Index: kore/src/domain.c -=================================================================== ---- kore.orig/src/domain.c 2018-06-23 18:52:47.147427329 +0800 -+++ kore/src/domain.c 2018-06-23 18:52:47.143427237 +0800 -@@ -14,6 +14,11 @@ - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -+/* -+ * XXX - Lots of OPENSSL ifdefs here for 1.0.2 and 1.1.0 release lines. -+ * The idea is to only support 1.1.0 down the line and remove the 1.0.2 goo. -+ */ -+ - #include - - #if !defined(KORE_NO_TLS) -@@ -59,12 +64,14 @@ - static ECDSA_SIG *keymgr_ecdsa_sign(const unsigned char *, int, - const BIGNUM *, const BIGNUM *, EC_KEY *); - -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+static RSA_METHOD *keymgr_rsa_meth = NULL; -+static EC_KEY_METHOD *keymgr_ec_meth = NULL; -+#else - #if !defined(LIBRESSL_VERSION_TEXT) - /* - * Run own ecdsa_method data structure as OpenSSL has this in ecs_locl.h - * and does not export this on systems. -- * -- * XXX - OpenSSL is merging ECDSA functionality into EC in 1.1.0. - */ - struct ecdsa_method { - const char *name; -@@ -105,12 +112,32 @@ - NULL - }; - --#endif -+#endif /* OPENSSL_VERSION_NUMBER */ -+#endif /* KORE_NO_TLS */ - - void - kore_domain_init(void) - { - TAILQ_INIT(&domains); -+ -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ if (keymgr_rsa_meth == NULL) { -+ if ((keymgr_rsa_meth = RSA_meth_new("kore RSA keymgr method", -+ RSA_METHOD_FLAG_NO_CHECK)) == NULL) -+ fatal("failed to allocate RSA method"); -+ } -+ -+ RSA_meth_set_init(keymgr_rsa_meth, keymgr_rsa_init); -+ RSA_meth_set_finish(keymgr_rsa_meth, keymgr_rsa_finish); -+ RSA_meth_set_priv_enc(keymgr_rsa_meth, keymgr_rsa_privenc); -+ -+ if (keymgr_ec_meth == NULL) { -+ if ((keymgr_ec_meth = EC_KEY_METHOD_new(NULL)) == NULL) -+ fatal("failed to allocate EC KEY method"); -+ } -+ -+ EC_KEY_METHOD_set_sign(keymgr_ec_meth, NULL, NULL, keymgr_ecdsa_sign); -+#endif - } - - void -@@ -122,6 +149,18 @@ - TAILQ_REMOVE(&domains, dom, list); - kore_domain_free(dom); - } -+ -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ if (keymgr_rsa_meth != NULL) { -+ RSA_meth_free(keymgr_rsa_meth); -+ keymgr_rsa_meth = NULL; -+ } -+ -+ if (keymgr_ec_meth != NULL) { -+ EC_KEY_METHOD_free(keymgr_ec_meth); -+ keymgr_ec_meth = NULL; -+ } -+#endif - } - - int -@@ -211,6 +250,10 @@ - - kore_debug("kore_domain_sslstart(%s)", dom->domain); - -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ if ((method = TLS_method()) == NULL) -+ fatal("TLS_method(): %s", ssl_errno_s); -+#else - switch (tls_version) { - case KORE_TLS_VERSION_1_2: - method = TLSv1_2_server_method(); -@@ -225,16 +268,40 @@ - fatal("unknown tls_version: %d", tls_version); - return; - } -+#endif -+ -+ if ((dom->ssl_ctx = SSL_CTX_new(method)) == NULL) -+ fatal("SSL_ctx_new(): %s", ssl_errno_s); -+ -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ if (!SSL_CTX_set_min_proto_version(dom->ssl_ctx, TLS1_VERSION)) -+ fatal("SSL_CTX_set_min_proto_version: %s", ssl_errno_s); -+ if (!SSL_CTX_set_max_proto_version(dom->ssl_ctx, TLS1_2_VERSION)) -+ fatal("SSL_CTX_set_max_proto_version: %s", ssl_errno_s); - -- dom->ssl_ctx = SSL_CTX_new(method); -- if (dom->ssl_ctx == NULL) -- fatal("kore_domain_sslstart(): SSL_ctx_new(): %s", ssl_errno_s); -+ switch (tls_version) { -+ case KORE_TLS_VERSION_1_2: -+ if (!SSL_CTX_set_min_proto_version(dom->ssl_ctx, -+ TLS1_2_VERSION)) -+ fatal("SSL_CTX_set_min_proto_version: %s", ssl_errno_s); -+ break; -+ case KORE_TLS_VERSION_1_0: -+ if (!SSL_CTX_set_max_proto_version(dom->ssl_ctx, TLS1_VERSION)) -+ fatal("SSL_CTX_set_min_proto_version: %s", ssl_errno_s); -+ break; -+ case KORE_TLS_VERSION_BOTH: -+ break; -+ default: -+ fatal("unknown tls_version: %d", tls_version); -+ return; -+ } -+#endif - if (!SSL_CTX_use_certificate_chain_file(dom->ssl_ctx, dom->certfile)) { - fatal("SSL_CTX_use_certificate_chain_file(%s): %s", - dom->certfile, ssl_errno_s); - } - -- if ((in = BIO_new(BIO_s_file_internal())) == NULL) -+ if ((in = BIO_new(BIO_s_file())) == NULL) - fatal("BIO_new: %s", ssl_errno_s); - if (BIO_read_filename(in, dom->certfile) <= 0) - fatal("BIO_read_filename: %s", ssl_errno_s); -@@ -251,13 +318,22 @@ - if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) - fatal("no RSA public key present"); - RSA_set_app_data(rsa, dom); -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ RSA_set_method(rsa, keymgr_rsa_meth); -+#else - RSA_set_method(rsa, &keymgr_rsa); -+#endif - break; - case EVP_PKEY_EC: - if ((eckey = EVP_PKEY_get1_EC_KEY(pkey)) == NULL) - fatal("no EC public key present"); -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ EC_KEY_set_ex_data(eckey, 0, dom); -+ EC_KEY_set_method(eckey, keymgr_ec_meth); -+#else - ECDSA_set_ex_data(eckey, 0, dom); - ECDSA_set_method(eckey, &keymgr_ecdsa); -+#endif - break; - default: - fatal("unknown public key in certificate"); -@@ -314,8 +390,10 @@ - * from its OpenSSL in base so we don't need to care about it. - */ - #if !defined(LIBRESSL_VERSION_TEXT) -+#if OPENSSL_VERSION_NUMBER < 0x10100000L - dom->ssl_ctx->freelist_max_len = 0; - #endif -+#endif - SSL_CTX_set_mode(dom->ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE); - - if (tls_version == KORE_TLS_VERSION_BOTH) { -@@ -429,16 +507,27 @@ - if ((meth = RSA_get_default_method()) == NULL) - fatal("failed to obtain RSA method"); - -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ RSA_meth_set_pub_enc(keymgr_rsa_meth, RSA_meth_get_pub_enc(meth)); -+ RSA_meth_set_pub_dec(keymgr_rsa_meth, RSA_meth_get_pub_dec(meth)); -+ RSA_meth_set_bn_mod_exp(keymgr_rsa_meth, RSA_meth_get_bn_mod_exp(meth)); -+#else - keymgr_rsa.rsa_pub_enc = meth->rsa_pub_enc; - keymgr_rsa.rsa_pub_dec = meth->rsa_pub_dec; - keymgr_rsa.bn_mod_exp = meth->bn_mod_exp; -+#endif - } - - static int - keymgr_rsa_init(RSA *rsa) - { - if (rsa != NULL) { -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ RSA_set_flags(rsa, RSA_flags(rsa) | -+ RSA_FLAG_EXT_PKEY | RSA_METHOD_FLAG_NO_CHECK); -+#else - rsa->flags |= RSA_FLAG_EXT_PKEY | RSA_METHOD_FLAG_NO_CHECK; -+#endif - return (1); - } - -@@ -515,8 +604,13 @@ - if (len > sizeof(keymgr_buf)) - fatal("keymgr_buf too small"); - -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ if ((dom = EC_KEY_get_ex_data(eckey, 0)) == NULL) -+ fatal("EC_KEY has no domain"); -+#else - if ((dom = ECDSA_get_ex_data(eckey, 0)) == NULL) - fatal("EC_KEY has no domain"); -+#endif - - memset(keymgr_buf, 0, sizeof(keymgr_buf)); - -Index: kore/src/keymgr.c -=================================================================== ---- kore.orig/src/keymgr.c 2018-06-23 18:52:47.147427329 +0800 -+++ kore/src/keymgr.c 2018-06-23 18:52:47.143427237 +0800 -@@ -174,19 +174,25 @@ - keymgr_rsa_encrypt(struct kore_msg *msg, const void *data, struct key *key) - { - int ret; -+ RSA *rsa; - const struct kore_keyreq *req; - size_t keylen; - u_int8_t buf[1024]; - - req = (const struct kore_keyreq *)data; - -- keylen = RSA_size(key->pkey->pkey.rsa); -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ rsa = EVP_PKEY_get0_RSA(key->pkey); -+#else -+ rsa = key->pkey->pkey.rsa; -+#endif -+ keylen = RSA_size(rsa); - if (req->data_len > keylen || keylen > sizeof(buf)) - return; - - ret = RSA_private_encrypt(req->data_len, req->data, -- buf, key->pkey->pkey.rsa, req->padding); -- if (ret != RSA_size(key->pkey->pkey.rsa)) -+ buf, rsa, req->padding); -+ if (ret != RSA_size(rsa)) - return; - - kore_msg_send(msg->src, KORE_MSG_KEYMGR_RESP, buf, ret); -@@ -196,18 +202,23 @@ - keymgr_ecdsa_sign(struct kore_msg *msg, const void *data, struct key *key) - { - size_t len; -+ EC_KEY *ec; - const struct kore_keyreq *req; - unsigned int siglen; - u_int8_t sig[1024]; - - req = (const struct kore_keyreq *)data; -- -- len = ECDSA_size(key->pkey->pkey.ec); -+#if OPENSSL_VERSION_NUMBER >= 0x10100000L -+ ec = EVP_PKEY_get0_EC_KEY(key->pkey); -+#else -+ ec = key->pkey->pkey.ec; -+#endif -+ len = ECDSA_size(ec); - if (req->data_len > len || len > sizeof(sig)) - return; - -- if (ECDSA_sign(key->pkey->save_type, req->data, req->data_len, -- sig, &siglen, key->pkey->pkey.ec) == 0) -+ if (ECDSA_sign(EVP_PKEY_NONE, req->data, req->data_len, -+ sig, &siglen, ec) == 0) - return; - - if (siglen > sizeof(sig)) diff -Nru kore-2.0.0/debian/patches/Deal-with-the-Host-header-in-a-way-IPv6-hosts-work.patch kore-3.3.1/debian/patches/Deal-with-the-Host-header-in-a-way-IPv6-hosts-work.patch --- kore-2.0.0/debian/patches/Deal-with-the-Host-header-in-a-way-IPv6-hosts-work.patch 2018-06-23 11:11:47.000000000 +0000 +++ kore-3.3.1/debian/patches/Deal-with-the-Host-header-in-a-way-IPv6-hosts-work.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,108 +0,0 @@ -Description: Deal with the Host header in a way IPv6 hosts work -Author: Joris Vink -Origin: upstream, https://github.com/jorisvink/kore/commit/57840a8366d61604ac845de5b21a645bef528f20 -Bug: https://github.com/jorisvink/kore/issues/164 -Bug-Debian: https://bugs.debian.org/853478 ---- -This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ - -From 57840a8366d61604ac845de5b21a645bef528f20 Mon Sep 17 00:00:00 2001 -From: Joris Vink -Date: Wed, 11 Jan 2017 11:01:58 +0100 -Subject: [PATCH] Deal with the Host header in a way IPv6 hosts work - -Fixes #164. ---- - src/domain.c | 2 ++ - src/http.c | 24 ++++++++++++------------ - 2 files changed, 14 insertions(+), 12 deletions(-) - -Index: kore/src/domain.c -=================================================================== ---- kore.orig/src/domain.c 2018-06-23 18:52:49.415479439 +0800 -+++ kore/src/domain.c 2018-06-23 18:52:49.411479348 +0800 -@@ -429,6 +429,8 @@ - struct kore_domain *dom; - - TAILQ_FOREACH(dom, &domains, list) { -+ if (!strcmp(dom->domain, domain)) -+ return (dom); - if (!fnmatch(dom->domain, domain, FNM_CASEFOLD)) - return (dom); - } -Index: kore/src/http.c -=================================================================== ---- kore.orig/src/http.c 2018-06-23 18:52:49.415479439 +0800 -+++ kore/src/http.c 2018-06-23 18:52:49.411479348 +0800 -@@ -132,6 +132,9 @@ - kore_debug("http_request_new(%p, %s, %s, %s, %s)", c, host, - method, path, version); - -+ if ((p = strrchr(host, ':')) != NULL) -+ *p = '\0'; -+ - if ((hostlen = strlen(host)) >= KORE_DOMAINNAME_LEN - 1) { - http_error_response(c, 500); - return (KORE_RESULT_ERROR); -@@ -201,9 +204,6 @@ - req->http_body_offset = 0; - req->http_body_path = NULL; - -- if ((p = strrchr(host, ':')) != NULL) -- *p = '\0'; -- - req->host = kore_pool_get(&http_host_pool); - memcpy(req->host, host, hostlen); - req->host[hostlen] = '\0'; -@@ -546,7 +546,7 @@ - u_int64_t bytes_left; - u_int8_t *end_headers; - int h, i, v, skip, l; -- char *request[4], *host[3], *hbuf; -+ char *request[4], *host, *hbuf; - char *p, *headers[HTTP_REQ_HEADER_MAX]; - struct connection *c = (struct connection *)nb->owner; - -@@ -582,34 +582,34 @@ - } - - skip = 0; -- host[0] = NULL; -+ host = NULL; - for (i = 0; i < h; i++) { - if (strncasecmp(headers[i], "host", 4)) - continue; - -- v = kore_split_string(headers[i], ":", host, 3); -- if (v != 2) { -+ if ((host = strchr(headers[i], ':')) == NULL) { - http_error_response(c, 400); - return (KORE_RESULT_OK); - } - -- if ((host[1] - host[0]) != 5 || -- strncasecmp(host[0], "host", 4) || host[1] == '\0') { -+ *(host)++ = '\0'; -+ -+ if (*host == '\0') { - http_error_response(c, 400); - return (KORE_RESULT_OK); - } - -- host[1]++; -+ host++; - skip = i; - break; - } - -- if (host[0] == NULL) { -+ if (host == NULL) { - http_error_response(c, 400); - return (KORE_RESULT_OK); - } - -- if (!http_request_new(c, host[1], -+ if (!http_request_new(c, host, - request[0], request[1], request[2], &req)) - return (KORE_RESULT_OK); - diff -Nru kore-2.0.0/debian/patches/debian.patch kore-3.3.1/debian/patches/debian.patch --- kore-2.0.0/debian/patches/debian.patch 2018-06-23 11:27:05.000000000 +0000 +++ kore-3.3.1/debian/patches/debian.patch 2018-10-01 07:40:24.000000000 +0000 @@ -2,27 +2,29 @@ Author: Shih-Yuan Lee (FourDollars) --- This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ -Index: kore/Makefile -=================================================================== ---- kore.orig/Makefile 2018-06-23 19:22:22.942532544 +0800 -+++ kore/Makefile 2018-06-23 19:24:40.798189689 +0800 -@@ -4,8 +4,15 @@ - PREFIX?=/usr/local - OBJDIR?=obj +--- a/Makefile ++++ b/Makefile +@@ -6,10 +6,19 @@ KORE=kore + KODEV=kodev/kodev + KORE_CRYPTO?=crypto +ifndef DESTDIR INSTALL_DIR=$(PREFIX)/bin + MAN_DIR=$(PREFIX)/share/man + SHARE_DIR=$(PREFIX)/share/kore INCLUDE_DIR=$(PREFIX)/include/kore +else +INSTALL_DIR=$(DESTDIR)$(PREFIX)/bin ++MAN_DIR=$(DESTDIR)$(PREFIX)/share/man ++SHARE_DIR=$(DESTDIR)$(PREFIX)/share/kore +INCLUDE_DIR=$(DESTDIR)$(PREFIX)/include/kore +endif +CFLAGS:=$(shell dpkg-buildflags --get CFLAGS) $(shell dpkg-buildflags --get CPPFLAGS) -D_FILE_OFFSET_BITS=64 +CXXFLAGS:=$(shell dpkg-buildflags --get CXXFLAGS) - S_SRC= src/kore.c src/buf.c src/cli.c src/config.c src/connection.c \ - src/domain.c src/mem.c src/msg.c src/module.c src/net.c \ -@@ -44,6 +51,8 @@ + VERSION=src/version.c + +@@ -72,6 +81,8 @@ endif endif @@ -31,10 +33,8 @@ ifneq ("$(PGSQL)", "") S_SRC+=src/pgsql.c LDFLAGS+=-L$(shell pg_config --libdir) -lpq -Index: kore/changelog -=================================================================== ---- /dev/null 1970-01-01 00:00:00.000000000 +0000 -+++ kore/changelog 2018-06-23 19:22:22.938532524 +0800 +--- /dev/null ++++ b/changelog @@ -0,0 +1,105 @@ +Kore 2.0.0 +========== @@ -141,3 +141,17 @@ +2015-04-09: call setrlimit() before we drop privs. +2015-04-09: Use UINT_MAX for limit of worker_max_connections. +2015-04-08: Support PUT, DELETE, and HEAD methods in cofigs. +--- a/kodev/Makefile ++++ b/kodev/Makefile +@@ -4,7 +4,11 @@ + PREFIX?=/usr/local + OBJDIR?=obj + KODEV=kodev ++ifndef DESTDIR + INSTALL_DIR=$(PREFIX)/bin ++else ++INSTALL_DIR=$(DESTDIR)$(PREFIX)/bin ++endif + + S_SRC= ../src/cli.c + diff -Nru kore-2.0.0/debian/patches/maybe-uninitialized.patch kore-3.3.1/debian/patches/maybe-uninitialized.patch --- kore-2.0.0/debian/patches/maybe-uninitialized.patch 2018-06-23 11:11:47.000000000 +0000 +++ kore-3.3.1/debian/patches/maybe-uninitialized.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,26 +0,0 @@ -Description: Always initialize va_list in kore_pgsql_query_params. -Author: Tobias Kortkamp -Origin: upstream, https://github.com/jorisvink/kore/commit/c071d64bdddacbe1b69d238e14994d666a86f7cf -Bug: https://github.com/jorisvink/kore/issues/144 ---- -This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ -Index: kore/src/pgsql.c -=================================================================== ---- kore.orig/src/pgsql.c 2018-06-23 18:52:44.387362863 +0800 -+++ kore/src/pgsql.c 2018-06-23 18:52:44.383362768 +0800 -@@ -219,13 +219,11 @@ - int ret; - va_list args; - -- if (count > 0) -- va_start(args, count); -+ va_start(args, count); - - ret = kore_pgsql_v_query_params(pgsql, query, result, count, args); - -- if (count > 0) -- va_end(args); -+ va_end(args); - - return (ret); - } diff -Nru kore-2.0.0/debian/patches/series kore-3.3.1/debian/patches/series --- kore-2.0.0/debian/patches/series 2018-06-22 15:11:12.000000000 +0000 +++ kore-3.3.1/debian/patches/series 2018-10-01 07:40:24.000000000 +0000 @@ -1,4 +1 @@ debian.patch -maybe-uninitialized.patch -Add-support-for-openssl-1.1.0-release-line.patch -Deal-with-the-Host-header-in-a-way-IPv6-hosts-work.patch diff -Nru kore-2.0.0/debian/rules kore-3.3.1/debian/rules --- kore-2.0.0/debian/rules 2018-06-22 14:59:37.000000000 +0000 +++ kore-3.3.1/debian/rules 2018-10-01 07:40:24.000000000 +0000 @@ -8,4 +8,4 @@ export DEB_BUILD_MAINT_OPTIONS=hardening=+all %: - dh $@ + dh $@ diff -Nru kore-2.0.0/examples/async-curl/conf/async-curl.conf kore-3.3.1/examples/async-curl/conf/async-curl.conf --- kore-2.0.0/examples/async-curl/conf/async-curl.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/async-curl/conf/async-curl.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,16 @@ +# ht configuration + +bind 127.0.0.1 8888 + +workers 1 +tls_dhparam dh2048.pem + +pledge dns + +domain * { + certfile cert/server.pem + certkey cert/key.pem + + static / http + static /ftp ftp +} diff -Nru kore-2.0.0/examples/async-curl/conf/build.conf kore-3.3.1/examples/async-curl/conf/build.conf --- kore-2.0.0/examples/async-curl/conf/build.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/async-curl/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,34 @@ +# ht build config +# You can switch flavors using: kodev flavor [newflavor] + +# Set to yes if you wish to produce a single binary instead +# of a dynamic library. If you set this to yes you must also +# set kore_source together with kore_flavor. +single_binary=yes +kore_source=../../ +kore_flavor=CURL=1 + +# The flags below are shared between flavors +cflags=-Wall -Wmissing-declarations -Wshadow +cflags=-Wstrict-prototypes -Wmissing-prototypes +cflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +cxxflags=-Wall -Wmissing-declarations -Wshadow +cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +# Mime types for assets served via the builtin asset_serve_* +#mime_add=txt:text/plain; charset=utf-8 +#mime_add=png:image/png +#mime_add=html:text/html; charset=utf-8 + +dev { + # These flags are added to the shared ones when + # you build the "dev" flavor. + cflags=-g + cxxflags=-g +} + +#prod { +# You can specify additional flags here which are only +# included if you build with the "prod" flavor. +#} diff -Nru kore-2.0.0/examples/async-curl/.gitignore kore-3.3.1/examples/async-curl/.gitignore --- kore-2.0.0/examples/async-curl/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/async-curl/.gitignore 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,6 @@ +*.o +.flavor +.objs +ht.so +assets.h +cert diff -Nru kore-2.0.0/examples/async-curl/README.md kore-3.3.1/examples/async-curl/README.md --- kore-2.0.0/examples/async-curl/README.md 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/async-curl/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,12 @@ +Kore asynchronous libcurl integration example. + +This example demonstrates how you can use the asynchronous libcurl +api from Kore to perform HTTP client requests, or FTP requests, or send +emails all in an asynchronous fashion. + +Run: +``` + $ kodev run + $ curl https://127.0.0.1:8888 + $ curl https://127.0.0.1:8888/ftp +``` diff -Nru kore-2.0.0/examples/async-curl/src/ftp.c kore-3.3.1/examples/async-curl/src/ftp.c --- kore-2.0.0/examples/async-curl/src/ftp.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/async-curl/src/ftp.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * This example is the same as the HTTP one (see src/http.c) except + * we fetch an FTP URL. + */ + +#include +#include +#include + +int ftp(struct http_request *); + +static int state_setup(struct http_request *); +static int state_result(struct http_request *); + +static struct http_state states[] = { + KORE_HTTP_STATE(state_setup), + KORE_HTTP_STATE(state_result) +}; + +int +ftp(struct http_request *req) +{ + return (http_state_run(states, 2, req)); +} + +static int +state_setup(struct http_request *req) +{ + struct kore_curl *client; + + client = http_state_create(req, sizeof(*client), NULL); + + if (!kore_curl_init(client, + "http://ftp.eu.openbsd.org/pub/OpenBSD/README")) { + http_response(req, 500, NULL, 0); + return (HTTP_STATE_COMPLETE); + } + + kore_curl_bind_request(client, req); + kore_curl_run(client); + + req->fsm_state = 1; + return (HTTP_STATE_RETRY); +} + +static int +state_result(struct http_request *req) +{ + size_t len; + const u_int8_t *body; + struct kore_curl *client; + + client = http_state_get(req); + + if (!kore_curl_success(client)) { + kore_curl_logerror(client); + http_response(req, 500, NULL, 0); + } else { + kore_curl_response_as_bytes(client, &body, &len); + http_response(req, HTTP_STATUS_OK, body, len); + } + + kore_curl_cleanup(client); + + return (HTTP_STATE_COMPLETE); +} diff -Nru kore-2.0.0/examples/async-curl/src/http.c kore-3.3.1/examples/async-curl/src/http.c --- kore-2.0.0/examples/async-curl/src/http.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/async-curl/src/http.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * This example demonstrates how easy it is to perform asynchronous + * HTTP client requests using the integrated libcurl support. + * + * In this example we setup 2 states for an HTTP request: + * 1) setup + * We initialize the HTTP request and fire it off. + * This will put our HTTP request to sleep and it be woken up + * by Kore when a response is available or something went wrong. + * + * 2) result + * After we have woken up we have access to the result. + */ + +#include +#include +#include + +int http(struct http_request *); + +static int state_setup(struct http_request *); +static int state_result(struct http_request *); + +/* Our states. */ +static struct http_state states[] = { + KORE_HTTP_STATE(state_setup), + KORE_HTTP_STATE(state_result) +}; + +/* Transcend into the HTTP state machine for a request. */ +int +http(struct http_request *req) +{ + return (http_state_run(states, 2, req)); +} + +/* + * Setup the HTTP client request using the integrated curl API and the easy + * to use HTTP client api. + */ +static int +state_setup(struct http_request *req) +{ + struct kore_curl *client; + + client = http_state_create(req, sizeof(*client), NULL); + + /* Initialize curl. */ + if (!kore_curl_init(client, "https://kore.io")) { + http_response(req, 500, NULL, 0); + return (HTTP_STATE_COMPLETE); + } + + /* Setup our HTTP client request. */ + kore_curl_http_setup(client, HTTP_METHOD_GET, NULL, 0); + + /* Add some headers. */ + kore_curl_http_set_header(client, "x-source", "from-example"); + + /* We could opt to override some settings ourselves if we wanted. */ + /* curl_easy_setopt(client->handle, CURLOPT_SSL_VERIFYHOST, 0); */ + /* curl_easy_setopt(client->handle, CURLOPT_SSL_VERIFYPEER, 0); */ + + /* + * Bind the HTTP client request to our HTTP request so we get woken + * up once a response is available. + * + * This will put us to sleep. + */ + kore_curl_bind_request(client, req); + + /* + * Now fire off the request onto the event loop. + */ + kore_curl_run(client); + + /* Make sure we go onto the next state once woken up. */ + req->fsm_state = 1; + + /* Tell Kore we can't complete this immediately. */ + return (HTTP_STATE_RETRY); +} + +/* + * This state is called when a result for the HTTP request call is + * available to us. + */ +static int +state_result(struct http_request *req) +{ + size_t len; + const u_int8_t *body; + const char *header; + struct kore_curl *client; + + /* Get the state attached to the HTTP request. */ + client = http_state_get(req); + + /* Check if we were succesfull, if not log an error. */ + if (!kore_curl_success(client)) { + kore_curl_logerror(client); + http_response(req, 500, NULL, 0); + } else { + /* + * Success! We now have the body available to us. + */ + kore_curl_response_as_bytes(client, &body, &len); + + /* We could check the existance of a header: */ + if (kore_curl_http_get_header(client, "server", &header)) + printf("got server header: '%s'\n", header); + + /* + * Respond to our client with the status and body from + * the HTTP client request we did. + */ + http_response(req, client->http.status, body, len); + } + + /* Cleanup. */ + kore_curl_cleanup(client); + + /* State is now finished. */ + return (HTTP_STATE_COMPLETE); +} diff -Nru kore-2.0.0/examples/cookies/conf/build.conf kore-3.3.1/examples/cookies/conf/build.conf --- kore-2.0.0/examples/cookies/conf/build.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/cookies/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,18 @@ +# generic build config +# You can switch flavors using: kodev flavor [newflavor] + +# The cflags below are shared between flavors +cflags=-Wall -Wmissing-declarations -Wshadow +cflags=-Wstrict-prototypes -Wmissing-prototypes +cflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +dev { + # These cflags are added to the shared ones when + # you build the "dev" flavor. + cflags=-g +} + +#prod { +# You can specify additional CFLAGS here which are only +# included if you build with the "prod" flavor. +#} diff -Nru kore-2.0.0/examples/cookies/conf/cookies.conf kore-3.3.1/examples/cookies/conf/cookies.conf --- kore-2.0.0/examples/cookies/conf/cookies.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/cookies/conf/cookies.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,15 @@ +# Placeholder configuration + +bind 127.0.0.1 8888 +load ./cookies.so + +tls_dhparam dh2048.pem + +domain * { + certfile cert/server.pem + certkey cert/key.pem + + static / serve_cookies + static /secure serve_cookies + static /vault serve_cookies +} diff -Nru kore-2.0.0/examples/cookies/README.md kore-3.3.1/examples/cookies/README.md --- kore-2.0.0/examples/cookies/README.md 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/cookies/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,10 @@ +This example shows cookies API usage + +* Simple key value cookie +* Complex cookie with RFC 6265 features +* Mix with cookie formatted in the header + +Run: +``` + # kodev run +``` diff -Nru kore-2.0.0/examples/cookies/src/cookies.c kore-3.3.1/examples/cookies/src/cookies.c --- kore-2.0.0/examples/cookies/src/cookies.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/cookies/src/cookies.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017 Stanislav Yudin + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include + +static char *html = "

Reload this page

"; + +int serve_cookies(struct http_request *); + +int +serve_cookies(struct http_request *req) +{ + char *value; + struct http_cookie *cookie; + + http_populate_cookies(req); + + if (http_request_cookie(req, "Simple", &value)) + kore_log(LOG_DEBUG, "Got simple: %s", value); + if (http_request_cookie(req, "Complex", &value)) + kore_log(LOG_DEBUG, "Got complex: %s", value); + if (http_request_cookie(req, "Formatted", &value)) + kore_log(LOG_DEBUG, "Got formatted: %s", value); + + /* no expire, no maxage for current path. */ + http_response_cookie(req, "Simple", "Hello World!", + req->path, 0, 0, NULL); + + /* expire, no maxage, for /secure. */ + http_response_cookie(req, "Complex", "Secure Value!", "/secure", + time(NULL) + (1 * 60 * 60), 0, NULL); + + /* maxage, no httponly, for current path. */ + http_response_cookie(req, "key", "value", req->path, 0, 60, &cookie); + cookie->flags &= ~HTTP_COOKIE_HTTPONLY; + + /* set formatted cookie via header directly. */ + http_response_header(req, "set-cookie", + "Formatted=TheValue; Path=/vault; HttpOnly"); + + http_response(req, 200, html, strlen(html)); + + return (KORE_RESULT_OK); +} diff -Nru kore-2.0.0/examples/cpp/conf/build.conf kore-3.3.1/examples/cpp/conf/build.conf --- kore-2.0.0/examples/cpp/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/cpp/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # cpp build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/cpp/conf/cpp.conf kore-3.3.1/examples/cpp/conf/cpp.conf --- kore-2.0.0/examples/cpp/conf/cpp.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/cpp/conf/cpp.conf 2019-06-03 13:29:24.000000000 +0000 @@ -4,8 +4,8 @@ load ./cpp.so tls_dhparam dh2048.pem -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / page } diff -Nru kore-2.0.0/examples/cpp/README.md kore-3.3.1/examples/cpp/README.md --- kore-2.0.0/examples/cpp/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/cpp/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -10,15 +10,15 @@ ``` In order to run this example with the default C++ settings (default compiler dialect, libstdc++): ``` - # kore run + # kodev run ``` In order to run with a specific dialect and C++ runtime: ``` - # env CXXSTD=c++11 CXXLIB=c++ kore run + # env CXXSTD=c++11 CXXLIB=c++ kodev run ``` You can also supply your own compiler combined with the above: ``` - # env CC=clang++ CXXSTD=c++11 CXXLIB=c++ kore run + # env CC=clang++ CXXSTD=c++11 CXXLIB=c++ kodev run ``` diff -Nru kore-2.0.0/examples/generic/conf/build.conf kore-3.3.1/examples/generic/conf/build.conf --- kore-2.0.0/examples/generic/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/generic/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,11 +1,15 @@ # generic build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow cflags=-Wstrict-prototypes -Wmissing-prototypes cflags=-Wpointer-arith -Wcast-qual -Wsign-compare +mime_add=jpg:image/jpg +mime_add=css:text/css; charset=utf-8 +mime_add=html:text/html; charset=utf-8 + dev { # These cflags are added to the shared ones when # you build the "dev" flavor. diff -Nru kore-2.0.0/examples/generic/conf/generic.conf kore-3.3.1/examples/generic/conf/generic.conf --- kore-2.0.0/examples/generic/conf/generic.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/generic/conf/generic.conf 2019-06-03 13:29:24.000000000 +0000 @@ -20,21 +20,21 @@ authentication_uri /private } -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem accesslog kore_access.log - static /css/style.css serve_style_css - static / serve_index - static /intro.jpg serve_intro + static /css/style.css asset_serve_style_css + static / asset_serve_index_html + static /intro.jpg asset_serve_intro_jpg static /b64test serve_b64test static /upload serve_file_upload static /validator serve_validator static /params-test serve_params_test static /private serve_private - static /private/test serve_private_test auth_example + static /private/test asset_serve_private_test_html auth_example params post /params-test { validate test1 v_example diff -Nru kore-2.0.0/examples/generic/README.md kore-3.3.1/examples/generic/README.md --- kore-2.0.0/examples/generic/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/generic/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -7,5 +7,5 @@ Run: ``` - # kore run + # kodev run ``` diff -Nru kore-2.0.0/examples/generic/src/example.c kore-3.3.1/examples/generic/src/example.c --- kore-2.0.0/examples/generic/src/example.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/generic/src/example.c 2019-06-03 13:29:24.000000000 +0000 @@ -27,9 +27,6 @@ int example_load(int); -int serve_style_css(struct http_request *); -int serve_index(struct http_request *); -int serve_intro(struct http_request *); int serve_b64test(struct http_request *); int serve_file_upload(struct http_request *); int serve_validator(struct http_request *); @@ -58,6 +55,9 @@ switch (state) { case KORE_MODULE_LOAD: kore_log(LOG_NOTICE, "module loading"); + + /* Set server version */ + http_server_version("Server/0.1"); break; case KORE_MODULE_UNLOAD: kore_log(LOG_NOTICE, "module unloading"); @@ -71,50 +71,6 @@ } int -serve_style_css(struct http_request *req) -{ - char *date; - time_t tstamp; - - tstamp = 0; - if (http_request_header(req, "if-modified-since", &date)) { - tstamp = kore_date_to_time(date); - kore_debug("header was present with %ld", tstamp); - } - - if (tstamp != 0 && tstamp <= asset_mtime_style_css) { - http_response(req, 304, NULL, 0); - } else { - date = kore_time_to_date(asset_mtime_style_css); - if (date != NULL) - http_response_header(req, "last-modified", date); - - http_response_header(req, "content-type", "text/css"); - http_response(req, 200, asset_style_css, asset_len_style_css); - } - - return (KORE_RESULT_OK); -} - -int -serve_index(struct http_request *req) -{ - http_response_header(req, "content-type", "text/html"); - http_response(req, 200, asset_index_html, asset_len_index_html); - - return (KORE_RESULT_OK); -} - -int -serve_intro(struct http_request *req) -{ - http_response_header(req, "content-type", "image/jpg"); - http_response(req, 200, asset_intro_jpg, asset_len_intro_jpg); - - return (KORE_RESULT_OK); -} - -int serve_b64test(struct http_request *req) { int i; @@ -311,17 +267,6 @@ return (KORE_RESULT_OK); } - -int -serve_private_test(struct http_request *req) -{ - http_response_header(req, "content-type", "text/html"); - - http_response(req, 200, asset_private_test_html, - asset_len_private_test_html); - - return (KORE_RESULT_OK); -} int v_example_func(struct http_request *req, char *data) diff -Nru kore-2.0.0/examples/headers/conf/build.conf kore-3.3.1/examples/headers/conf/build.conf --- kore-2.0.0/examples/headers/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/headers/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # headers build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/headers/conf/headers.conf kore-3.3.1/examples/headers/conf/headers.conf --- kore-2.0.0/examples/headers/conf/headers.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/headers/conf/headers.conf 2019-06-03 13:29:24.000000000 +0000 @@ -5,8 +5,8 @@ tls_dhparam dh2048.pem -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / page } diff -Nru kore-2.0.0/examples/headers/README.md kore-3.3.1/examples/headers/README.md --- kore-2.0.0/examples/headers/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/headers/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -2,7 +2,7 @@ Run: ``` - # kore run + # kodev run ``` Test: diff -Nru kore-2.0.0/examples/headers/src/headers.c kore-3.3.1/examples/headers/src/headers.c --- kore-2.0.0/examples/headers/src/headers.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/headers/src/headers.c 2019-06-03 13:29:24.000000000 +0000 @@ -6,7 +6,7 @@ int page(struct http_request *req) { - char *custom; + const char *custom; /* * We'll lookup if the X-Custom-Header is given in the request. diff -Nru kore-2.0.0/examples/integers/conf/build.conf kore-3.3.1/examples/integers/conf/build.conf --- kore-2.0.0/examples/integers/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/integers/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # integers build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/integers/conf/integers.conf kore-3.3.1/examples/integers/conf/integers.conf --- kore-2.0.0/examples/integers/conf/integers.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/integers/conf/integers.conf 2019-06-03 13:29:24.000000000 +0000 @@ -8,14 +8,15 @@ tls_dhparam dh2048.pem -validator v_id regex ^-?[0-9]*$ +validator v_id regex ^-?[0-9]*.?[0-9]+$ -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / page - params get / { + # allowed parameters in the query string for GETs + params qs:get / { validate id v_id } } diff -Nru kore-2.0.0/examples/integers/README.md kore-3.3.1/examples/integers/README.md --- kore-2.0.0/examples/integers/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/integers/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -2,7 +2,7 @@ Run: ``` - # kore run + # kodev run ``` Test: diff -Nru kore-2.0.0/examples/integers/src/check_integers.c kore-3.3.1/examples/integers/src/check_integers.c --- kore-2.0.0/examples/integers/src/check_integers.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/integers/src/check_integers.c 2019-06-03 13:29:24.000000000 +0000 @@ -6,6 +6,8 @@ int page(struct http_request *req) { + float fl; + double dbl; int16_t s16; u_int16_t u16; int32_t s32; @@ -40,6 +42,12 @@ if (http_argument_get_uint64(req, "id", &u64)) kore_buf_appendf(buf, "uint64\t%lu\n", u64); + if (http_argument_get_float(req, "id", &fl)) + kore_buf_appendf(buf, "float\t%g\n", fl); + + if (http_argument_get_double(req, "id", &dbl)) + kore_buf_appendf(buf, "double\t%g\n", dbl); + data = kore_buf_release(buf, &len); http_response(req, 200, data, len); kore_free(data); diff -Nru kore-2.0.0/examples/jsonrpc/conf/build.conf kore-3.3.1/examples/jsonrpc/conf/build.conf --- kore-2.0.0/examples/jsonrpc/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/jsonrpc/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # jsonrpc build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/jsonrpc/conf/jsonrpc.conf kore-3.3.1/examples/jsonrpc/conf/jsonrpc.conf --- kore-2.0.0/examples/jsonrpc/conf/jsonrpc.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/jsonrpc/conf/jsonrpc.conf 2019-06-03 13:29:24.000000000 +0000 @@ -5,9 +5,9 @@ tls_dhparam dh2048.pem -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / homepage static /v1 v1 diff -Nru kore-2.0.0/examples/jsonrpc/README.md kore-3.3.1/examples/jsonrpc/README.md --- kore-2.0.0/examples/jsonrpc/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/jsonrpc/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -12,7 +12,7 @@ Run: ``` - $ kore run + $ kodev run ``` Test: diff -Nru kore-2.0.0/examples/json_yajl/conf/build.conf kore-3.3.1/examples/json_yajl/conf/build.conf --- kore-2.0.0/examples/json_yajl/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/json_yajl/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # json_yajl build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/json_yajl/conf/json_yajl.conf kore-3.3.1/examples/json_yajl/conf/json_yajl.conf --- kore-2.0.0/examples/json_yajl/conf/json_yajl.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/json_yajl/conf/json_yajl.conf 2019-06-03 13:29:24.000000000 +0000 @@ -6,8 +6,8 @@ tls_dhparam dh2048.pem domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key + certfile cert/server.pem + certkey cert/key.pem static / page } diff -Nru kore-2.0.0/examples/json_yajl/README.md kore-3.3.1/examples/json_yajl/README.md --- kore-2.0.0/examples/json_yajl/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/json_yajl/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -8,7 +8,7 @@ Run: ``` - $ kore run + $ kodev run ``` Test: diff -Nru kore-2.0.0/examples/json_yajl/src/json_yajl.c kore-3.3.1/examples/json_yajl/src/json_yajl.c --- kore-2.0.0/examples/json_yajl/src/json_yajl.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/json_yajl/src/json_yajl.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2018 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff -Nru kore-2.0.0/examples/ktunnel/client/client.c kore-3.3.1/examples/ktunnel/client/client.c --- kore-2.0.0/examples/ktunnel/client/client.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/ktunnel/client/client.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,685 +0,0 @@ -/* - * Copyright (c) 2014 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include -#include -#include -#include - -#include -#include - -#include "openssl/err.h" -#include "openssl/ssl.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#define errno_s strerror(errno) -#define ssl_errno_s ERR_error_string(ERR_get_error(), NULL) - -#define KQUEUE_EVENT_COUNT 100 -#define NETBUF_RECV_MAX 8192 - -#define HTTP_REQUEST_FMT \ - "GET %s?host=%s&port=%s HTTP/1.1\r\nHost: %s\r\n\r\n" - -struct netbuf { - u_int8_t *data; - u_int32_t offset; - u_int32_t length; - - TAILQ_ENTRY(netbuf) list; -}; - -TAILQ_HEAD(netbuf_list, netbuf); - -#define PEER_CAN_READ 0x01 -#define PEER_CAN_WRITE 0x02 - -struct peer { - int fd; - int family; - int flags; - - char *name; - char *host; - char *port; - - int (*write)(struct peer *); - int (*read)(struct peer *, struct peer *); - - SSL *ssl; - SSL_CTX *ssl_ctx; - void *connection; - struct peer *opposite; - - struct netbuf *recv_buf; - struct netbuf_list write_queue; -}; - -#define CONNECTION_WILL_DISCONNECT 0x01 - -struct connection { - int flags; - struct peer local; - struct peer remote; - TAILQ_ENTRY(connection) list; -}; - -TAILQ_HEAD(, connection) clients; -TAILQ_HEAD(, connection) disconnects; - -void usage(void); -void fatal(const char *, ...); - -int ktunnel_peer_handle(struct peer *); -void ktunnel_peer_cleanup(struct peer *); -void ktunnel_connection_close(struct connection *); -void ktunnel_connection_cleanup(struct connection *); - -void ktunnel_event_schedule(int, int, int, void *); - -void ktunnel_set_nonblock(int); -int ktunnel_write_local(struct peer *); -int ktunnel_write_remote(struct peer *); -int ktunnel_read_local(struct peer *, struct peer *); -int ktunnel_read_remote(struct peer *, struct peer *); - -void ktunnel_accept(struct peer *); -void ktunnel_bind(struct peer *, struct addrinfo *); -void ktunnel_connect(struct peer *, struct addrinfo *); -void ktunnel_peer_init(struct peer *, const char *, - void (*cb)(struct peer *, struct addrinfo *)); - -void ktunnel_netbuf_create(struct netbuf **, struct netbuf_list *, - u_int8_t *, u_int32_t); - -int kfd = - 1; -u_int32_t nchanges = 0; -struct kevent *events = NULL; -struct kevent *changelist = NULL; -char *target_host = NULL; -char *target_port = NULL; -char *remote_name = NULL; -char *http_hostname = NULL; -char *http_path = "/connect"; - -void -usage(void) -{ - fprintf(stderr, - "Usage: ktunnel-client [-h host] [-p path] " - "local:port remote:port target:port\n"); - - exit(1); -} - -int -main(int argc, char *argv[]) -{ - int n, i, ch; - struct connection *c, *cnext; - struct peer lpeer, *peer; - - while ((ch = getopt(argc, argv, "h:p:")) != -1) { - switch (ch) { - case 'h': - http_hostname = optarg; - break; - case 'p': - http_path = optarg; - break; - default: - usage(); - } - } - - argc -= optind; - argv += optind; - - if (argc != 3) - usage(); - - TAILQ_INIT(&clients); - TAILQ_INIT(&disconnects); - - if ((kfd = kqueue()) == -1) - fatal("kqueue(): %s", errno_s); - - nchanges = 0; - events = calloc(KQUEUE_EVENT_COUNT, sizeof(struct kevent)); - changelist = calloc(KQUEUE_EVENT_COUNT, sizeof(struct kevent)); - if (events == NULL || changelist == NULL) - fatal("calloc(): %s", errno_s); - - memset(&lpeer, 0, sizeof(lpeer)); - ktunnel_peer_init(&lpeer, argv[0], ktunnel_bind); - ktunnel_event_schedule(lpeer.fd, EVFILT_READ, EV_ADD, &lpeer); - - remote_name = argv[1]; - target_host = argv[2]; - - if ((target_port = strchr(target_host, ':')) == NULL) - fatal("Target host does not contain a port"); - *(target_port)++ = '\0'; - - if (http_hostname == NULL) - http_hostname = target_host; - - for (;;) { - n = kevent(kfd, changelist, nchanges, - events, KQUEUE_EVENT_COUNT, NULL); - if (n == -1) { - if (errno == EINTR) - continue; - fatal("kevent(): %s", errno_s); - } - - nchanges = 0; - for (i = 0; i < n; i++) { - if (events[i].udata == NULL) - fatal("events[%d].udata == NULL", i); - - peer = (struct peer *)events[i].udata; - - if (events[i].flags & EV_EOF || - events[i].flags & EV_ERROR) { - if (peer->fd == lpeer.fd) - fatal("error on listening socket"); - - ktunnel_connection_close(peer->connection); - continue; - } - - if (peer->fd == lpeer.fd) { - ktunnel_accept(peer); - continue; - } - - if (events[i].filter == EVFILT_READ) - peer->flags |= PEER_CAN_READ; - if (events[i].filter == EVFILT_WRITE) - peer->flags |= PEER_CAN_WRITE; - - if (ktunnel_peer_handle(peer) == -1) { - ktunnel_connection_close(peer->connection); - } else { - if (!TAILQ_EMPTY(&peer->write_queue)) { - ktunnel_event_schedule(peer->fd, - EVFILT_WRITE, - EV_ADD | EV_ONESHOT, peer); - } - } - } - - for (c = TAILQ_FIRST(&disconnects); c != NULL; c = cnext) { - cnext = TAILQ_NEXT(c, list); - TAILQ_REMOVE(&disconnects, c, list); - ktunnel_connection_cleanup(c); - } - } - - return (0); -} - -void -ktunnel_peer_init(struct peer *peer, const char *name, void (*cb)(struct peer *, - struct addrinfo *)) -{ - int r; - struct addrinfo *ai, *results; - - if ((peer->name = strdup(name)) == NULL) - fatal("strdup() messed up"); - - peer->host = peer->name; - if ((peer->port = strchr(peer->host, ':')) == NULL) - fatal("No port section in given local host '%s'", peer->name); - *(peer->port)++ = '\0'; - - r = getaddrinfo(peer->host, peer->port, NULL, &results); - if (r != 0) - fatal("%s: %s", name, gai_strerror(r)); - - for (ai = results; ai != NULL; ai = ai->ai_next) { - if (ai->ai_socktype != SOCK_STREAM) - continue; - if (ai->ai_family != AF_INET && ai->ai_family != AF_INET6) - continue; - - cb(peer, ai); - peer->family = ai->ai_family; - - break; - } - - freeaddrinfo(results); -} - -void -ktunnel_accept(struct peer *peer) -{ - int fd; - struct connection *c; - struct sockaddr_in sin4; - struct sockaddr_in6 sin6; - struct sockaddr *sin; - socklen_t slen; - - sin = NULL; - - switch (peer->family) { - case AF_INET: - sin = (struct sockaddr *)&sin4; - slen = sizeof(struct sockaddr_in); - break; - case AF_INET6: - sin = (struct sockaddr *)&sin6; - slen = sizeof(struct sockaddr_in6); - break; - default: - fatal("Unknown peer family %d", peer->family); - /* NOTREACHED */ - } - - if ((fd = accept(peer->fd, sin, &slen)) == -1) - fatal("accept(): %s", errno_s); - - if ((c = malloc(sizeof(*c))) == NULL) - fatal("malloc(): %s", errno_s); - - memset(c, 0, sizeof(*c)); - c->local.fd = fd; - TAILQ_INIT(&c->local.write_queue); - - ktunnel_event_schedule(c->local.fd, EVFILT_READ, EV_ADD, &c->local); - ktunnel_event_schedule(c->local.fd, EVFILT_WRITE, - EV_ADD | EV_ONESHOT, &c->local); - - c->local.connection = c; - c->local.opposite = &c->remote; - c->local.read = ktunnel_read_local; - c->local.write = ktunnel_write_local; - - c->remote.connection = c; - c->remote.opposite = &c->local; - c->remote.read = ktunnel_read_remote; - c->remote.write = ktunnel_write_remote; - - ktunnel_peer_init(&c->remote, remote_name, ktunnel_connect); - ktunnel_netbuf_create(&c->local.recv_buf, - NULL, NULL, NETBUF_RECV_MAX); - - ktunnel_set_nonblock(c->local.fd); - ktunnel_set_nonblock(c->remote.fd); - - TAILQ_INSERT_TAIL(&clients, c, list); - - printf("new connection %p (%p<->%p)\n", c, &c->local, &c->remote); -} - -void -ktunnel_bind(struct peer *peer, struct addrinfo *ai) -{ - if ((peer->fd = socket(ai->ai_family, ai->ai_socktype, 0)) == -1) - fatal("socket(): %s", errno_s); - - if (bind(peer->fd, ai->ai_addr, ai->ai_addrlen) == -1) { - fatal("Cannot bind to %s:%s: %s", - peer->host, peer->port, errno_s); - } - - if (listen(peer->fd, 10) == -1) - fatal("Cannot listen on socket: %s", errno_s); - - TAILQ_INIT(&peer->write_queue); - ktunnel_netbuf_create(&peer->recv_buf, NULL, NULL, NETBUF_RECV_MAX); -} - -void -ktunnel_connect(struct peer *peer, struct addrinfo *ai) -{ - int l; - char *req; - - if ((peer->fd = socket(ai->ai_family, ai->ai_socktype, 0)) == -1) - fatal("socket(): %s", errno_s); - - if (connect(peer->fd, ai->ai_addr, ai->ai_addrlen) == -1) { - fatal("Cannot connect to %s:%s: %s", - peer->host, peer->port, errno_s); - } - - TAILQ_INIT(&peer->write_queue); - ktunnel_netbuf_create(&peer->recv_buf, NULL, NULL, NETBUF_RECV_MAX); - - /* - * XXX - * - Add our client certs - * - Verify server cert properly - * - ... - */ - SSL_library_init(); - SSL_load_error_strings(); - - if ((peer->ssl_ctx = SSL_CTX_new(SSLv23_method())) == NULL) - fatal("SSL_CTX_new(): %s", ssl_errno_s); - - SSL_CTX_set_mode(peer->ssl_ctx, SSL_MODE_AUTO_RETRY); - SSL_CTX_set_options(peer->ssl_ctx, SSL_OP_NO_SSLv2); - SSL_CTX_set_options(peer->ssl_ctx, SSL_OP_NO_SSLv3); - SSL_CTX_set_options(peer->ssl_ctx, SSL_OP_NO_TLSv1); - SSL_CTX_set_options(peer->ssl_ctx, SSL_OP_NO_TLSv1_1); - - if ((peer->ssl = SSL_new(peer->ssl_ctx)) == NULL) - fatal("SSL_new(): %s", ssl_errno_s); - if (!SSL_set_fd(peer->ssl, peer->fd)) - fatal("SSL_set_fd(): %s", ssl_errno_s); - if (!SSL_connect(peer->ssl)) { - fatal("Could not establish an SSL connection to %s: %s", - peer->host, ssl_errno_s); - } - - /* Send custom HTTP command. */ - l = asprintf(&req, HTTP_REQUEST_FMT, http_path, - target_host, target_port, http_hostname); - if (l == -1) - fatal("asprintf(): %s", errno_s); - - if (SSL_write(peer->ssl, req, l) != l) { - fatal("Failed to talk to %s:%s: %s", - peer->host, peer->port, ssl_errno_s); - } - - free(req); - - ktunnel_event_schedule(peer->fd, EVFILT_READ, EV_ADD, peer); - ktunnel_event_schedule(peer->fd, EVFILT_WRITE, - EV_ADD | EV_ONESHOT, peer); - - printf("Connected over SSL to %s:%s\n", peer->host, peer->port); -} - -int -ktunnel_peer_handle(struct peer *peer) -{ - int r; - - printf("handling peer %p (%d)\n", peer, peer->flags); - - if (peer->flags & PEER_CAN_READ) { - printf("\treading\n"); - r = peer->read(peer, peer->opposite); - } - - if (peer->flags & PEER_CAN_WRITE) { - printf("\twriting\n"); - r = peer->write(peer); - } - - return (r); -} - -void -ktunnel_connection_close(struct connection *c) -{ - printf("ktunnel_connection_close(%p)\n", c); - - if (!(c->flags & CONNECTION_WILL_DISCONNECT)) { - c->flags |= CONNECTION_WILL_DISCONNECT; - - TAILQ_REMOVE(&clients, c, list); - TAILQ_INSERT_TAIL(&disconnects, c, list); - } -} - -void -ktunnel_connection_cleanup(struct connection *c) -{ - ktunnel_peer_cleanup(&c->local); - ktunnel_peer_cleanup(&c->remote); - - free(c); -} - -void -ktunnel_peer_cleanup(struct peer *peer) -{ - struct netbuf *nb, *next; - - printf("ktunnel_peer_cleanup(%p)\n", peer); - - close(peer->fd); - - if (peer->ssl != NULL) - SSL_free(peer->ssl); - if (peer->ssl_ctx != NULL) - SSL_CTX_free(peer->ssl_ctx); - - for (nb = TAILQ_FIRST(&peer->write_queue); nb != NULL; nb = next) { - next = TAILQ_NEXT(nb, list); - TAILQ_REMOVE(&peer->write_queue, nb, list); - - free(nb->data); - free(nb); - } - - free(peer->recv_buf->data); -} - -void -ktunnel_netbuf_create(struct netbuf **out, struct netbuf_list *head, - u_int8_t *data, u_int32_t length) -{ - struct netbuf *nb; - - if ((nb = malloc(sizeof(struct netbuf))) == NULL) - fatal("malloc(): %s", errno_s); - - nb->offset = 0; - nb->length = length; - - if ((nb->data = malloc(nb->length)) == NULL) - fatal("malloc(): %s", errno_s); - - if (data != NULL) - memcpy(nb->data, data, nb->length); - - if (head != NULL) - TAILQ_INSERT_TAIL(head, nb, list); - - if (out != NULL) - *out = nb; -} - -void -ktunnel_event_schedule(int fd, int type, int flags, void *udata) -{ - if (nchanges >= KQUEUE_EVENT_COUNT) - fatal("nchanges > KQUEUE_EVENT_COUNT"); - - EV_SET(&changelist[nchanges], fd, type, flags, 0, 0, udata); - nchanges++; -} - -int -ktunnel_read_local(struct peer *in, struct peer *out) -{ - int r; - - printf("ktunnel_read_local: %p\n", in); - - r = read(in->fd, in->recv_buf->data, in->recv_buf->length); - if (r == -1) { - if (errno != EINTR && errno != EAGAIN) { - printf("read error on local peer: %s\n", errno_s); - return (-1); - } - - return (0); - } - - if (r == 0) { - printf("local peer closed connection\n"); - return (-1); - } - - printf("ktunnel_read_local: %p -- %d bytes --> %p\n", in, r, out); - - ktunnel_netbuf_create(NULL, &(out->write_queue), in->recv_buf->data, r); - return (ktunnel_write_remote(out)); -} - -int -ktunnel_write_local(struct peer *peer) -{ - int r; - struct netbuf *nb; - - while (!TAILQ_EMPTY(&peer->write_queue)) { - nb = TAILQ_FIRST(&peer->write_queue); - - printf("ktunnel_write_local: %p writing %d/%d\n", peer, - nb->offset, nb->length); - - r = write(peer->fd, (nb->data + nb->offset), - (nb->length - nb->offset)); - if (r == -1) { - switch (errno) { - case EINTR: - case EAGAIN: - peer->flags &= ~PEER_CAN_WRITE; - return (0); - default: - printf("failed to write to local peer: %s\n", - errno_s); - return (-1); - } - } - - nb->offset += r; - printf("ktunnel_write_local: %p progress %d/%d\n", peer, - nb->offset, nb->length); - - if (nb->offset == nb->length) { - TAILQ_REMOVE(&peer->write_queue, nb, list); - free(nb->data); - free(nb); - } - } - - return (0); -} - -int -ktunnel_read_remote(struct peer *in, struct peer *out) -{ - int r; - - r = SSL_read(in->ssl, in->recv_buf->data, in->recv_buf->length); - if (r <= 0) { - r = SSL_get_error(in->ssl, r); - switch (r) { - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - in->flags &= ~PEER_CAN_READ; - return (0); - default: - printf("failed to read from remote peer: %d, %s\n", - r, ssl_errno_s); - return (-1); - } - } - - ktunnel_netbuf_create(NULL, &(out->write_queue), in->recv_buf->data, r); - return (ktunnel_write_local(out)); -} - -int -ktunnel_write_remote(struct peer *peer) -{ - int r; - struct netbuf *nb; - - while (!TAILQ_EMPTY(&peer->write_queue)) { - nb = TAILQ_FIRST(&peer->write_queue); - - printf("ktunnel_write_remote: %p writing %d/%d bytes\n", peer, - nb->offset, nb->length); - - r = SSL_write(peer->ssl, (nb->data + nb->offset), - (nb->length - nb->offset)); - if (r <= 0) { - r = SSL_get_error(peer->ssl, r); - switch (r) { - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - peer->flags &= ~PEER_CAN_WRITE; - return (0); - default: - printf("failed to write to remote peer: %s\n", - ssl_errno_s); - return (-1); - } - } - - nb->offset += r; - printf("ktunnel_write_remote: %p progress %d/%d\n", peer, - nb->offset, nb->length); - - if (nb->offset == nb->length) { - TAILQ_REMOVE(&peer->write_queue, nb, list); - free(nb->data); - free(nb); - } - } - - return (0); -} - -void -ktunnel_set_nonblock(int fd) -{ - int flags; - - if ((flags = fcntl(fd, F_GETFL, 0)) == -1) - fatal("fcntl(): get %s", errno_s); - - flags |= O_NONBLOCK; - if (fcntl(fd, F_SETFL, flags) == -1) - fatal("fnctl(): set %s", errno_s); -} - -void -fatal(const char *fmt, ...) -{ - va_list args; - - va_start(args, fmt); - vprintf(fmt, args); - va_end(args); - - printf("\n"); - - exit(1); -} diff -Nru kore-2.0.0/examples/ktunnel/client/Makefile kore-3.3.1/examples/ktunnel/client/Makefile --- kore-2.0.0/examples/ktunnel/client/Makefile 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/ktunnel/client/Makefile 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -# -# You probably want to change the include and library -# paths before compiling. -# - -CFLAGS+=-Wall -Wstrict-prototypes -Wmissing-prototypes -CFLAGS+=-Wmissing-declarations -Wshadow -Wpointer-arith -Wcast-qual -CFLAGS+=-Wsign-compare -Iincludes -g -CFLAGS+=-I../../openssl-1.0.1i/include - -all: - gcc $(CFLAGS) -c client.c -o client.o - gcc -L../../openssl-1.0.1i/ client.o -o client -lcrypto -lssl - -clean: - rm -f client *.o diff -Nru kore-2.0.0/examples/ktunnel/conf/build.conf kore-3.3.1/examples/ktunnel/conf/build.conf --- kore-2.0.0/examples/ktunnel/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/ktunnel/conf/build.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,18 +0,0 @@ -# ktunnel build config -# You can switch flavors using: kore flavor [newflavor] - -# The cflags below are shared between flavors -cflags=-Wall -Wmissing-declarations -Wshadow -cflags=-Wstrict-prototypes -Wmissing-prototypes -cflags=-Wpointer-arith -Wcast-qual -Wsign-compare - -dev { - # These cflags are added to the shared ones when - # you build the "dev" flavor. - cflags=-g -} - -#prod { -# You can specify additional CFLAGS here which are only -# included if you build with the "prod" flavor. -#} diff -Nru kore-2.0.0/examples/ktunnel/conf/ktunnel.conf kore-3.3.1/examples/ktunnel/conf/ktunnel.conf --- kore-2.0.0/examples/ktunnel/conf/ktunnel.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/ktunnel/conf/ktunnel.conf 1970-01-01 00:00:00.000000000 +0000 @@ -1,25 +0,0 @@ -# kTunnel configuration - -bind 127.0.0.1 8888 -load ./ktunnel.so - -tls_dhparam dh2048.pem - -# Regexes here are incorrect. -validator v_host regex ^.*$ -validator v_port regex ^[0-9]*$ - -# Disable timeouts -http_keepalive_time 0 - -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key - - static /connect open_connection - - params get /connect { - validate host v_host - validate port v_port - } -} diff -Nru kore-2.0.0/examples/ktunnel/.gitignore kore-3.3.1/examples/ktunnel/.gitignore --- kore-2.0.0/examples/ktunnel/.gitignore 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/ktunnel/.gitignore 1970-01-01 00:00:00.000000000 +0000 @@ -1,5 +0,0 @@ -*.o -.objs -ktunnel.so -assets.h -cert diff -Nru kore-2.0.0/examples/ktunnel/README.md kore-3.3.1/examples/ktunnel/README.md --- kore-2.0.0/examples/ktunnel/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/ktunnel/README.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,35 +0,0 @@ -KTunnel (anything over HTTPS) - -This example demonstrates how we can use Kore to create an -anything-over-HTTPS tunnel. - -Build: -``` - # kore build -``` - -Run: -``` - # kore run -``` - -Test: -``` - # openssl s_client -connect 127.0.0.1:8888 - - Then enter: - - GET /connect?host=74.125.232.248&port=80 HTTP/1.1 - Host: 127.0.0.1 - - GET / HTTP/1.1 - Host: www.google.se - - (And hit enter) -``` - -You should see Kore connect to the google server given and -return the results back to you. - -A client for OSX exists under the **client/** directory. It requires -you to link with -lssl and -lcrypto. diff -Nru kore-2.0.0/examples/ktunnel/src/ktunnel.c kore-3.3.1/examples/ktunnel/src/ktunnel.c --- kore-2.0.0/examples/ktunnel/src/ktunnel.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/ktunnel/src/ktunnel.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,188 +0,0 @@ -/* - * Copyright (c) 2014 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include - -#include -#include - -#include - -/* - * KTunnel shows how Kore exposes its net internals to its libraries - * and how we can "abuse" these internals to create a "anything" - * over HTTPS tunnel. - */ - -int open_connection(struct http_request *); - -static int ktunnel_pipe_data(struct netbuf *); -static void ktunnel_pipe_disconnect(struct connection *); -static int ktunnel_pipe_create(struct connection *, - const char *, const char *); - -/* - * Receive a request to open a new connection. - */ -int -open_connection(struct http_request *req) -{ - char *host, *port; - - /* Make sure its HTTP. */ - if (req->owner->proto != CONN_PROTO_HTTP) { - http_response(req, HTTP_STATUS_BAD_REQUEST, NULL, 0); - return (KORE_RESULT_OK); - } - - /* Parse the query string and grab our arguments. */ - http_populate_get(req); - if (!http_argument_get_string(req, "host", &host) || - !http_argument_get_string(req, "port", &port)) { - http_response(req, HTTP_STATUS_BAD_REQUEST, NULL, 0); - return (KORE_RESULT_OK); - } - - /* Create our tunnel. */ - if (!ktunnel_pipe_create(req->owner, host, port)) { - http_response(req, HTTP_STATUS_INTERNAL_ERROR, NULL, 0); - return (KORE_RESULT_OK); - } - - /* - * Hack so http_response() doesn't end up queueing a new - * netbuf for receiving more HTTP requests on the same connection. - */ - req->owner->flags |= CONN_CLOSE_EMPTY; - - /* Respond to the client now that we're good to go. */ - http_response(req, HTTP_STATUS_OK, NULL, 0); - - /* Unset this so we don't disconnect after returning. */ - req->owner->flags &= ~CONN_CLOSE_EMPTY; - - return (KORE_RESULT_OK); -} - -/* - * Connect to our target host:port and attach it to a struct connection that - * Kore understands. We set the disconnect method so we get a callback - * whenever either of the connections will go away so we can cleanup the - * one it is attached to. - */ -static int -ktunnel_pipe_create(struct connection *c, const char *host, const char *port) -{ - struct sockaddr_in sin; - struct connection *cpipe; - u_int16_t nport; - int fd, err; - - nport = kore_strtonum(port, 10, 1, SHRT_MAX, &err); - if (err == KORE_RESULT_ERROR) { - kore_log(LOG_ERR, "invalid port given %s", port); - return (KORE_RESULT_ERROR); - } - - if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { - kore_log(LOG_ERR, "socket(): %s", errno_s); - return (KORE_RESULT_ERROR); - } - - memset(&sin, 0, sizeof(sin)); - sin.sin_family = AF_INET; - sin.sin_port = htons(nport); - sin.sin_addr.s_addr = inet_addr(host); - - kore_log(LOG_NOTICE, "Attempting to connect to %s:%s", host, port); - - if (connect(fd, (struct sockaddr *)&sin, sizeof(sin)) == -1) { - close(fd); - kore_log(LOG_ERR, "connect(): %s", errno_s); - return (KORE_RESULT_ERROR); - } - - if (!kore_connection_nonblock(fd, 1)) { - close(fd); - return (KORE_RESULT_ERROR); - } - - cpipe = kore_connection_new(c); - TAILQ_INSERT_TAIL(&connections, cpipe, list); - - cpipe->fd = fd; - cpipe->addr.ipv4 = sin; - cpipe->read = net_read; - cpipe->write = net_write; - cpipe->addrtype = AF_INET; - cpipe->proto = CONN_PROTO_UNKNOWN; - cpipe->state = CONN_STATE_ESTABLISHED; - - /* Don't let these connections timeout any time soon. */ - cpipe->idle_timer.length = 10000000000; - c->idle_timer.length = 10000000000; - - c->hdlr_extra = cpipe; - cpipe->hdlr_extra = c; - c->disconnect = ktunnel_pipe_disconnect; - cpipe->disconnect = ktunnel_pipe_disconnect; - - kore_connection_start_idletimer(cpipe); - kore_platform_event_all(cpipe->fd, cpipe); - - net_recv_reset(c, NETBUF_SEND_PAYLOAD_MAX, ktunnel_pipe_data); - net_recv_queue(cpipe, NETBUF_SEND_PAYLOAD_MAX, NETBUF_CALL_CB_ALWAYS, - ktunnel_pipe_data); - - printf("connection started to %s (%p -> %p)\n", host, c, cpipe); - return (KORE_RESULT_OK); -} - -/* - * Called everytime new data is read from any of the connections - * that are part of a pipe. - */ -static int -ktunnel_pipe_data(struct netbuf *nb) -{ - struct connection *src = nb->owner; - struct connection *dst = src->hdlr_extra; - - printf("received %zu bytes on pipe %p (-> %p)\n", nb->s_off, src, dst); - - net_send_queue(dst, nb->buf, nb->s_off); - net_send_flush(dst); - net_recv_reset(src, NETBUF_SEND_PAYLOAD_MAX, ktunnel_pipe_data); - - return (KORE_RESULT_OK); -} - -/* - * Called when either part of the pipe disconnects. - */ -static void -ktunnel_pipe_disconnect(struct connection *c) -{ - struct connection *cpipe = c->hdlr_extra; - - printf("ktunnel_pipe_disconnect(%p)->%p\n", c, cpipe); - - if (cpipe != NULL) { - /* Prevent Kore from calling kore_free() on hdlr_extra. */ - c->hdlr_extra = NULL; - kore_connection_disconnect(cpipe); - } -} diff -Nru kore-2.0.0/examples/Makefile kore-3.3.1/examples/Makefile --- kore-2.0.0/examples/Makefile 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/Makefile 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,50 @@ +# +# Build all relevant examples. +# This only exists to quickly test building all examples. +# +# Kore must be built with PGSQL=1 TASKS=1 PYTHON=1 to get all +# of the below examples to build correctly. +# +# Don't run this directly, run it from the top level as +# $ make releng-build-examples +# + +CURDIR= $(shell pwd) +KODEV= /tmp/kore_releng/bin/kodev + +EXAMPLES= async-curl \ + cookies \ + cpp \ + generic \ + headers \ + integers \ + memtag \ + messaging \ + nohttp \ + parameters \ + pgsql \ + pgsql-sync \ + pipe_task \ + python-async \ + python-echo \ + python-pgsql \ + sse \ + tasks \ + tls-proxy \ + upload \ + video_stream \ + websocket \ + +all: + @for example in $(EXAMPLES); do \ + cd $$example; \ + $(KODEV) clean && $(KODEV) build || exit 1; \ + cd $(CURDIR); \ + done + +clean: + @for example in $(EXAMPLES); do \ + cd $$example; \ + $(KODEV) clean; \ + cd $(CURDIR); \ + done diff -Nru kore-2.0.0/examples/memtag/conf/build.conf kore-3.3.1/examples/memtag/conf/build.conf --- kore-2.0.0/examples/memtag/conf/build.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/memtag/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,34 @@ +# memtag build config +# You can switch flavors using: kodev flavor [newflavor] + +# Set to yes if you wish to produce a single binary instead +# of a dynamic library. If you set this to yes you must also +# set kore_source together with kore_flavor. +#single_binary=no +#kore_source=/home/joris/src/kore +#kore_flavor= + +# The flags below are shared between flavors +cflags=-Wall -Wmissing-declarations -Wshadow +cflags=-Wstrict-prototypes -Wmissing-prototypes +cflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +cxxflags=-Wall -Wmissing-declarations -Wshadow +cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +# Mime types for assets served via the builtin asset_serve_* +#mime_add=txt:text/plain; charset=utf-8 +#mime_add=png:image/png +#mime_add=html:text/html; charset=utf-8 + +dev { + # These flags are added to the shared ones when + # you build the "dev" flavor. + cflags=-g + cxxflags=-g +} + +#prod { +# You can specify additional flags here which are only +# included if you build with the "prod" flavor. +#} diff -Nru kore-2.0.0/examples/memtag/conf/memtag.conf kore-3.3.1/examples/memtag/conf/memtag.conf --- kore-2.0.0/examples/memtag/conf/memtag.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/memtag/conf/memtag.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,13 @@ +# memtag configuration + +bind 127.0.0.1 8888 +load ./memtag.so init + +tls_dhparam dh2048.pem + +domain * { + certfile cert/server.pem + certkey cert/key.pem + + static / page +} diff -Nru kore-2.0.0/examples/memtag/.gitignore kore-3.3.1/examples/memtag/.gitignore --- kore-2.0.0/examples/memtag/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/memtag/.gitignore 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,6 @@ +*.o +.flavor +.objs +memtag.so +assets.h +cert diff -Nru kore-2.0.0/examples/memtag/src/memtag.c kore-3.3.1/examples/memtag/src/memtag.c --- kore-2.0.0/examples/memtag/src/memtag.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/memtag/src/memtag.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2017-2018 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include + +/* + * This example demonstrates how dynamically reloadable modules + * can use the memory tagging system in Kore in order to restore + * the global pointers in the module. + */ + +/* Some unique value. */ +#define MEM_TAG_HELLO 100 + +int init(int); +int page(struct http_request *); + +/* Global pointer, gets initialized to NULL when module loads/reloads. */ +char *fixed_ptr = NULL; + +int +init(int state) +{ + /* Ignore unload(s). */ + if (state == KORE_MODULE_UNLOAD) + return (KORE_RESULT_OK); + + printf("fixed_ptr: %p\n", (void *)fixed_ptr); + + /* Attempt to lookup the original pointer. */ + if ((fixed_ptr = kore_mem_lookup(MEM_TAG_HELLO)) == NULL) { + /* Failed, grab a new chunk of memory and tag it. */ + printf(" allocating fixed_ptr for the first time\n"); + fixed_ptr = kore_malloc_tagged(6, MEM_TAG_HELLO); + kore_strlcpy(fixed_ptr, "hello", 6); + } else { + printf(" fixed_ptr address resolved\n"); + } + + printf(" fixed_ptr: %p\n", (void *)fixed_ptr); + printf(" value : %s\n", fixed_ptr); + + return (KORE_RESULT_OK); +} + +int +page(struct http_request *req) +{ + http_response(req, 200, fixed_ptr, strlen(fixed_ptr)); + return (KORE_RESULT_OK); +} diff -Nru kore-2.0.0/examples/messaging/conf/build.conf kore-3.3.1/examples/messaging/conf/build.conf --- kore-2.0.0/examples/messaging/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/messaging/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # messaging build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/messaging/conf/messaging.conf kore-3.3.1/examples/messaging/conf/messaging.conf --- kore-2.0.0/examples/messaging/conf/messaging.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/messaging/conf/messaging.conf 2019-06-03 13:29:24.000000000 +0000 @@ -5,8 +5,9 @@ tls_dhparam dh2048.pem workers 4 -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key - static / page +domain * { + certfile cert/server.pem + certkey cert/key.pem + static / page + static /shutdown page_shutdown } diff -Nru kore-2.0.0/examples/messaging/README.md kore-3.3.1/examples/messaging/README.md --- kore-2.0.0/examples/messaging/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/messaging/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -2,7 +2,7 @@ Run: ``` - # kore run + # kodev run ``` Test: diff -Nru kore-2.0.0/examples/messaging/src/messaging.c kore-3.3.1/examples/messaging/src/messaging.c --- kore-2.0.0/examples/messaging/src/messaging.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/messaging/src/messaging.c 2019-06-03 13:29:24.000000000 +0000 @@ -28,6 +28,7 @@ int init(int); int page(struct http_request *); +int page_shutdown(struct http_request *req); void received_message(struct kore_msg *, const void *); /* Initialization callback. */ @@ -53,8 +54,8 @@ void received_message(struct kore_msg *msg, const void *data) { - kore_log(LOG_INFO, "got message from %u (%d bytes): %.*s", msg->src, - msg->length, msg->length, (const char *)data); + kore_log(LOG_INFO, "got message from %u (%zu bytes): %.*s", msg->src, + msg->length, (int)msg->length, (const char *)data); } /* @@ -72,4 +73,18 @@ http_response(req, 200, NULL, 0); return (KORE_RESULT_OK); +} + +/* + * Page request which will send a message to the parent + * requesting process shutdown. + */ +int +page_shutdown(struct http_request *req) +{ + /* Send shutdown request to parent. */ + kore_msg_send(KORE_MSG_PARENT, KORE_MSG_SHUTDOWN, "1", 1); + + http_response(req, 200, NULL, 0); + return (KORE_RESULT_OK); } diff -Nru kore-2.0.0/examples/nohttp/conf/build.conf kore-3.3.1/examples/nohttp/conf/build.conf --- kore-2.0.0/examples/nohttp/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/nohttp/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # nohttp build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/nohttp/conf/nohttp.conf kore-3.3.1/examples/nohttp/conf/nohttp.conf --- kore-2.0.0/examples/nohttp/conf/nohttp.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/nohttp/conf/nohttp.conf 2019-06-03 13:29:24.000000000 +0000 @@ -16,7 +16,7 @@ # We must still define a domain to use TLS. This might go away # in the future for NOHTTP=1 -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem } diff -Nru kore-2.0.0/examples/nohttp/README.md kore-3.3.1/examples/nohttp/README.md --- kore-2.0.0/examples/nohttp/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/nohttp/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -4,7 +4,7 @@ Run: ``` - $ kore run + $ kodev run ``` Test: diff -Nru kore-2.0.0/examples/nohttp/src/nohttp.c kore-3.3.1/examples/nohttp/src/nohttp.c --- kore-2.0.0/examples/nohttp/src/nohttp.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/nohttp/src/nohttp.c 2019-06-03 13:29:24.000000000 +0000 @@ -56,9 +56,10 @@ * * In this callback you would generally look at the state of the connection * in c->state and perform the required actions like writing / reading using - * net_send_flush() or net_recv_flush() if CONN_SEND_POSSIBLE or - * CONN_READ_POSSIBLE are set respectively. Returning KORE_RESULT_ERROR from - * this callback will disconnect the connection alltogether. + * net_send_flush() or net_recv_flush() if KORE_EVENT_WRITE or + * KORE_EVENT_READ are set respectively in c->evt.flags. + * Returning KORE_RESULT_ERROR from this callback will disconnect the + * connection alltogether. */ int connection_handle(struct connection *c) @@ -78,7 +79,7 @@ { struct connection *c = (struct connection *)nb->owner; - kore_log(LOG_NOTICE, "%p: received %u bytes", c, nb->s_off); + kore_log(LOG_NOTICE, "%p: received %zu bytes", (void *)c, nb->s_off); /* We will just dump these back to the client. */ net_send_queue(c, nb->buf, nb->s_off); diff -Nru kore-2.0.0/examples/parameters/conf/build.conf kore-3.3.1/examples/parameters/conf/build.conf --- kore-2.0.0/examples/parameters/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/parameters/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # parameters build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/parameters/conf/parameters.conf kore-3.3.1/examples/parameters/conf/parameters.conf --- kore-2.0.0/examples/parameters/conf/parameters.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/parameters/conf/parameters.conf 2019-06-03 13:29:24.000000000 +0000 @@ -10,9 +10,9 @@ # it only matches positive numbers. validator v_id regex ^[0-9]*$ -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / page @@ -21,7 +21,7 @@ # If you would want to declare parameters available # to the page handler for POST, swap the 'get' setting # to 'post' instead, Kore takes care of the rest. - params get / { + params qs:get / { # Validate the id parameter with the v_id validator. validate id v_id } diff -Nru kore-2.0.0/examples/parameters/README.md kore-3.3.1/examples/parameters/README.md --- kore-2.0.0/examples/parameters/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/parameters/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -2,7 +2,7 @@ Run: ``` - # kore run + # kodev run ``` Test: diff -Nru kore-2.0.0/examples/pgsql/conf/build.conf kore-3.3.1/examples/pgsql/conf/build.conf --- kore-2.0.0/examples/pgsql/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pgsql/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # pgsql build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/pgsql/conf/pgsql.conf kore-3.3.1/examples/pgsql/conf/pgsql.conf --- kore-2.0.0/examples/pgsql/conf/pgsql.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pgsql/conf/pgsql.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,14 +1,18 @@ # Placeholder configuration -bind 127.0.0.1 8888 load ./pgsql.so init +bind 127.0.0.1 8888 +bind 127.0.0.1 8889 connection_new + tls_dhparam dh2048.pem http_keepalive_time 0 -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem + static / page + static /hello hello } diff -Nru kore-2.0.0/examples/pgsql/README.md kore-3.3.1/examples/pgsql/README.md --- kore-2.0.0/examples/pgsql/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pgsql/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -10,5 +10,5 @@ Run: ``` - # kore run + # kodev run ``` diff -Nru kore-2.0.0/examples/pgsql/src/init.c kore-3.3.1/examples/pgsql/src/init.c --- kore-2.0.0/examples/pgsql/src/init.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/pgsql/src/init.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2017-2018 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include + +#if !defined(KORE_NO_HTTP) +#include +#endif + +int init(int); + +#if !defined(KORE_NO_HTTP) +int hello(struct http_request *); +#endif + +/* Called when our module is loaded (see config) */ +int +init(int state) +{ + /* Register our database. */ + kore_pgsql_register("db", "host=/tmp dbname=test"); + + return (KORE_RESULT_OK); +} + +#if !defined(KORE_NO_HTTP) +int +hello(struct http_request *req) +{ + http_response(req, HTTP_STATUS_OK, "hello", 5); + return (KORE_RESULT_OK); +} +#endif diff -Nru kore-2.0.0/examples/pgsql/src/pgsql.c kore-3.3.1/examples/pgsql/src/pgsql.c --- kore-2.0.0/examples/pgsql/src/pgsql.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pgsql/src/pgsql.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Joris Vink + * Copyright (c) 2014-2018 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -28,11 +28,10 @@ * The state machine framework present in Kore makes it trivial * to get going into dropping from your page handler into the right * state that you are currently in. - * - * The example connects to a local pgsql database (test) using a table - * called "coders" (which has 2 columns): name, surname. */ +#if !defined(KORE_NO_HTTP) + #include #include #include @@ -44,15 +43,14 @@ #define REQ_STATE_ERROR 4 #define REQ_STATE_DONE 5 -int init(int); -int page(struct http_request *); +int page(struct http_request *); -static int request_perform_init(struct http_request *); -static int request_perform_query(struct http_request *); -static int request_db_wait(struct http_request *); -static int request_db_read(struct http_request *); -static int request_error(struct http_request *); -static int request_done(struct http_request *); +static int request_perform_init(struct http_request *); +static int request_perform_query(struct http_request *); +static int request_db_wait(struct http_request *); +static int request_db_read(struct http_request *); +static int request_error(struct http_request *); +static int request_done(struct http_request *); struct http_state mystates[] = { { "REQ_STATE_INIT", request_perform_init }, @@ -70,22 +68,12 @@ struct kore_pgsql sql; }; -/* Called when our module is loaded (see config) */ -int -init(int state) -{ - /* Register our database. */ - kore_pgsql_register("db", "host=/tmp dbname=test"); - - return (KORE_RESULT_OK); -} - /* Page handler entry point (see config) */ int page(struct http_request *req) { /* Drop into our state machine. */ - kore_log(LOG_NOTICE, "page start"); + kore_log(LOG_NOTICE, "%p: page start", (void *)req); return (http_state_run(mystates, mystates_size, req)); } @@ -96,16 +84,30 @@ struct rstate *state; /* Setup our state context (if not yet set). */ - if (req->hdlr_extra == NULL) { - state = kore_malloc(sizeof(*state)); - req->hdlr_extra = state; + if (!http_state_exists(req)) { + state = http_state_create(req, sizeof(*state), NULL); + + /* + * Initialize the kore_pgsql data structure and bind it + * to this request so we can be put to sleep / woken up + * by the pgsql layer when required. + */ + kore_pgsql_init(&state->sql); + kore_pgsql_bind_request(&state->sql, req); } else { - state = req->hdlr_extra; + state = http_state_get(req); } - /* Initialize our kore_pgsql data structure. */ - if (!kore_pgsql_query_init(&state->sql, req, "db", KORE_PGSQL_ASYNC)) { - /* If the state was still INIT, we'll try again later. */ + /* + * Setup the query to be asynchronous in nature, aka just fire it + * off and return back to us. + */ + if (!kore_pgsql_setup(&state->sql, "db", KORE_PGSQL_ASYNC)) { + /* + * If the state was still in INIT we need to go to sleep and + * wait until the pgsql layer wakes us up again when there + * an available connection to the database. + */ if (state->sql.state == KORE_PGSQL_STATE_INIT) { req->fsm_state = REQ_STATE_INIT; return (HTTP_STATE_RETRY); @@ -114,6 +116,9 @@ kore_pgsql_logerror(&state->sql); req->fsm_state = REQ_STATE_ERROR; } else { + /* + * The initial setup was complete, go for query. + */ req->fsm_state = REQ_STATE_QUERY; } @@ -124,13 +129,14 @@ int request_perform_query(struct http_request *req) { - struct rstate *state = req->hdlr_extra; + struct rstate *state = http_state_get(req); /* We want to move to read result after this. */ req->fsm_state = REQ_STATE_DB_WAIT; /* Fire off the query. */ - if (!kore_pgsql_query(&state->sql, "SELECT * FROM coders")) { + if (!kore_pgsql_query(&state->sql, + "SELECT * FROM coders, pg_sleep(5)")) { /* * Let the state machine continue immediately since we * have an error anyway. @@ -150,7 +156,7 @@ int request_db_wait(struct http_request *req) { - struct rstate *state = req->hdlr_extra; + struct rstate *state = http_state_get(req); kore_log(LOG_NOTICE, "request_db_wait: %d", state->sql.state); @@ -173,7 +179,7 @@ break; default: /* This MUST be present in order to advance the pgsql state */ - kore_pgsql_continue(req, &state->sql); + kore_pgsql_continue(&state->sql); break; } @@ -190,7 +196,7 @@ { char *name; int i, rows; - struct rstate *state = req->hdlr_extra; + struct rstate *state = http_state_get(req); /* We have sql data to read! */ rows = kore_pgsql_ntuples(&state->sql); @@ -200,7 +206,7 @@ } /* Continue processing our query results. */ - kore_pgsql_continue(req, &state->sql); + kore_pgsql_continue(&state->sql); /* Back to our DB waiting state. */ req->fsm_state = REQ_STATE_DB_WAIT; @@ -211,9 +217,11 @@ int request_error(struct http_request *req) { - struct rstate *state = req->hdlr_extra; + struct rstate *state = http_state_get(req); kore_pgsql_cleanup(&state->sql); + http_state_cleanup(req); + http_response(req, 500, NULL, 0); return (HTTP_STATE_COMPLETE); @@ -223,10 +231,14 @@ int request_done(struct http_request *req) { - struct rstate *state = req->hdlr_extra; + struct rstate *state = http_state_get(req); kore_pgsql_cleanup(&state->sql); + http_state_cleanup(req); + http_response(req, 200, NULL, 0); return (HTTP_STATE_COMPLETE); } + +#endif /* !KORE_NO_HTTP */ diff -Nru kore-2.0.0/examples/pgsql/src/pgsql_cb.c kore-3.3.1/examples/pgsql/src/pgsql_cb.c --- kore-2.0.0/examples/pgsql/src/pgsql_cb.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/pgsql/src/pgsql_cb.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2018 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * This is the same as pgsql.c except the query is fired off when + * a new connection is made to Kore on port 8889. + * + * Instead of binding an http_request to the pgsql data structure we + * use a callback function that is called for every state change. + * + * We pass the connection as an argument to this function. + */ + +#include +#include +#include + +void connection_del(struct connection *c); +void connection_new(struct connection *); + +void db_state_change(struct kore_pgsql *, void *); +void db_init(struct connection *, struct kore_pgsql *); +void db_results(struct kore_pgsql *, struct connection *); + +void +connection_new(struct connection *c) +{ + struct kore_pgsql *pgsql; + + c->disconnect = connection_del; + c->proto = CONN_PROTO_UNKNOWN; + c->state = CONN_STATE_ESTABLISHED; + + pgsql = kore_calloc(1, sizeof(*pgsql)); + + kore_pgsql_init(pgsql); + kore_pgsql_bind_callback(pgsql, db_state_change, c); + + c->hdlr_extra = pgsql; + printf("new connection %p\n", (void *)c); + + db_init(c, pgsql); +} + +void +db_init(struct connection *c, struct kore_pgsql *pgsql) +{ + if (!kore_pgsql_setup(pgsql, "db", KORE_PGSQL_ASYNC)) { + if (pgsql->state == KORE_PGSQL_STATE_INIT) { + printf("\twaiting for available pgsql connection\n"); + return; + } + + kore_pgsql_logerror(pgsql); + kore_connection_disconnect(c); + return; + } + + printf("\tgot pgsql connection\n"); + if (!kore_pgsql_query(pgsql, "SELECT * FROM coders, pg_sleep(5)")) { + kore_pgsql_logerror(pgsql); + kore_connection_disconnect(c); + return; + } + printf("\tquery fired off!\n"); +} + +void +connection_del(struct connection *c) +{ + printf("%p: disconnecting\n", (void *)c); + + if (c->hdlr_extra != NULL) + kore_pgsql_cleanup(c->hdlr_extra); + + kore_free(c->hdlr_extra); + c->hdlr_extra = NULL; +} + +void +db_state_change(struct kore_pgsql *pgsql, void *arg) +{ + struct connection *c = arg; + + printf("%p: state change on pgsql %d\n", arg, pgsql->state); + + switch (pgsql->state) { + case KORE_PGSQL_STATE_INIT: + db_init(c, pgsql); + break; + case KORE_PGSQL_STATE_WAIT: + break; + case KORE_PGSQL_STATE_COMPLETE: + kore_connection_disconnect(c); + break; + case KORE_PGSQL_STATE_ERROR: + kore_pgsql_logerror(pgsql); + kore_connection_disconnect(c); + break; + case KORE_PGSQL_STATE_RESULT: + db_results(pgsql, c); + break; + default: + kore_pgsql_continue(pgsql); + break; + } +} + +void +db_results(struct kore_pgsql *pgsql, struct connection *c) +{ + char *name; + int i, rows; + + rows = kore_pgsql_ntuples(pgsql); + for (i = 0; i < rows; i++) { + name = kore_pgsql_getvalue(pgsql, i, 0); + net_send_queue(c, name, strlen(name)); + } + + net_send_flush(c); + kore_pgsql_continue(pgsql); +} diff -Nru kore-2.0.0/examples/pgsql-sync/conf/build.conf kore-3.3.1/examples/pgsql-sync/conf/build.conf --- kore-2.0.0/examples/pgsql-sync/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pgsql-sync/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # pgsql-sync build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/pgsql-sync/conf/pgsql-sync.conf kore-3.3.1/examples/pgsql-sync/conf/pgsql-sync.conf --- kore-2.0.0/examples/pgsql-sync/conf/pgsql-sync.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pgsql-sync/conf/pgsql-sync.conf 2019-06-03 13:29:24.000000000 +0000 @@ -4,8 +4,8 @@ load ./pgsql-sync.so init tls_dhparam dh2048.pem -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / page } diff -Nru kore-2.0.0/examples/pgsql-sync/src/pgsql-sync.c kore-3.3.1/examples/pgsql-sync/src/pgsql-sync.c --- kore-2.0.0/examples/pgsql-sync/src/pgsql-sync.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pgsql-sync/src/pgsql-sync.c 2019-06-03 13:29:24.000000000 +0000 @@ -49,14 +49,15 @@ req->status = HTTP_STATUS_INTERNAL_ERROR; + kore_pgsql_init(&sql); + /* * Initialise our kore_pgsql data structure with the database name * we want to connect to (note that we registered this earlier with * kore_pgsql_register()). We also say we will perform a synchronous - * query (KORE_PGSQL_SYNC) and we do not need to pass our http_request - * so we pass NULL instead. + * query (KORE_PGSQL_SYNC). */ - if (!kore_pgsql_query_init(&sql, NULL, "db", KORE_PGSQL_SYNC)) { + if (!kore_pgsql_setup(&sql, "db", KORE_PGSQL_SYNC)) { kore_pgsql_logerror(&sql); goto out; } diff -Nru kore-2.0.0/examples/pipe_task/conf/build.conf kore-3.3.1/examples/pipe_task/conf/build.conf --- kore-2.0.0/examples/pipe_task/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pipe_task/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # pipe_task build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/pipe_task/conf/pipe_task.conf kore-3.3.1/examples/pipe_task/conf/pipe_task.conf --- kore-2.0.0/examples/pipe_task/conf/pipe_task.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pipe_task/conf/pipe_task.conf 2019-06-03 13:29:24.000000000 +0000 @@ -8,9 +8,9 @@ websocket_maxframe 65536 websocket_timeout 10000 -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / page static /connect page_ws_connect diff -Nru kore-2.0.0/examples/pipe_task/README.md kore-3.3.1/examples/pipe_task/README.md --- kore-2.0.0/examples/pipe_task/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pipe_task/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -8,7 +8,7 @@ Run: ``` - $ kore run + $ kodev run ``` Test: diff -Nru kore-2.0.0/examples/pipe_task/src/pipe_task.c kore-3.3.1/examples/pipe_task/src/pipe_task.c --- kore-2.0.0/examples/pipe_task/src/pipe_task.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/pipe_task/src/pipe_task.c 2019-06-03 13:29:24.000000000 +0000 @@ -46,13 +46,6 @@ int pipe_reader(struct kore_task *); void pipe_data_available(struct kore_task *); -/* Websocket callbacks. */ -struct kore_wscbs wscbs = { - websocket_connect, - websocket_message, - websocket_disconnect -}; - /* Our pipe reader. */ struct kore_task pipe_task; @@ -65,7 +58,7 @@ return (KORE_RESULT_ERROR); /* Only do this on a dedicated worker. */ - if (worker->id != 0) + if (worker->id != 1) return (KORE_RESULT_OK); /* Create our task. */ @@ -115,7 +108,8 @@ int page_ws_connect(struct http_request *req) { - kore_websocket_handshake(req, &wscbs); + kore_websocket_handshake(req, "websocket_connect", + "websocket_message", "websocket_disconnect"); return (KORE_RESULT_OK); } @@ -134,6 +128,8 @@ fd = -1; + kore_log(LOG_INFO, "pipe_reader starting"); + /* Just run forever. */ for (;;) { /* Attempt to open the pipe if needed. */ @@ -142,7 +138,7 @@ if ((fd = open("/tmp/pipe", O_RDONLY)) == -1) { kore_log(LOG_NOTICE, "failed to open pipe"); - sleep(10); + sleep(1); continue; } @@ -196,7 +192,8 @@ kore_log(LOG_WARNING, "truncated data from task"); /* Broadcast it to all connected websocket clients. */ - kore_log(LOG_NOTICE, "got %d bytes from task", len); + kore_log(LOG_NOTICE, "got %zu bytes from task", len); + kore_websocket_broadcast(NULL, WEBSOCKET_OP_TEXT, buf, len, WEBSOCKET_BROADCAST_GLOBAL); } diff -Nru kore-2.0.0/examples/python-async/conf/build.conf kore-3.3.1/examples/python-async/conf/build.conf --- kore-2.0.0/examples/python-async/conf/build.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,34 @@ +# python-async build config +# You can switch flavors using: kodev flavor [newflavor] + +# Set to yes if you wish to produce a single binary instead +# of a dynamic library. If you set this to yes you must also +# set kore_source together with kore_flavor. +single_binary=yes +kore_source=../../ +kore_flavor=PYTHON=1 CURL=1 NOTLS=1 + +# The flags below are shared between flavors +cflags=-Wall -Wmissing-declarations -Wshadow +cflags=-Wstrict-prototypes -Wmissing-prototypes +cflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +cxxflags=-Wall -Wmissing-declarations -Wshadow +cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +# Mime types for assets served via the builtin asset_serve_* +#mime_add=txt:text/plain; charset=utf-8 +#mime_add=png:image/png +#mime_add=html:text/html; charset=utf-8 + +dev { + # These flags are added to the shared ones when + # you build the "dev" flavor. + cflags=-g + cxxflags=-g +} + +#prod { +# You can specify additional flags here which are only +# included if you build with the "prod" flavor. +#} diff -Nru kore-2.0.0/examples/python-async/conf/python-async.conf kore-3.3.1/examples/python-async/conf/python-async.conf --- kore-2.0.0/examples/python-async/conf/python-async.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/conf/python-async.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,23 @@ +# python-async configuration + +bind 127.0.0.1 8888 + +python_path src + +python_import ./src/setup.py +python_import ./src/async_lock.py +python_import ./src/async_queue.py +python_import ./src/async_process.py +python_import ./src/async_socket.py +python_import ./src/async_http.py + +domain * { + static /queue async_queue + static /lock async_lock + static /proc async_proc + + static /socket async_socket + static /socket-test socket_test + + static /httpclient httpclient +} diff -Nru kore-2.0.0/examples/python-async/.gitignore kore-3.3.1/examples/python-async/.gitignore --- kore-2.0.0/examples/python-async/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/.gitignore 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,6 @@ +*.o +.flavor +.objs +python-async.so +assets.h +cert diff -Nru kore-2.0.0/examples/python-async/README.md kore-3.3.1/examples/python-async/README.md --- kore-2.0.0/examples/python-async/README.md 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,18 @@ +Kore python async/await examples. + +This example also shows off the asynchronous HTTP client support +and requires libcurl on your machine. + +Run: +``` + $ kodev run +``` + +Test: +``` + $ curl -k http://127.0.0.1:8888/queue + $ curl -k http://127.0.0.1:8888/lock + $ curl -k http://127.0.0.1:8888/proc + $ curl -k http://127.0.0.1:8888/socket + $ curl -k http://127.0.0.1:8888/httpclient +``` diff -Nru kore-2.0.0/examples/python-async/src/async_http.py kore-3.3.1/examples/python-async/src/async_http.py --- kore-2.0.0/examples/python-async/src/async_http.py 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/src/async_http.py 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,47 @@ +# +# Copyright (c) 2019 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +# +# Asynchronous HTTP client example. +# + +import kore + +# Handler called for /httpclient +async def httpclient(req): + # Create an httpclient. + client = kore.httpclient("https://kore.io") + + # Do a simple GET request. + status, body = await client.get() + print("status: %d, body: '%s'" % (status, body)) + + # Reuse and perform another GET request, returning headers too this time. + status, headers, body = await client.get(return_headers=True) + print("status: %d, headers: '%s'" % (status, headers)) + + # What happens if we post something? + status, body = await client.post(body=b"hello world") + print("status: %d, body: '%s'" % (status, body)) + + # Add some custom headers to our requests. + status, body = await client.get( + headers={ + "x-my-header": "async-http" + } + ) + + req.response(200, b'async done') diff -Nru kore-2.0.0/examples/python-async/src/async_lock.py kore-3.3.1/examples/python-async/src/async_lock.py --- kore-2.0.0/examples/python-async/src/async_lock.py 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/src/async_lock.py 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,38 @@ +# +# Copyright (c) 2018 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +# +# Locking example. +# +# The handler for /lock will grab the shared lock, suspend itself for +# 5 seconds before releasing the lock and responding. +# +# While the lock is held, other requests to /lock will block until it +# is released. + +import kore + +# The shared lock +lock = kore.lock() + +async def async_lock(req): + # A kore.lock should be used with the "async with" syntax. + async with lock: + # Suspend for 5 seconds. + await kore.suspend(5000) + + # Now respond. + req.response(200, b'') diff -Nru kore-2.0.0/examples/python-async/src/async_process.py kore-3.3.1/examples/python-async/src/async_process.py --- kore-2.0.0/examples/python-async/src/async_process.py 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/src/async_process.py 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,71 @@ +# +# Copyright (c) 2018 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +# +# Asynchronous process example. +# +# Wait for the result of an external process asynchronously. +# The handler will execute "/bin/ls" on the current directory and +# read the result. +# + +import kore +import json + +async def async_proc(req): + # + # You may specify a timeout when creating the kore.proc object. + # If the timeout is reached before the process exits kore will + # raise a TimeoutError exception. + # + # Ex: set timeout to 100ms: + # proc = kore.proc("/bin/ls -lR", 100) + + proc = kore.proc("/bin/ls -lR") + + try: + stdout = "" + + # Read until EOF (None is returned) + while True: + try: + # Read from the process, with an optional 1 second timeout. + # The recv() call will throw a TimeoutError exception if + # the timeout has elapsed before any data was read. + chunk = await proc.recv(1024, 1000) + if chunk is None: + break + except TimeoutError as e: + print("recv() timed out: %s" % e) + continue + stdout += chunk.decode() + + # Reap the process. + retcode = await proc.reap() + + # Respond with the return code + the result as JSON. + payload = { + "retcode": retcode, + "stdout": stdout + } + + data = json.dumps(payload, indent=4) + req.response(200, data.encode()) + except Exception as e: + # If an exception occurs we must kill the process first. + proc.kill() + errmsg = "Exception: %s" % e + req.response(500, errmsg.encode()) diff -Nru kore-2.0.0/examples/python-async/src/async_queue.py kore-3.3.1/examples/python-async/src/async_queue.py --- kore-2.0.0/examples/python-async/src/async_queue.py 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/src/async_queue.py 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,57 @@ +# +# Copyright (c) 2018 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +# +# Asynchronous queue example. +# + +import kore + +# The shared queue. +tq = kore.queue() + +# Entry point for our independent coroutine that is created when kore starts. +async def queue_helper(): + while True: + # Wait for a dictionary to arrive. + obj = await tq.pop() + kore.log(kore.LOG_INFO, "coro(): received %s" % obj) + + # Create a message to send back. + msg = "%d = %s" % (kore.time(), obj["msg"]) + + # Send it on the received queue. + obj["rq"].push(msg) + +async def async_queue(req): + # Create our own queue. + rq = kore.queue() + + # The dictionary we are going to send. + obj = { + # Receive queue object. + "rq": rq, + "msg": "hello" + } + + # Push it onto the tq queue now, which will wake up the other coroutine. + tq.push(obj) + + # Wait for a response. + response = await rq.pop() + + # Send the response to the client. + req.response(200, response.encode()) diff -Nru kore-2.0.0/examples/python-async/src/async_socket.py kore-3.3.1/examples/python-async/src/async_socket.py --- kore-2.0.0/examples/python-async/src/async_socket.py 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/src/async_socket.py 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,58 @@ +# +# Copyright (c) 2018 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +# +# Simple socket example. +# +# The handler will asynchronously connect to the kore app itself and +# send an GET request to /socket-test and read the response. + +import kore +import socket + +async def async_socket(req): + # Create the socket using Pythons built-in socket class. + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + # Set it to nonblocking. + sock.setblocking(False) + + # Create a kore.socket with kore.socket_wrap(). + conn = kore.socket_wrap(sock) + + # Asynchronously connect to 127.0.0.1 port 8888 + await conn.connect("127.0.0.1", 8888) + kore.log(kore.LOG_INFO, "connected!") + + # Now send the GET request + msg = "GET /socket-test HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n" + await conn.send(msg.encode()) + kore.log(kore.LOG_INFO, "request sent!") + + # Read the response. + data = await conn.recv(8192) + kore.log(kore.LOG_INFO, "got response!") + + # Respond with the response from /socket-test. + req.response(200, data) + + # Close the underlying socket, no need to close the wrapped kore.socket + sock.close() + +async def socket_test(req): + # Delay response a bit, just cause we can. + await kore.suspend(5000) + req.response(200, b'response from /socket-test') diff -Nru kore-2.0.0/examples/python-async/src/setup.py kore-3.3.1/examples/python-async/src/setup.py --- kore-2.0.0/examples/python-async/src/setup.py 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-async/src/setup.py 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,23 @@ +# +# Copyright (c) 2018 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +import kore + +from async_queue import queue_helper + +# Kore worker started, start the queue helper coroutine. +def kore_worker_configure(): + kore.task_create(queue_helper()) diff -Nru kore-2.0.0/examples/python-echo/conf/build.conf kore-3.3.1/examples/python-echo/conf/build.conf --- kore-2.0.0/examples/python-echo/conf/build.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-echo/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,34 @@ +# python-echo build config +# You can switch flavors using: kodev flavor [newflavor] + +# Set to yes if you wish to produce a single binary instead +# of a dynamic library. If you set this to yes you must also +# set kore_source together with kore_flavor. +single_binary=yes +kore_source=../../ +kore_flavor=NOTLS=1 PYTHON=1 + +# The flags below are shared between flavors +cflags=-Wall -Wmissing-declarations -Wshadow +cflags=-Wstrict-prototypes -Wmissing-prototypes +cflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +cxxflags=-Wall -Wmissing-declarations -Wshadow +cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +# Mime types for assets served via the builtin asset_serve_* +#mime_add=txt:text/plain; charset=utf-8 +#mime_add=png:image/png +#mime_add=html:text/html; charset=utf-8 + +dev { + # These flags are added to the shared ones when + # you build the "dev" flavor. + cflags=-g + cxxflags=-g +} + +#prod { +# You can specify additional flags here which are only +# included if you build with the "prod" flavor. +#} diff -Nru kore-2.0.0/examples/python-echo/conf/python-echo.conf kore-3.3.1/examples/python-echo/conf/python-echo.conf --- kore-2.0.0/examples/python-echo/conf/python-echo.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-echo/conf/python-echo.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,3 @@ +# python-echo configuration + +python_import src/echo.py diff -Nru kore-2.0.0/examples/python-echo/.gitignore kore-3.3.1/examples/python-echo/.gitignore --- kore-2.0.0/examples/python-echo/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-echo/.gitignore 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,6 @@ +*.o +.flavor +.objs +python-echo +assets.h +cert diff -Nru kore-2.0.0/examples/python-echo/src/echo.py kore-3.3.1/examples/python-echo/src/echo.py --- kore-2.0.0/examples/python-echo/src/echo.py 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-echo/src/echo.py 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,67 @@ +# +# Copyright (c) 2013-2018 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +import kore +import socket + +class EchoServer: + # Setup socket + wrap it inside of a kore socket so we can use it. + def __init__(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(False) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", 6969)) + sock.listen() + + self.conn = kore.socket_wrap(sock) + + # Wait for a new client to connect, then create a new task + # that calls handle_client with the ocnnected client as + # the argument. + async def run(self): + while True: + try: + client = await self.conn.accept() + kore.task_create(self.handle_client(client)) + client = None + except Exception as e: + kore.fatal("exception %s" % e) + + # Each client will run as this co-routine. + # In this case we pass a timeout of 1 second to the recv() call + # which will throw a TimeoutError exception in case the timeout + # is hit before data is read from the socket. + # + # This timeout argument is optional. If none is specified the call + # will wait until data becomes available. + async def handle_client(self, client): + while True: + try: + data = await client.recv(1024, 1000) + if data is None: + break + await client.send(data) + except TimeoutError as e: + print("timed out reading (%s)" % e) + except Exception as e: + print("client got exception %s" % e) + client.close() + +# Setup the server object. +server = EchoServer() + +# Create a task that will execute inside of Kore as a co-routine. +kore.task_create(server.run()) diff -Nru kore-2.0.0/examples/python-pgsql/conf/build.conf kore-3.3.1/examples/python-pgsql/conf/build.conf --- kore-2.0.0/examples/python-pgsql/conf/build.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-pgsql/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,35 @@ +# python-pgsql build config +# You can switch flavors using: kodev flavor [newflavor] + +# Set to yes if you wish to produce a single binary instead +# of a dynamic library. If you set this to yes you must also +# set kore_source together with kore_flavor and update ldflags +# to include the appropriate libraries you will be linking with. +#single_binary=no +#kore_source=/home/joris/src/kore +#kore_flavor= + +# The flags below are shared between flavors +cflags=-Wall -Wmissing-declarations -Wshadow +cflags=-Wstrict-prototypes -Wmissing-prototypes +cflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +cxxflags=-Wall -Wmissing-declarations -Wshadow +cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare + +# Mime types for assets served via the builtin asset_serve_* +#mime_add=txt:text/plain; charset=utf-8 +#mime_add=png:image/png +#mime_add=html:text/html; charset=utf-8 + +dev { + # These flags are added to the shared ones when + # you build the "dev" flavor. + cflags=-g + cxxflags=-g +} + +#prod { +# You can specify additional flags here which are only +# included if you build with the "prod" flavor. +#} diff -Nru kore-2.0.0/examples/python-pgsql/conf/python-pgsql.conf kore-3.3.1/examples/python-pgsql/conf/python-pgsql.conf --- kore-2.0.0/examples/python-pgsql/conf/python-pgsql.conf 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-pgsql/conf/python-pgsql.conf 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,16 @@ +# python-pgsql configuration + +bind 127.0.0.1 8888 + +python_import src/query.py + +tls_dhparam dh2048.pem + +domain * { + certfile cert/server.pem + certkey cert/key.pem + + static / query + static /hello hello + static /slow slow +} diff -Nru kore-2.0.0/examples/python-pgsql/.gitignore kore-3.3.1/examples/python-pgsql/.gitignore --- kore-2.0.0/examples/python-pgsql/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-pgsql/.gitignore 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,6 @@ +*.o +.flavor +.objs +python-pgsql.so +assets.h +cert diff -Nru kore-2.0.0/examples/python-pgsql/README.md kore-3.3.1/examples/python-pgsql/README.md --- kore-2.0.0/examples/python-pgsql/README.md 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-pgsql/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,15 @@ +Kore pgsql python module example. + +This application requires kore to be built with PYTHON=1 PGSQL=1. + +Run: +``` + $ kodev run +``` + +Test: +``` + $ curl -k https://127.0.0.1:8888 + $ curl -k https://127.0.0.1:8888/hello + $ curl -k https://127.0.0.1:8888/slow +``` diff -Nru kore-2.0.0/examples/python-pgsql/src/query.py kore-3.3.1/examples/python-pgsql/src/query.py --- kore-2.0.0/examples/python-pgsql/src/query.py 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/examples/python-pgsql/src/query.py 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,51 @@ +# +# Copyright (c) 2017-2018 Joris Vink +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# + +# Asynchronous postgresql queries with Python. + +import json +import kore + +# Register the path to our database when the worker starts. +def kore_worker_configure(): + kore.register_database("db", "host=/tmp dbname=kore") + +# A handler that returns 200 OK with hello as body. +def hello(req): + req.response(200, b'hello\n') + +# +# The query handler that fires of the query and returns a coroutine. +# +# Kore will resume this handler when the query returns a result or +# is succesfull. +# +# The req.pgsql() method can throw exceptions, most notably a +# GeneratorExit in case the client connection went away before +# the query was able to be completed. +# +# In this example we're not doing any exception handling. +# +async def query(req): + result = await req.pgsql("db", "SELECT * FROM coders") + req.response(200, json.dumps(result).encode("utf-8")) + +# +# A slow query that returns after 10 seconds. +# +async def slow(req): + result = await req.pgsql("db", "SELECT * FROM pg_sleep(10)") + req.response(200, json.dumps(result).encode("utf-8")) diff -Nru kore-2.0.0/examples/sse/conf/build.conf kore-3.3.1/examples/sse/conf/build.conf --- kore-2.0.0/examples/sse/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/sse/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # sse build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/sse/conf/sse.conf kore-3.3.1/examples/sse/conf/sse.conf --- kore-2.0.0/examples/sse/conf/sse.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/sse/conf/sse.conf 2019-06-03 13:29:24.000000000 +0000 @@ -6,9 +6,9 @@ http_keepalive_time 600 -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / page static /subscribe subscribe diff -Nru kore-2.0.0/examples/sse/README.md kore-3.3.1/examples/sse/README.md --- kore-2.0.0/examples/sse/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/sse/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -2,7 +2,7 @@ Run: ``` - $ kore run + $ kodev run ``` Test (run different times to see the events broadcast): diff -Nru kore-2.0.0/examples/sse/src/sse.c kore-3.3.1/examples/sse/src/sse.c --- kore-2.0.0/examples/sse/src/sse.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/sse/src/sse.c 2019-06-03 13:29:24.000000000 +0000 @@ -91,9 +91,6 @@ /* Set a disconnection method so we know when this client goes away. */ req->owner->disconnect = sse_disconnect; - /* We do not expect any more data to arrive. */ - req->owner->flags |= CONN_READ_BLOCK; - /* Allocate a state to be carried by our connection. */ state = kore_malloc(sizeof(*state)); req->owner->hdlr_extra = state; @@ -169,7 +166,7 @@ int check_header(struct http_request *req, const char *name, const char *value) { - char *hdr; + const char *hdr; if (!http_request_header(req, name, &hdr)) { http_response(req, 400, NULL, 0); diff -Nru kore-2.0.0/examples/tasks/conf/build.conf kore-3.3.1/examples/tasks/conf/build.conf --- kore-2.0.0/examples/tasks/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/tasks/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,16 +1,18 @@ # tasks build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow cflags=-Wstrict-prototypes -Wmissing-prototypes cflags=-Wpointer-arith -Wcast-qual -Wsign-compare +cflags=-I/usr/local/include + dev { # These cflags are added to the shared ones when # you build the "dev" flavor. cflags=-g - ldflags=-lcurl + ldflags=-L/usr/local/lib -lcurl } #prod { diff -Nru kore-2.0.0/examples/tasks/conf/tasks.conf kore-3.3.1/examples/tasks/conf/tasks.conf --- kore-2.0.0/examples/tasks/conf/tasks.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/tasks/conf/tasks.conf 2019-06-03 13:29:24.000000000 +0000 @@ -11,15 +11,15 @@ validator v_user regex ^[a-z]*$ -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem accesslog kore_access.log static / page_handler static /post_back post_back - params get / { + params qs:get / { validate user v_user } diff -Nru kore-2.0.0/examples/tasks/README.md kore-3.3.1/examples/tasks/README.md --- kore-2.0.0/examples/tasks/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/tasks/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -6,12 +6,12 @@ Build: ``` - $ kore build + $ kodev build ``` Run: ``` - $ kore run + $ kodev run ``` Test: diff -Nru kore-2.0.0/examples/tasks/src/tasks.c kore-3.3.1/examples/tasks/src/tasks.c --- kore-2.0.0/examples/tasks/src/tasks.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/tasks/src/tasks.c 2019-06-03 13:29:24.000000000 +0000 @@ -172,7 +172,7 @@ run_curl(struct kore_task *t) { struct kore_buf *b; - u_int32_t len; + size_t len; CURLcode res; u_int8_t *data; CURL *curl; @@ -203,7 +203,11 @@ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, fields); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb); +#if !defined(KORE_NO_TLS) curl_easy_setopt(curl, CURLOPT_URL, "https://127.0.0.1:8888/post_back"); +#else + curl_easy_setopt(curl, CURLOPT_URL, "http://127.0.0.1:8888/post_back"); +#endif res = curl_easy_perform(curl); if (res != CURLE_OK) { diff -Nru kore-2.0.0/examples/tls-proxy/conf/build.conf kore-3.3.1/examples/tls-proxy/conf/build.conf --- kore-2.0.0/examples/tls-proxy/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/tls-proxy/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # tls-proxy build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/tls-proxy/conf/tls-proxy.conf kore-3.3.1/examples/tls-proxy/conf/tls-proxy.conf --- kore-2.0.0/examples/tls-proxy/conf/tls-proxy.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/tls-proxy/conf/tls-proxy.conf 2019-06-03 13:29:24.000000000 +0000 @@ -11,7 +11,7 @@ bind 127.0.0.1 8888 client_setup # Setup domain for TLS usage. -domain localhost { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem } diff -Nru kore-2.0.0/examples/tls-proxy/README.md kore-3.3.1/examples/tls-proxy/README.md --- kore-2.0.0/examples/tls-proxy/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/tls-proxy/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -7,7 +7,7 @@ Run: ``` - $ kore run + $ kodev run ``` Test: diff -Nru kore-2.0.0/examples/tls-proxy/src/proxy.c kore-3.3.1/examples/tls-proxy/src/proxy.c --- kore-2.0.0/examples/tls-proxy/src/proxy.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/tls-proxy/src/proxy.c 2019-06-03 13:29:24.000000000 +0000 @@ -60,19 +60,19 @@ client_setup(struct connection *c) { int i, fd; + const char *name; struct connection *backend; /* Paranoia. */ - if (c->ssl->session == NULL || - c->ssl->session->tlsext_hostname == NULL) { + name = SSL_get_servername(c->ssl, TLSEXT_NAMETYPE_host_name); + if (name == NULL) { kore_connection_disconnect(c); return; } /* Figure out what backend to use. */ for (i = 0; backends[i].name != NULL; i++) { - if (!strcasecmp(backends[i].name, - c->ssl->session->tlsext_hostname)) + if (!strcasecmp(backends[i].name, name)) break; } @@ -100,7 +100,7 @@ backend = kore_connection_new(NULL); /* Prepare our connection. */ - backend->addrtype = AF_INET; + backend->family = AF_INET; backend->addr.ipv4.sin_family = AF_INET; backend->addr.ipv4.sin_port = htons(backends[i].port); backend->addr.ipv4.sin_addr.s_addr = inet_addr(backends[i].ip); @@ -150,7 +150,7 @@ TAILQ_INSERT_TAIL(&connections, backend, list); /* Kick off connecting. */ - backend->flags |= CONN_WRITE_POSSIBLE; + backend->evt.flags |= KORE_EVENT_WRITE; backend->handle(backend); } @@ -170,7 +170,7 @@ struct connection *src; /* We will get a write notification when we can progress. */ - if (!(c->flags & CONN_WRITE_POSSIBLE)) + if (!(c->evt.flags & KORE_EVENT_WRITE)) return (KORE_RESULT_OK); kore_connection_stop_idletimer(c); @@ -190,7 +190,7 @@ /* Clean the write flag, we'll be called later. */ if (errno != EISCONN) { - c->flags &= ~CONN_WRITE_POSSIBLE; + c->evt.flags &= ~KORE_EVENT_WRITE; kore_connection_start_idletimer(c); return (KORE_RESULT_OK); } diff -Nru kore-2.0.0/examples/upload/conf/build.conf kore-3.3.1/examples/upload/conf/build.conf --- kore-2.0.0/examples/upload/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/upload/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # upload build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/upload/conf/upload.conf kore-3.3.1/examples/upload/conf/upload.conf --- kore-2.0.0/examples/upload/conf/upload.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/upload/conf/upload.conf 2019-06-03 13:29:24.000000000 +0000 @@ -8,9 +8,9 @@ http_body_max 1024000000 http_body_disk_offload 4096 -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / page } diff -Nru kore-2.0.0/examples/upload/src/upload.c kore-3.3.1/examples/upload/src/upload.c --- kore-2.0.0/examples/upload/src/upload.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/upload/src/upload.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Joris Vink + * Copyright (c) 2016-2018 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff -Nru kore-2.0.0/examples/video_stream/conf/build.conf kore-3.3.1/examples/video_stream/conf/build.conf --- kore-2.0.0/examples/video_stream/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/video_stream/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,11 +1,13 @@ # video_stream build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow cflags=-Wstrict-prototypes -Wmissing-prototypes cflags=-Wpointer-arith -Wcast-qual -Wsign-compare +mime_add=html:text/html; charset=utf-8 + dev { # These cflags are added to the shared ones when # you build the "dev" flavor. diff -Nru kore-2.0.0/examples/video_stream/conf/video_stream.conf kore-3.3.1/examples/video_stream/conf/video_stream.conf --- kore-2.0.0/examples/video_stream/conf/video_stream.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/video_stream/conf/video_stream.conf 2019-06-03 13:29:24.000000000 +0000 @@ -7,11 +7,11 @@ http_keepalive_time 600 -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem accesslog access.log - static / serve_page + static / asset_serve_video_html dynamic ^/[a-z]*.[a-z0-9]{3}$ video_stream } diff -Nru kore-2.0.0/examples/video_stream/README.md kore-3.3.1/examples/video_stream/README.md --- kore-2.0.0/examples/video_stream/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/video_stream/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -9,12 +9,12 @@ If you did not save your video as videos/video.ogg make sure you update the assets/video.html file to point to the right video. - When done, run a kore build. + When done, run a kodev build. ``` Run: ``` - kore run + kodev run ``` Visit the URI and you should see a video stream. diff -Nru kore-2.0.0/examples/video_stream/src/stream.c kore-3.3.1/examples/video_stream/src/stream.c --- kore-2.0.0/examples/video_stream/src/stream.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/video_stream/src/stream.c 2019-06-03 13:29:24.000000000 +0000 @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -38,7 +39,6 @@ }; int init(int); -int serve_page(struct http_request *); int video_stream(struct http_request *); static void video_unmap(struct video *); @@ -61,22 +61,13 @@ } int -serve_page(struct http_request *req) -{ - http_response_header(req, "content-type", "text/html"); - http_response_stream(req, 200, asset_video_html, - asset_len_video_html, NULL, NULL); - - return (KORE_RESULT_OK); -} - -int video_stream(struct http_request *req) { struct video *v; + const char *header; off_t start, end; int n, err, status; - char *header, *bytes, *range[3], rb[128], *ext, ctype[32]; + char *bytes, *range[3], rb[128], *ext, ctype[32]; if (!video_open(req, &v)) return (KORE_RESULT_OK); @@ -93,8 +84,8 @@ return (KORE_RESULT_OK); } - kore_log(LOG_NOTICE, "%p: opened %s (%s) for streaming (%ld ref:%d)", - req->owner, v->path, ctype, v->size, v->ref); + kore_log(LOG_NOTICE, "%p: opened %s (%s) for streaming (%jd ref:%d)", + (void *)req->owner, v->path, ctype, (intmax_t)v->size, v->ref); if (http_request_header(req, "range", &header)) { if ((bytes = strchr(header, '=')) == NULL) { @@ -104,7 +95,7 @@ } bytes++; - n = kore_split_string(bytes, "-", range, 2); + n = kore_split_string(bytes, "-", range, 3); if (n == 0) { v->ref--; http_response(req, 416, NULL, 0); @@ -148,8 +139,9 @@ return (KORE_RESULT_OK); } - kore_log(LOG_NOTICE, "%p: %s sending: %ld-%ld/%ld", - req->owner, v->path, start, end - 1, v->size); + kore_log(LOG_NOTICE, "%p: %s sending: %jd-%jd/%jd", + (void *)req->owner, v->path, (intmax_t)start, + (intmax_t)end - 1, (intmax_t)v->size); http_response_header(req, "content-range", rb); } else { start = 0; @@ -261,8 +253,8 @@ struct video *v = nb->extra; v->ref--; - kore_log(LOG_NOTICE, "%p: video stream %s done (%d/%d ref:%d)", - nb->owner, v->path, nb->s_off, nb->b_len, v->ref); + kore_log(LOG_NOTICE, "%p: video stream %s done (%zu/%zu ref:%d)", + (void *)nb->owner, v->path, nb->s_off, nb->b_len, v->ref); if (v->ref == 0) video_unmap(v); diff -Nru kore-2.0.0/examples/websocket/conf/build.conf kore-3.3.1/examples/websocket/conf/build.conf --- kore-2.0.0/examples/websocket/conf/build.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/websocket/conf/build.conf 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ # websocket build config -# You can switch flavors using: kore flavor [newflavor] +# You can switch flavors using: kodev flavor [newflavor] # The cflags below are shared between flavors cflags=-Wall -Wmissing-declarations -Wshadow diff -Nru kore-2.0.0/examples/websocket/conf/websocket.conf kore-3.3.1/examples/websocket/conf/websocket.conf --- kore-2.0.0/examples/websocket/conf/websocket.conf 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/websocket/conf/websocket.conf 2019-06-03 13:29:24.000000000 +0000 @@ -12,9 +12,9 @@ websocket_maxframe 65536 websocket_timeout 20 -domain 127.0.0.1 { - certfile cert/server.crt - certkey cert/server.key +domain * { + certfile cert/server.pem + certkey cert/key.pem static / page static /connect page_ws_connect diff -Nru kore-2.0.0/examples/websocket/README.md kore-3.3.1/examples/websocket/README.md --- kore-2.0.0/examples/websocket/README.md 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/websocket/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -2,7 +2,7 @@ Run: ``` - # kore run + # kodev run ``` Test: diff -Nru kore-2.0.0/examples/websocket/src/websocket.c kore-3.3.1/examples/websocket/src/websocket.c --- kore-2.0.0/examples/websocket/src/websocket.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/examples/websocket/src/websocket.c 2019-06-03 13:29:24.000000000 +0000 @@ -27,13 +27,6 @@ void websocket_message(struct connection *, u_int8_t, void *, size_t); -/* Websocket callbacks. */ -struct kore_wscbs wscbs = { - websocket_connect, - websocket_message, - websocket_disconnect -}; - /* Called whenever we get a new websocket connection. */ void websocket_connect(struct connection *c) @@ -66,7 +59,8 @@ page_ws_connect(struct http_request *req) { /* Perform the websocket handshake, passing our callbacks. */ - kore_websocket_handshake(req, &wscbs); + kore_websocket_handshake(req, "websocket_connect", + "websocket_message", "websocket_disconnect"); return (KORE_RESULT_OK); } diff -Nru kore-2.0.0/.gitignore kore-3.3.1/.gitignore --- kore-2.0.0/.gitignore 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/.gitignore 2019-06-03 13:29:24.000000000 +0000 @@ -7,3 +7,6 @@ cert obj .lvimrc +kodev/kodev +kore.features +src/version.c diff -Nru kore-2.0.0/include/kore/curl.h kore-3.3.1/include/kore/curl.h --- kore-2.0.0/include/kore/curl.h 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/include/kore/curl.h 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __H_CURL_H +#define __H_CURL_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include + +#include "http.h" + +#define KORE_CURL_TIMEOUT 60 +#define KORE_CURL_RECV_MAX (1024 * 1024 * 2) + +#define KORE_CURL_FLAG_HTTP_PARSED_HEADERS 0x0001 +#define KORE_CURL_FLAG_BOUND 0x0002 + +#define KORE_CURL_TYPE_CUSTOM 1 +#define KORE_CURL_TYPE_HTTP_CLIENT 2 + +struct kore_curl { + int type; + int flags; + CURLcode result; + + char *url; + CURL *handle; + struct kore_buf *response; + + struct http_request *req; + void *arg; + void (*cb)(struct kore_curl *, void *); + + char errbuf[CURL_ERROR_SIZE]; + + /* For the simplified HTTP api. */ + struct { + long status; + struct curl_slist *hdrlist; + + struct kore_buf *tosend; + struct kore_buf *headers; + + TAILQ_HEAD(, http_header) resp_hdrs; + } http; + + LIST_ENTRY(kore_curl) list; +}; + +extern u_int16_t kore_curl_timeout; +extern u_int64_t kore_curl_recv_max; + +void kore_curl_sysinit(void); +void kore_curl_run(struct kore_curl *); +void kore_curl_cleanup(struct kore_curl *); +int kore_curl_success(struct kore_curl *); +void kore_curl_logerror(struct kore_curl *); +int kore_curl_init(struct kore_curl *, const char *); + +size_t kore_curl_tobuf(char *, size_t, size_t, void *); +size_t kore_curl_frombuf(char *, size_t, size_t, void *); + +void kore_curl_http_parse_headers(struct kore_curl *); +void kore_curl_http_set_header(struct kore_curl *, const char *, + const char *); +int kore_curl_http_get_header(struct kore_curl *, const char *, + const char **); +void kore_curl_http_setup(struct kore_curl *, int, const void *, size_t); + +char *kore_curl_response_as_string(struct kore_curl *); +void kore_curl_response_as_bytes(struct kore_curl *, + const u_int8_t **, size_t *); + +void kore_curl_bind_request(struct kore_curl *, struct http_request *); +void kore_curl_bind_callback(struct kore_curl *, + void (*cb)(struct kore_curl *, void *), void *); + +const char *kore_curl_strerror(struct kore_curl *); + +#if defined(__cplusplus) +} +#endif + +#endif diff -Nru kore-2.0.0/include/kore/http.h kore-3.3.1/include/kore/http.h --- kore-2.0.0/include/kore/http.h 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/include/kore/http.h 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2013 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#if !defined(KORE_NO_HTTP) + +#ifndef __H_HTTP_H +#define __H_HTTP_H + +#include +#include + +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +/* Keep the http_populate_get symbol around. */ +#define http_populate_get http_populate_qs + +#define HTTP_KEEPALIVE_TIME 20 +#define HTTP_HSTS_ENABLE 31536000 +#define HTTP_HEADER_MAX_LEN 4096 +#define HTTP_BODY_MAX_LEN 1024000 +#define HTTP_URI_LEN 2000 +#define HTTP_USERAGENT_LEN 256 +#define HTTP_REFERER_LEN 256 +#define HTTP_REQ_HEADER_MAX 25 +#define HTTP_MAX_QUERY_ARGS 20 +#define HTTP_MAX_COOKIES 10 +#define HTTP_MAX_COOKIENAME 255 +#define HTTP_HEADER_BUFSIZE 1024 +#define HTTP_COOKIE_BUFSIZE 1024 +#define HTTP_DATE_MAXSIZE 255 +#define HTTP_REQUEST_LIMIT 1000 +#define HTTP_REQUEST_MS 10 +#define HTTP_BODY_DISK_PATH "tmp_files" +#define HTTP_BODY_DISK_OFFLOAD 0 +#define HTTP_BODY_PATH_MAX 256 +#define HTTP_BOUNDARY_MAX 80 +#define HTTP_HEADER_TIMEOUT 10 +#define HTTP_BODY_TIMEOUT 60 + +#define HTTP_ARG_TYPE_RAW 0 +#define HTTP_ARG_TYPE_BYTE 1 +#define HTTP_ARG_TYPE_INT16 2 +#define HTTP_ARG_TYPE_UINT16 3 +#define HTTP_ARG_TYPE_INT32 4 +#define HTTP_ARG_TYPE_UINT32 5 +#define HTTP_ARG_TYPE_STRING 6 +#define HTTP_ARG_TYPE_INT64 7 +#define HTTP_ARG_TYPE_UINT64 8 +#define HTTP_ARG_TYPE_FLOAT 9 +#define HTTP_ARG_TYPE_DOUBLE 10 + +#define HTTP_STATE_ERROR 0 +#define HTTP_STATE_CONTINUE 1 +#define HTTP_STATE_COMPLETE 2 +#define HTTP_STATE_RETRY 3 + +struct http_runlock_queue { + struct http_request *req; + LIST_ENTRY(http_runlock_queue) list; +}; + +struct http_runlock { + struct http_request *owner; + LIST_HEAD(, http_runlock_queue) queue; +}; + +struct http_header { + char *header; + char *value; + + TAILQ_ENTRY(http_header) list; +}; + +#define HTTP_COOKIE_HTTPONLY 0x0001 +#define HTTP_COOKIE_SECURE 0x0002 + +struct http_cookie { + char *name; + char *value; + char *path; + char *domain; + u_int32_t maxage; + time_t expires; + u_int16_t flags; + + TAILQ_ENTRY(http_cookie) list; +}; + +struct http_arg { + char *name; + char *s_value; + + TAILQ_ENTRY(http_arg) list; +}; + +#define COPY_ARG_TYPE(v, t) \ + do { \ + *(t *)nout = v; \ + } while (0) + +#define COPY_ARG_INT64(type, sign) \ + do { \ + int err; \ + type nval; \ + nval = (type)kore_strtonum64(q->s_value, sign, &err); \ + if (err != KORE_RESULT_OK) \ + return (KORE_RESULT_ERROR); \ + COPY_ARG_TYPE(nval, type); \ + } while (0) + +#define COPY_ARG_DOUBLE(min, max, type) \ + do { \ + int err; \ + type nval; \ + nval = kore_strtodouble(q->s_value, min, max, &err); \ + if (err != KORE_RESULT_OK) \ + return (KORE_RESULT_ERROR); \ + COPY_ARG_TYPE(nval, type); \ + } while (0) + +#define COPY_ARG_INT(min, max, type) \ + do { \ + int err; \ + int64_t nval; \ + nval = kore_strtonum(q->s_value, 10, min, max, &err); \ + if (err != KORE_RESULT_OK) \ + return (KORE_RESULT_ERROR); \ + COPY_ARG_TYPE(nval, type); \ + } while (0) + +#define COPY_AS_INTTYPE_64(type, sign) \ + do { \ + if (nout == NULL) \ + return (KORE_RESULT_ERROR); \ + COPY_ARG_INT64(type, sign); \ + } while (0) + +#define COPY_AS_INTTYPE(min, max, type) \ + do { \ + if (nout == NULL) \ + return (KORE_RESULT_ERROR); \ + COPY_ARG_INT(min, max, type); \ + } while (0) + +#define http_argument_type(r, n, so, no, t) \ + http_argument_get(r, n, so, no, t) + +#define http_argument_get_string(r, n, o) \ + http_argument_type(r, n, (void **)o, NULL, HTTP_ARG_TYPE_STRING) + +#define http_argument_get_byte(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_BYTE) + +#define http_argument_get_uint16(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT16) + +#define http_argument_get_int16(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT16) + +#define http_argument_get_uint32(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT32) + +#define http_argument_get_int32(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT32) + +#define http_argument_get_uint64(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT64) + +#define http_argument_get_int64(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT64) + +#define http_argument_get_float(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_FLOAT) + +#define http_argument_get_double(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_DOUBLE) + +struct http_file { + char *name; + char *filename; + size_t position; + size_t offset; + size_t length; + struct http_request *req; + TAILQ_ENTRY(http_file) list; +}; + +#define HTTP_METHOD_GET 0x0001 +#define HTTP_METHOD_POST 0x0002 +#define HTTP_METHOD_PUT 0x0004 +#define HTTP_METHOD_DELETE 0x0010 +#define HTTP_METHOD_HEAD 0x0020 +#define HTTP_METHOD_OPTIONS 0x0040 +#define HTTP_METHOD_PATCH 0x0080 + +#define HTTP_METHOD_ALL (HTTP_METHOD_GET | HTTP_METHOD_POST | \ + HTTP_METHOD_PUT | HTTP_METHOD_DELETE | HTTP_METHOD_HEAD | \ + HTTP_METHOD_OPTIONS | HTTP_METHOD_PATCH) + +#define HTTP_REQUEST_COMPLETE 0x0001 +#define HTTP_REQUEST_DELETE 0x0002 +#define HTTP_REQUEST_SLEEPING 0x0004 +#define HTTP_REQUEST_EXPECT_BODY 0x0020 +#define HTTP_REQUEST_RETAIN_EXTRA 0x0040 +#define HTTP_REQUEST_NO_CONTENT_LENGTH 0x0080 +#define HTTP_REQUEST_AUTHED 0x0100 + +#define HTTP_VERSION_1_1 0x1000 +#define HTTP_VERSION_1_0 0x2000 + +#define HTTP_VALIDATOR_IS_REQUEST 0x8000 + +#define HTTP_BODY_DIGEST_LEN 32 +#define HTTP_BODY_DIGEST_STRLEN ((HTTP_BODY_DIGEST_LEN * 2) + 1) + +struct kore_task; +struct http_client; + +struct http_request { + u_int8_t method; + u_int8_t fsm_state; + u_int16_t flags; + u_int16_t status; + u_int64_t ms; + u_int64_t start; + u_int64_t end; + u_int64_t total; + const char *path; + const char *host; + const char *agent; + const char *referer; + struct connection *owner; + SHA256_CTX hashctx; + u_int8_t *headers; + struct kore_buf *http_body; + int http_body_fd; + char *http_body_path; + size_t http_body_length; + size_t http_body_offset; + size_t content_length; + void *hdlr_extra; + size_t state_len; + char *query_string; + struct kore_module_handle *hdlr; + struct http_runlock_queue *runlock; + void (*onfree)(struct http_request *); + +#if defined(KORE_USE_PYTHON) + void *py_coro; +#endif + + u_int8_t http_body_digest[HTTP_BODY_DIGEST_LEN]; + +#if defined(KORE_USE_CURL) + LIST_HEAD(, kore_curl) chandles; +#endif + +#if defined(KORE_USE_TASKS) + LIST_HEAD(, kore_task) tasks; +#endif + +#if defined(KORE_USE_PGSQL) + LIST_HEAD(, kore_pgsql) pgsqls; +#endif + + TAILQ_HEAD(, http_cookie) req_cookies; + TAILQ_HEAD(, http_cookie) resp_cookies; + TAILQ_HEAD(, http_header) req_headers; + TAILQ_HEAD(, http_header) resp_headers; + TAILQ_HEAD(, http_arg) arguments; + TAILQ_HEAD(, http_file) files; + TAILQ_ENTRY(http_request) list; + TAILQ_ENTRY(http_request) olist; +}; + +#define KORE_HTTP_STATE(f) { #f, f } + +struct http_state { + const char *name; + int (*cb)(struct http_request *); +}; + +struct http_media_type { + char *ext; + char *type; + LIST_ENTRY(http_media_type) list; +}; + +extern size_t http_body_max; +extern u_int16_t http_body_timeout; +extern u_int16_t http_header_max; +extern u_int16_t http_header_timeout; +extern u_int32_t http_request_ms; +extern u_int64_t http_hsts_enable; +extern u_int16_t http_keepalive_time; +extern u_int32_t http_request_limit; +extern u_int32_t http_request_count; +extern u_int64_t http_body_disk_offload; +extern char *http_body_disk_path; +extern struct kore_pool http_header_pool; + +void kore_accesslog(struct http_request *); + +void http_init(void); +void http_parent_init(void); +void http_cleanup(void); +void http_server_version(const char *); +void http_process(void); +const char *http_status_text(int); +const char *http_method_text(int); +time_t http_date_to_time(char *); +char *http_validate_header(char *); +void http_start_recv(struct connection *); +void http_request_free(struct http_request *); +void http_request_sleep(struct http_request *); +void http_request_wakeup(struct http_request *); +void http_process_request(struct http_request *); +int http_body_rewind(struct http_request *); +int http_media_register(const char *, const char *); +int http_check_timeout(struct connection *, u_int64_t); +ssize_t http_body_read(struct http_request *, void *, size_t); +int http_body_digest(struct http_request *, char *, size_t); +void http_response(struct http_request *, int, const void *, size_t); +void http_response_fileref(struct http_request *, int, + struct kore_fileref *); +void http_serveable(struct http_request *, const void *, + size_t, const char *, const char *); +void http_response_stream(struct http_request *, int, void *, + size_t, int (*cb)(struct netbuf *), void *); +int http_request_header(struct http_request *, + const char *, const char **); +void http_response_header(struct http_request *, + const char *, const char *); +int http_state_run(struct http_state *, u_int8_t, + struct http_request *); +int http_request_cookie(struct http_request *, + const char *, char **); +void http_response_cookie(struct http_request *, const char *, + const char *, const char *, time_t, u_int32_t, + struct http_cookie **); + +void http_runlock_init(struct http_runlock *); +void http_runlock_release(struct http_runlock *, + struct http_request *); +int http_runlock_acquire(struct http_runlock *, + struct http_request *); + +const char *http_media_type(const char *); +void *http_state_get(struct http_request *); +int http_state_exists(struct http_request *); +void http_state_cleanup(struct http_request *); +void *http_state_create(struct http_request *, size_t, + void (*onfree)(struct http_request *)); + +int http_argument_urldecode(char *); +int http_header_recv(struct netbuf *); +void http_populate_qs(struct http_request *); +void http_populate_post(struct http_request *); +void http_populate_multipart_form(struct http_request *); +void http_populate_cookies(struct http_request *); +int http_argument_get(struct http_request *, + const char *, void **, void *, int); + +void http_file_rewind(struct http_file *); +ssize_t http_file_read(struct http_file *, void *, size_t); +struct http_file *http_file_lookup(struct http_request *, const char *); + +enum http_status_code { + HTTP_STATUS_CONTINUE = 100, + HTTP_STATUS_SWITCHING_PROTOCOLS = 101, + HTTP_STATUS_OK = 200, + HTTP_STATUS_CREATED = 201, + HTTP_STATUS_ACCEPTED = 202, + HTTP_STATUS_NON_AUTHORITATIVE = 203, + HTTP_STATUS_NO_CONTENT = 204, + HTTP_STATUS_RESET_CONTENT = 205, + HTTP_STATUS_PARTIAL_CONTENT = 206, + HTTP_STATUS_MULTIPLE_CHOICES = 300, + HTTP_STATUS_MOVED_PERMANENTLY = 301, + HTTP_STATUS_FOUND = 302, + HTTP_STATUS_SEE_OTHER = 303, + HTTP_STATUS_NOT_MODIFIED = 304, + HTTP_STATUS_USE_PROXY = 305, + HTTP_STATUS_TEMPORARY_REDIRECT = 307, + HTTP_STATUS_BAD_REQUEST = 400, + HTTP_STATUS_UNAUTHORIZED = 401, + HTTP_STATUS_PAYMENT_REQUIRED = 402, + HTTP_STATUS_FORBIDDEN = 403, + HTTP_STATUS_NOT_FOUND = 404, + HTTP_STATUS_METHOD_NOT_ALLOWED = 405, + HTTP_STATUS_NOT_ACCEPTABLE = 406, + HTTP_STATUS_PROXY_AUTH_REQUIRED = 407, + HTTP_STATUS_REQUEST_TIMEOUT = 408, + HTTP_STATUS_CONFLICT = 409, + HTTP_STATUS_GONE = 410, + HTTP_STATUS_LENGTH_REQUIRED = 411, + HTTP_STATUS_PRECONDITION_FAILED = 412, + HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE = 413, + HTTP_STATUS_REQUEST_URI_TOO_LARGE = 414, + HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE = 415, + HTTP_STATUS_REQUEST_RANGE_INVALID = 416, + HTTP_STATUS_EXPECTATION_FAILED = 417, + HTTP_STATUS_INTERNAL_ERROR = 500, + HTTP_STATUS_NOT_IMPLEMENTED = 501, + HTTP_STATUS_BAD_GATEWAY = 502, + HTTP_STATUS_SERVICE_UNAVAILABLE = 503, + HTTP_STATUS_GATEWAY_TIMEOUT = 504, + HTTP_STATUS_BAD_VERSION = 505 +}; +#if defined(__cplusplus) +} +#endif +#endif /* !__H_HTTP_H */ + +#endif /* ! KORE_NO_HTTP */ diff -Nru kore-2.0.0/include/kore/jsonrpc.h kore-3.3.1/include/kore/jsonrpc.h --- kore-2.0.0/include/kore/jsonrpc.h 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/include/kore/jsonrpc.h 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2016 Raphaël Monrouzeau + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#if !defined(KORE_NO_HTTP) + +#ifndef __H_JSONRPC_H +#define __H_JSONRPC_H + +#if defined(__cplusplus) +extern "C" { +#endif + +/* JSON RPC request handling log entry. */ +struct jsonrpc_log +{ + char *msg; + struct jsonrpc_log *next, *prev; + int lvl; +}; + +/* JSON RPC request. */ +struct jsonrpc_request +{ + struct jsonrpc_log log; + struct kore_buf buf; + struct http_request *http; + yajl_gen gen; + yajl_val json; + yajl_val id; + char *method; + yajl_val params; + unsigned int flags; + int log_levels; +}; + +#define YAJL_GEN_CONST_STRING(CTX, STR) \ + yajl_gen_string((CTX), (unsigned char *)(STR), sizeof (STR) - 1) + +#define YAJL_GEN_CONST_NUMBER(CTX, STR) \ + yajl_gen_number((CTX), (unsigned char *)(STR), sizeof (STR) - 1) + +#define YAJL_GEN_KO(OPERATION) \ + ((OPERATION) != yajl_gen_status_ok) + +enum jsonrpc_error_code +{ +#define JSONRPC_PARSE_ERROR_MSG "Parse error" + JSONRPC_PARSE_ERROR = -32700, +#define JSONRPC_INVALID_REQUEST_MSG "Invalid Request" + JSONRPC_INVALID_REQUEST = -32600, +#define JSONRPC_METHOD_NOT_FOUND_MSG "Method not found" + JSONRPC_METHOD_NOT_FOUND = -32601, +#define JSONRPC_INVALID_PARAMS_MSG "Invalid params" + JSONRPC_INVALID_PARAMS = -32602, +#define JSONRPC_INTERNAL_ERROR_MSG "Internal error" + JSONRPC_INTERNAL_ERROR = -32603, +#define JSONRPC_SERVER_ERROR_MSG "Server error" + JSONRPC_SERVER_ERROR = -32000, +#define JSONRPC_LIMIT_REACHED_MSG "Limit reached" + JSONRPC_LIMIT_REACHED = -31997 +}; + +void jsonrpc_log(struct jsonrpc_request *, int, const char *, ...); +int jsonrpc_read_request(struct http_request *, struct jsonrpc_request *); +void jsonrpc_destroy_request(struct jsonrpc_request *); +int jsonrpc_error(struct jsonrpc_request *, int, const char *); +int jsonrpc_result(struct jsonrpc_request *, + int (*)(struct jsonrpc_request *, void *), void *); +#if defined(__cplusplus) +} +#endif +#endif /* !__H_JSONRPC_H */ + +#endif /* ! KORE_NO_HTTP */ diff -Nru kore-2.0.0/include/kore/kore.h kore-3.3.1/include/kore/kore.h --- kore-2.0.0/include/kore/kore.h 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/include/kore/kore.h 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,866 @@ +/* + * Copyright (c) 2013-2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __H_KORE_H +#define __H_KORE_H + +#if defined(__APPLE__) +#define daemon portability_is_king +#endif + +#include +#include +#include +#include +#include + +#include +#include + +#if !defined(KORE_NO_TLS) +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +#if defined(__APPLE__) +#undef daemon +extern int daemon(int, int); +#define st_mtim st_mtimespec +#endif + +#if !defined(KORE_NO_SENDFILE) && defined(KORE_NO_TLS) +#if defined(__MACH__) || defined(__FreeBSD_version) || defined(__linux__) +#define KORE_USE_PLATFORM_SENDFILE 1 +#endif +#endif + +#if defined(__OpenBSD__) +#define KORE_USE_PLATFORM_PLEDGE 1 +#endif + +#define KORE_RESULT_ERROR 0 +#define KORE_RESULT_OK 1 +#define KORE_RESULT_RETRY 2 + +#define KORE_TLS_VERSION_1_3 0 +#define KORE_TLS_VERSION_1_2 1 +#define KORE_TLS_VERSION_BOTH 2 + +#define KORE_WAIT_INFINITE (u_int64_t)-1 +#define KORE_RESEED_TIME (1800 * 1000) + +#define errno_s strerror(errno) +#define ssl_errno_s ERR_error_string(ERR_get_error(), NULL) + +#define KORE_DOMAINNAME_LEN 255 +#define KORE_PIDFILE_DEFAULT "kore.pid" +#define KORE_DEFAULT_CIPHER_LIST "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK:!kRSA:!kDSA" + +#if defined(KORE_DEBUG) +#define kore_debug(...) \ + if (kore_debug) \ + kore_debug_internal(__FILE__, __LINE__, __VA_ARGS__) +#else +#define kore_debug(...) +#endif + +#define NETBUF_RECV 0 +#define NETBUF_SEND 1 +#define NETBUF_SEND_PAYLOAD_MAX 8192 +#define SENDFILE_PAYLOAD_MAX (1024 * 1024 * 10) + +#define NETBUF_LAST_CHAIN 0 +#define NETBUF_BEFORE_CHAIN 1 + +#define NETBUF_CALL_CB_ALWAYS 0x01 +#define NETBUF_FORCE_REMOVE 0x02 +#define NETBUF_MUST_RESEND 0x04 +#define NETBUF_IS_STREAM 0x10 +#define NETBUF_IS_FILEREF 0x20 + +#define X509_GET_CN(c, o, l) \ + X509_NAME_get_text_by_NID(X509_get_subject_name(c), \ + NID_commonName, o, l) + +#define X509_CN_LENGTH (ub_common_name + 1) + +/* XXX hackish. */ +#if !defined(KORE_NO_HTTP) +struct http_request; +#endif + +#define KORE_FILEREF_SOFT_REMOVED 0x1000 + +struct kore_fileref { + int cnt; + int flags; + off_t size; + char *path; + u_int64_t mtime; + time_t mtime_sec; + u_int64_t expiration; +#if !defined(KORE_USE_PLATFORM_SENDFILE) + void *base; +#else + int fd; +#endif + TAILQ_ENTRY(kore_fileref) list; +}; + +struct netbuf { + u_int8_t *buf; + size_t s_off; + size_t b_len; + size_t m_len; + u_int8_t type; + u_int8_t flags; + + struct kore_fileref *file_ref; +#if defined(KORE_USE_PLATFORM_SENDFILE) + off_t fd_off; + off_t fd_len; +#endif + + void *owner; + void *extra; + int (*cb)(struct netbuf *); + + TAILQ_ENTRY(netbuf) list; +}; + +TAILQ_HEAD(netbuf_head, netbuf); + +#define KORE_TYPE_LISTENER 1 +#define KORE_TYPE_CONNECTION 2 +#define KORE_TYPE_PGSQL_CONN 3 +#define KORE_TYPE_TASK 4 +#define KORE_TYPE_PYSOCKET 5 +#define KORE_TYPE_CURL_HANDLE 6 + +#define CONN_STATE_UNKNOWN 0 +#define CONN_STATE_TLS_SHAKE 1 +#define CONN_STATE_ESTABLISHED 2 +#define CONN_STATE_DISCONNECTING 3 + +#define CONN_PROTO_UNKNOWN 0 +#define CONN_PROTO_HTTP 1 +#define CONN_PROTO_WEBSOCKET 2 +#define CONN_PROTO_MSG 3 + +#define KORE_EVENT_READ 0x01 +#define KORE_EVENT_WRITE 0x02 +#define KORE_EVENT_ERROR 0x04 + +#define CONN_IDLE_TIMER_ACT 0x01 +#define CONN_CLOSE_EMPTY 0x02 +#define CONN_WS_CLOSE_SENT 0x04 +#define CONN_IS_BUSY 0x08 + +#define KORE_IDLE_TIMER_MAX 5000 + +#define WEBSOCKET_OP_CONT 0x00 +#define WEBSOCKET_OP_TEXT 0x01 +#define WEBSOCKET_OP_BINARY 0x02 +#define WEBSOCKET_OP_CLOSE 0x08 +#define WEBSOCKET_OP_PING 0x09 +#define WEBSOCKET_OP_PONG 0x0a + +#define WEBSOCKET_BROADCAST_LOCAL 1 +#define WEBSOCKET_BROADCAST_GLOBAL 2 + +#define KORE_TIMER_ONESHOT 0x01 +#define KORE_TIMER_FLAGS (KORE_TIMER_ONESHOT) + +#define KORE_CONNECTION_PRUNE_DISCONNECT 0 +#define KORE_CONNECTION_PRUNE_ALL 1 + +struct kore_event { + int type; + int flags; + void (*handle)(void *, int); +} __attribute__((packed)); + +struct connection { + struct kore_event evt; + int fd; + u_int8_t state; + u_int8_t proto; + void *owner; +#if !defined(KORE_NO_TLS) + X509 *cert; + SSL *ssl; + int tls_reneg; +#endif + u_int8_t flags; + void *hdlr_extra; + + int (*handle)(struct connection *); + void (*disconnect)(struct connection *); + int (*read)(struct connection *, size_t *); + int (*write)(struct connection *, size_t, size_t *); + + int family; + union { + struct sockaddr_in ipv4; + struct sockaddr_in6 ipv6; + struct sockaddr_un sun; + } addr; + + struct { + u_int64_t length; + u_int64_t start; + } idle_timer; + + struct netbuf_head send_queue; + struct netbuf *snb; + struct netbuf *rnb; + +#if !defined(KORE_NO_HTTP) + u_int64_t http_start; + u_int64_t http_timeout; + struct kore_runtime_call *ws_connect; + struct kore_runtime_call *ws_message; + struct kore_runtime_call *ws_disconnect; + TAILQ_HEAD(, http_request) http_requests; +#endif + + TAILQ_ENTRY(connection) list; +}; + +TAILQ_HEAD(connection_list, connection); +extern struct connection_list connections; +extern struct connection_list disconnected; + +#define KORE_RUNTIME_NATIVE 0 +#define KORE_RUNTIME_PYTHON 1 + +struct kore_runtime { + int type; +#if !defined(KORE_NO_HTTP) + int (*http_request)(void *, struct http_request *); + int (*validator)(void *, struct http_request *, const void *); + void (*wsconnect)(void *, struct connection *); + void (*wsdisconnect)(void *, struct connection *); + void (*wsmessage)(void *, struct connection *, + u_int8_t, const void *, size_t); +#endif + void (*execute)(void *); + int (*onload)(void *, int); + void (*connect)(void *, struct connection *); + void (*configure)(void *, int, char **); +}; + +struct kore_runtime_call { + void *addr; + struct kore_runtime *runtime; +}; + +extern struct kore_runtime kore_native_runtime; + +struct listener { + struct kore_event evt; + int fd; + int family; + struct kore_runtime_call *connect; + + LIST_ENTRY(listener) list; +}; + +LIST_HEAD(listener_head, listener); + +#if !defined(KORE_NO_HTTP) + +#define KORE_PARAMS_QUERY_STRING 0x0001 + +struct kore_handler_params { + char *name; + int flags; + u_int8_t method; + struct kore_validator *validator; + + TAILQ_ENTRY(kore_handler_params) list; +}; + +#define KORE_AUTH_TYPE_COOKIE 1 +#define KORE_AUTH_TYPE_HEADER 2 +#define KORE_AUTH_TYPE_REQUEST 3 + +struct kore_auth { + u_int8_t type; + char *name; + char *value; + char *redirect; + struct kore_validator *validator; + + TAILQ_ENTRY(kore_auth) list; +}; + +#define HANDLER_TYPE_STATIC 1 +#define HANDLER_TYPE_DYNAMIC 2 + +#endif /* !KORE_NO_HTTP */ + +#define KORE_MODULE_LOAD 1 +#define KORE_MODULE_UNLOAD 2 + +#define KORE_MODULE_NATIVE 0 +#define KORE_MODULE_PYTHON 1 + +struct kore_module; + +struct kore_module_functions { + void (*free)(struct kore_module *); + void (*reload)(struct kore_module *); + int (*callback)(struct kore_module *, int); + void (*load)(struct kore_module *); + void *(*getsym)(struct kore_module *, const char *); +}; + +struct kore_module { + void *handle; + char *path; + char *onload; + int type; + time_t mtime; + struct kore_runtime_call *ocb; + + struct kore_module_functions *fun; + struct kore_runtime *runtime; + + TAILQ_ENTRY(kore_module) list; +}; + +#if !defined(KORE_NO_HTTP) +struct kore_module_handle { + char *path; + char *func; + int type; + int errors; + regex_t rctx; + struct kore_domain *dom; + struct kore_runtime_call *rcall; + struct kore_auth *auth; + int methods; + TAILQ_HEAD(, kore_handler_params) params; + TAILQ_ENTRY(kore_module_handle) list; +}; +#endif + +/* + * The workers get a 128KB log buffer per worker, and parent will fetch their + * logs when it reached at least 75% of that or if its been > 1 second since + * it was last synced. + */ +#define KORE_ACCESSLOG_BUFLEN 131072U +#define KORE_ACCESSLOG_SYNC 98304U + +struct kore_alog_header { + u_int16_t domain; + u_int16_t loglen; +} __attribute__((packed)); + +struct kore_worker { + u_int8_t id; + u_int8_t cpu; + pid_t pid; + int pipe[2]; + struct connection *msg[2]; + u_int8_t has_lock; + int restarted; + u_int64_t time_locked; + struct kore_module_handle *active_hdlr; + + /* Used by the workers to store accesslogs. */ + struct { + int lock; + size_t offset; + char buf[KORE_ACCESSLOG_BUFLEN]; + } lb; +}; + +struct kore_domain { + u_int16_t id; + char *domain; + int accesslog; + struct kore_buf *logbuf; + int logerr; + u_int64_t logwarn; +#if !defined(KORE_NO_TLS) + char *cafile; + char *crlfile; + char *certfile; + char *certkey; + SSL_CTX *ssl_ctx; + int x509_verify_depth; +#endif +#if !defined(KORE_NO_HTTP) + TAILQ_HEAD(, kore_module_handle) handlers; +#endif + TAILQ_ENTRY(kore_domain) list; +}; + +TAILQ_HEAD(kore_domain_h, kore_domain); + +#if !defined(KORE_NO_HTTP) + +#define KORE_VALIDATOR_TYPE_REGEX 1 +#define KORE_VALIDATOR_TYPE_FUNCTION 2 + +struct kore_validator { + u_int8_t type; + char *name; + char *arg; + regex_t rctx; + struct kore_runtime_call *rcall; + + TAILQ_ENTRY(kore_validator) list; +}; +#endif /* !KORE_NO_HTTP */ + +#define KORE_BUF_OWNER_API 0x0001 + +struct kore_buf { + u_int8_t *data; + int flags; + size_t length; + size_t offset; +}; + +struct kore_pool_region { + void *start; + size_t length; + LIST_ENTRY(kore_pool_region) list; +}; + +struct kore_pool_entry { + u_int8_t state; + struct kore_pool_region *region; + LIST_ENTRY(kore_pool_entry) list; +}; + +struct kore_pool { + size_t elen; + size_t slen; + size_t elms; + size_t inuse; + size_t growth; + volatile int lock; + char *name; + + LIST_HEAD(, kore_pool_region) regions; + LIST_HEAD(, kore_pool_entry) freelist; +}; + +struct kore_timer { + u_int64_t nextrun; + u_int64_t interval; + int flags; + void *arg; + void (*cb)(void *, u_int64_t); + + TAILQ_ENTRY(kore_timer) list; +}; + +#define KORE_WORKER_KEYMGR 0 +#define KORE_WORKER_POLICY_RESTART 1 +#define KORE_WORKER_POLICY_TERMINATE 2 + +/* Reserved message ids, registered on workers. */ +#define KORE_MSG_WEBSOCKET 1 +#define KORE_MSG_KEYMGR_REQ 2 +#define KORE_MSG_KEYMGR_RESP 3 +#define KORE_MSG_SHUTDOWN 4 +#define KORE_MSG_ENTROPY_REQ 5 +#define KORE_MSG_ENTROPY_RESP 6 +#define KORE_MSG_CERTIFICATE 7 +#define KORE_MSG_CERTIFICATE_REQ 8 +#define KORE_MSG_CRL 9 +#define KORE_MSG_ACCEPT_AVAILABLE 10 + +/* Predefined message targets. */ +#define KORE_MSG_PARENT 1000 +#define KORE_MSG_WORKER_ALL 1001 + +struct kore_msg { + u_int8_t id; + u_int16_t src; + u_int16_t dst; + size_t length; +}; + +#if !defined(KORE_NO_TLS) +struct kore_keyreq { + int padding; + char domain[KORE_DOMAINNAME_LEN]; + u_int16_t domain_len; + u_int16_t data_len; + u_int8_t data[]; +}; + +struct kore_x509_msg { + char domain[KORE_DOMAINNAME_LEN]; + u_int16_t domain_len; + size_t data_len; + u_int8_t data[]; +}; +#endif + +#if !defined(KORE_SINGLE_BINARY) +extern char *config_file; +#endif + +extern pid_t kore_pid; +extern int foreground; +extern int kore_quiet; +extern int kore_debug; +extern int skip_chroot; +extern int skip_runas; +extern char *kore_pidfile; +extern char *kore_root_path; +extern char *kore_runas_user; +extern char *kore_tls_cipher_list; + +extern volatile sig_atomic_t sig_recv; + +#if !defined(KORE_NO_TLS) +extern int tls_version; +extern DH *tls_dhparam; +extern char *rand_file; +extern char *keymgr_runas_user; +extern char *keymgr_root_path; +#endif + +extern u_int8_t nlisteners; +extern u_int16_t cpu_count; +extern u_int8_t worker_count; +extern const char *kore_version; +extern int worker_policy; +extern u_int8_t worker_set_affinity; +extern u_int32_t worker_rlimit_nofiles; +extern u_int32_t worker_max_connections; +extern u_int32_t worker_active_connections; +extern u_int32_t worker_accept_threshold; +extern u_int64_t kore_websocket_maxframe; +extern u_int64_t kore_websocket_timeout; +extern u_int32_t kore_socket_backlog; + +extern struct listener_head listeners; +extern struct kore_worker *worker; +extern struct kore_domain_h domains; +extern struct kore_domain *primary_dom; +extern struct kore_pool nb_pool; + +void kore_signal(int); +void kore_shutdown(void); +void kore_signal_setup(void); +void kore_proctitle(const char *); + +void kore_worker_reap(void); +void kore_worker_init(void); +void kore_worker_make_busy(void); +void kore_worker_shutdown(void); +void kore_worker_dispatch_signal(int); +void kore_worker_spawn(u_int16_t, u_int16_t); +void kore_worker_entry(struct kore_worker *); +void kore_worker_privdrop(const char *, const char *); + +struct kore_worker *kore_worker_data(u_int8_t); + +void kore_platform_init(void); +void kore_platform_event_init(void); +void kore_platform_event_cleanup(void); +void kore_platform_proctitle(char *); +void kore_platform_disable_read(int); +void kore_platform_disable_write(int); +void kore_platform_enable_accept(void); +void kore_platform_disable_accept(void); +void kore_platform_event_wait(u_int64_t); +void kore_platform_event_all(int, void *); +void kore_platform_schedule_read(int, void *); +void kore_platform_schedule_write(int, void *); +void kore_platform_event_schedule(int, int, int, void *); +void kore_platform_worker_setcpu(struct kore_worker *); + +#if defined(KORE_USE_PLATFORM_SENDFILE) +int kore_platform_sendfile(struct connection *, struct netbuf *); +#endif + +#if defined(KORE_USE_PLATFORM_PLEDGE) +void kore_platform_pledge(void); +void kore_platform_add_pledge(const char *); +#endif + +void kore_accesslog_init(u_int16_t); +void kore_accesslog_worker_init(void); +void kore_accesslog_run(void *, u_int64_t); +void kore_accesslog_gather(void *, u_int64_t, int); + +#if !defined(KORE_NO_HTTP) +int kore_auth_run(struct http_request *, struct kore_auth *); +int kore_auth_cookie(struct http_request *, struct kore_auth *); +int kore_auth_header(struct http_request *, struct kore_auth *); +int kore_auth_request(struct http_request *, struct kore_auth *); +void kore_auth_init(void); +int kore_auth_new(const char *); +struct kore_auth *kore_auth_lookup(const char *); +#endif + +void kore_timer_init(void); +void kore_timer_run(u_int64_t); +u_int64_t kore_timer_next_run(u_int64_t); +void kore_timer_remove(struct kore_timer *); +struct kore_timer *kore_timer_add(void (*cb)(void *, u_int64_t), + u_int64_t, void *, int); + +void kore_listener_cleanup(void); +void kore_listener_accept(void *, int); +void kore_listener_free(struct listener *); +struct listener *kore_listener_alloc(int, const char *); + +int kore_sockopt(int, int, int); +int kore_server_bind_unix(const char *, const char *); +int kore_server_bind(const char *, const char *, const char *); +#if !defined(KORE_NO_TLS) +int kore_tls_sni_cb(SSL *, int *, void *); +void kore_tls_info_callback(const SSL *, int, int); +#endif + +void kore_connection_init(void); +void kore_connection_cleanup(void); +void kore_connection_prune(int); +struct connection *kore_connection_new(void *); +void kore_connection_event(void *, int); +int kore_connection_nonblock(int, int); +void kore_connection_check_timeout(u_int64_t); +int kore_connection_handle(struct connection *); +void kore_connection_remove(struct connection *); +void kore_connection_disconnect(struct connection *); +void kore_connection_start_idletimer(struct connection *); +void kore_connection_stop_idletimer(struct connection *); +void kore_connection_check_idletimer(u_int64_t, + struct connection *); +int kore_connection_accept(struct listener *, + struct connection **); + +u_int64_t kore_time_ms(void); +void kore_log_init(void); + +void *kore_malloc(size_t); +void kore_parse_config(void); +void kore_parse_config_file(FILE *); +void *kore_calloc(size_t, size_t); +void *kore_realloc(void *, size_t); +void kore_free(void *); +void kore_mem_init(void); +void kore_mem_cleanup(void); +void kore_mem_untag(void *); +void *kore_mem_lookup(u_int32_t); +void kore_mem_tag(void *, u_int32_t); +void *kore_malloc_tagged(size_t, u_int32_t); + +void *kore_pool_get(struct kore_pool *); +void kore_pool_put(struct kore_pool *, void *); +void kore_pool_init(struct kore_pool *, const char *, + size_t, size_t); +void kore_pool_cleanup(struct kore_pool *); + +char *kore_time_to_date(time_t); +char *kore_strdup(const char *); +time_t kore_date_to_time(const char *); +void kore_log(int, const char *, ...) + __attribute__((format (printf, 2, 3))); +u_int64_t kore_strtonum64(const char *, int, int *); +size_t kore_strlcpy(char *, const char *, const size_t); +void kore_server_disconnect(struct connection *); +int kore_split_string(char *, const char *, char **, size_t); +void kore_strip_chars(char *, const char, char **); +int kore_snprintf(char *, size_t, int *, const char *, ...); +long long kore_strtonum(const char *, int, long long, long long, int *); +double kore_strtodouble(const char *, long double, long double, int *); +int kore_base64_encode(const void *, size_t, char **); +int kore_base64_decode(const char *, u_int8_t **, size_t *); +void *kore_mem_find(void *, size_t, const void *, size_t); +char *kore_text_trim(char *, size_t); +char *kore_read_line(FILE *, char *, size_t); + +#if !defined(KORE_NO_HTTP) +void kore_websocket_handshake(struct http_request *, + const char *, const char *, const char *); +int kore_websocket_send_clean(struct netbuf *); +void kore_websocket_send(struct connection *, + u_int8_t, const void *, size_t); +void kore_websocket_broadcast(struct connection *, + u_int8_t, const void *, size_t, int); +#endif + +void kore_msg_init(void); +void kore_msg_worker_init(void); +void kore_msg_parent_init(void); +void kore_msg_parent_add(struct kore_worker *); +void kore_msg_parent_remove(struct kore_worker *); +void kore_msg_send(u_int16_t, u_int8_t, const void *, size_t); +int kore_msg_register(u_int8_t, + void (*cb)(struct kore_msg *, const void *)); + +#if !defined(KORE_NO_HTTP) +void kore_filemap_init(void); +void kore_filemap_resolve_paths(void); +int kore_filemap_create(struct kore_domain *, const char *, + const char *); +extern char *kore_filemap_ext; +extern char *kore_filemap_index; +#endif + +void kore_fileref_init(void); +struct kore_fileref *kore_fileref_get(const char *); +struct kore_fileref *kore_fileref_create(const char *, int, off_t, + struct timespec *); +void kore_fileref_release(struct kore_fileref *); + +void kore_domain_init(void); +void kore_domain_cleanup(void); +int kore_domain_new(char *); +void kore_domain_free(struct kore_domain *); +void kore_module_init(void); +void kore_module_cleanup(void); +void kore_module_reload(int); +void kore_module_onload(void); +int kore_module_loaded(void); +void kore_domain_closelogs(void); +void *kore_module_getsym(const char *, struct kore_runtime **); +void kore_domain_load_crl(void); +void kore_domain_keymgr_init(void); +void kore_domain_callback(void (*cb)(struct kore_domain *)); +void kore_domain_tlsinit(struct kore_domain *, const void *, size_t); +void kore_domain_crl_add(struct kore_domain *, const void *, size_t); +#if !defined(KORE_NO_HTTP) +int kore_module_handler_new(const char *, const char *, + const char *, const char *, int); +void kore_module_handler_free(struct kore_module_handle *); +struct kore_module_handle *kore_module_handler_find(const char *, + const char *); +#endif + +struct kore_runtime_call *kore_runtime_getcall(const char *); +struct kore_module *kore_module_load(const char *, + const char *, int); + +void kore_runtime_execute(struct kore_runtime_call *); +int kore_runtime_onload(struct kore_runtime_call *, int); +void kore_runtime_configure(struct kore_runtime_call *, int, char **); +void kore_runtime_connect(struct kore_runtime_call *, struct connection *); +#if !defined(KORE_NO_HTTP) +int kore_runtime_http_request(struct kore_runtime_call *, + struct http_request *); +int kore_runtime_validator(struct kore_runtime_call *, + struct http_request *, const void *); +void kore_runtime_wsconnect(struct kore_runtime_call *, struct connection *); +void kore_runtime_wsdisconnect(struct kore_runtime_call *, + struct connection *); +void kore_runtime_wsmessage(struct kore_runtime_call *, + struct connection *, u_int8_t, const void *, size_t); +#endif + +struct kore_domain *kore_domain_byid(u_int16_t); +struct kore_domain *kore_domain_lookup(const char *); + +#if !defined(KORE_NO_HTTP) +void kore_validator_init(void); +void kore_validator_reload(void); +int kore_validator_add(const char *, u_int8_t, const char *); +int kore_validator_run(struct http_request *, const char *, char *); +int kore_validator_check(struct http_request *, + struct kore_validator *, const void *); +struct kore_validator *kore_validator_lookup(const char *); +#endif + +void fatal(const char *, ...) __attribute__((noreturn)); +void fatalx(const char *, ...) __attribute__((noreturn)); +void kore_debug_internal(char *, int, const char *, ...); + +u_int16_t net_read16(u_int8_t *); +u_int32_t net_read32(u_int8_t *); +u_int64_t net_read64(u_int8_t *); +void net_write16(u_int8_t *, u_int16_t); +void net_write32(u_int8_t *, u_int32_t); +void net_write64(u_int8_t *, u_int64_t); + +void net_init(void); +void net_cleanup(void); +struct netbuf *net_netbuf_get(void); +int net_send(struct connection *); +int net_send_flush(struct connection *); +int net_recv_flush(struct connection *); +int net_read(struct connection *, size_t *); +int net_read_tls(struct connection *, size_t *); +int net_write(struct connection *, size_t, size_t *); +int net_write_tls(struct connection *, size_t, size_t *); +void net_recv_reset(struct connection *, size_t, + int (*cb)(struct netbuf *)); +void net_remove_netbuf(struct connection *, struct netbuf *); +void net_recv_queue(struct connection *, size_t, int, + int (*cb)(struct netbuf *)); +void net_recv_expand(struct connection *c, size_t, + int (*cb)(struct netbuf *)); +void net_send_queue(struct connection *, const void *, size_t); +void net_send_stream(struct connection *, void *, + size_t, int (*cb)(struct netbuf *), struct netbuf **); +void net_send_fileref(struct connection *, struct kore_fileref *); + +void kore_buf_free(struct kore_buf *); +struct kore_buf *kore_buf_alloc(size_t); +void kore_buf_init(struct kore_buf *, size_t); +void kore_buf_append(struct kore_buf *, const void *, size_t); +u_int8_t *kore_buf_release(struct kore_buf *, size_t *); +void kore_buf_reset(struct kore_buf *); +void kore_buf_cleanup(struct kore_buf *); + +char *kore_buf_stringify(struct kore_buf *, size_t *); +void kore_buf_appendf(struct kore_buf *, const char *, ...); +void kore_buf_appendv(struct kore_buf *, const char *, va_list); +void kore_buf_replace_string(struct kore_buf *, + const char *, const void *, size_t); + +void kore_keymgr_run(void); +void kore_keymgr_cleanup(int); + +void kore_worker_teardown(void); +void kore_parent_teardown(void); +void kore_worker_configure(void); +void kore_parent_daemonized(void); +void kore_parent_configure(int, char **); + +#if defined(__cplusplus) +} +#endif + +#endif /* !__H_KORE_H */ diff -Nru kore-2.0.0/include/kore/pgsql.h kore-3.3.1/include/kore/pgsql.h --- kore-2.0.0/include/kore/pgsql.h 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/include/kore/pgsql.h 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2014-2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef _H_KORE_PGSQL +#define _H_KORE_PGSQL + +#include + +#define KORE_PGSQL_FORMAT_TEXT 0 +#define KORE_PGSQL_FORMAT_BINARY 1 + +#define KORE_PGSQL_SYNC 0x0001 +#define KORE_PGSQL_ASYNC 0x0002 +#define KORE_PGSQL_SCHEDULED 0x0004 + +#define KORE_PGSQL_PARAM_BINARY(v, l) v, l, 1 +#define KORE_PGSQL_PARAM_TEXT_LEN(v, l) v, l, 0 +#define KORE_PGSQL_PARAM_TEXT(v) v, strlen(v), 0 + +#if defined(__cplusplus) +extern "C" { +#endif + +struct pgsql_conn { + struct kore_event evt; + u_int8_t flags; + char *name; + + PGconn *db; + struct pgsql_job *job; + TAILQ_ENTRY(pgsql_conn) list; +}; + +struct pgsql_db { + char *name; + char *conn_string; + u_int16_t conn_max; + u_int16_t conn_count; + + LIST_ENTRY(pgsql_db) rlist; +}; + +struct kore_pgsql { + u_int8_t state; + int flags; + char *error; + PGresult *result; + struct pgsql_conn *conn; + + struct { + char *channel; + char *extra; + } notify; + + struct http_request *req; + void *arg; + void (*cb)(struct kore_pgsql *, void *); + + LIST_ENTRY(kore_pgsql) rlist; +}; + +extern u_int16_t pgsql_conn_max; +extern u_int32_t pgsql_queue_limit; + +void kore_pgsql_sys_init(void); +void kore_pgsql_sys_cleanup(void); +void kore_pgsql_init(struct kore_pgsql *); +void kore_pgsql_bind_request(struct kore_pgsql *, struct http_request *); +void kore_pgsql_bind_callback(struct kore_pgsql *, + void (*cb)(struct kore_pgsql *, void *), void *); +int kore_pgsql_setup(struct kore_pgsql *, const char *, int); +void kore_pgsql_handle(void *, int); +void kore_pgsql_cleanup(struct kore_pgsql *); +void kore_pgsql_continue(struct kore_pgsql *); +int kore_pgsql_query(struct kore_pgsql *, const void *); +int kore_pgsql_query_params(struct kore_pgsql *, + const void *, int, int, ...); +int kore_pgsql_v_query_params(struct kore_pgsql *, + const void *, int, int, va_list); +int kore_pgsql_query_param_fields(struct kore_pgsql *, const void *, + int, int, const char **, int *, int *); +int kore_pgsql_register(const char *, const char *); +int kore_pgsql_ntuples(struct kore_pgsql *); +int kore_pgsql_nfields(struct kore_pgsql *); +void kore_pgsql_logerror(struct kore_pgsql *); +char *kore_pgsql_fieldname(struct kore_pgsql *, int); +char *kore_pgsql_getvalue(struct kore_pgsql *, int, int); +int kore_pgsql_getlength(struct kore_pgsql *, int, int); +int kore_pgsql_column_binary(struct kore_pgsql *, int); + +#if defined(__cplusplus) +} +#endif + +#define KORE_PGSQL_STATE_INIT 1 +#define KORE_PGSQL_STATE_WAIT 2 +#define KORE_PGSQL_STATE_RESULT 3 +#define KORE_PGSQL_STATE_ERROR 4 +#define KORE_PGSQL_STATE_DONE 5 +#define KORE_PGSQL_STATE_COMPLETE 6 +#define KORE_PGSQL_STATE_NOTIFY 7 + +#endif diff -Nru kore-2.0.0/include/kore/python_api.h kore-3.3.1/include/kore/python_api.h --- kore-2.0.0/include/kore/python_api.h 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/include/kore/python_api.h 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 Stanislav Yudin + * Copyright (c) 2017-2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __H_PYTHON_H +#define __H_PYTHON_H + +#undef _POSIX_C_SOURCE +#undef _XOPEN_SOURCE + +#include + +void kore_python_init(void); +void kore_python_preinit(void); +void kore_python_cleanup(void); +void kore_python_coro_run(void); +void kore_python_proc_reap(void); +void kore_python_path(const char *); +void kore_python_coro_delete(void *); +void kore_python_log_error(const char *); + +PyObject *kore_python_callable(PyObject *, const char *); + +extern struct kore_module_functions kore_python_module; +extern struct kore_runtime kore_python_runtime; + +#endif diff -Nru kore-2.0.0/include/kore/python_methods.h kore-3.3.1/include/kore/python_methods.h --- kore-2.0.0/include/kore/python_methods.h 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/include/kore/python_methods.h 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,789 @@ +/* + * Copyright (c) 2017-2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#define CORO_STATE_RUNNABLE 1 +#define CORO_STATE_SUSPENDED 2 + +struct python_coro { + u_int64_t id; + int state; + PyObject *obj; + PyObject *result; + struct pysocket_op *sockop; + struct pygather_op *gatherop; + struct http_request *request; + PyObject *exception; + char *exception_msg; + TAILQ_ENTRY(python_coro) list; +}; + +TAILQ_HEAD(coro_list, python_coro); + +static PyObject *python_kore_log(PyObject *, PyObject *); +static PyObject *python_kore_time(PyObject *, PyObject *); +static PyObject *python_kore_lock(PyObject *, PyObject *); +static PyObject *python_kore_proc(PyObject *, PyObject *); +static PyObject *python_kore_bind(PyObject *, PyObject *); +static PyObject *python_kore_timer(PyObject *, PyObject *); +static PyObject *python_kore_fatal(PyObject *, PyObject *); +static PyObject *python_kore_queue(PyObject *, PyObject *); +static PyObject *python_kore_tracer(PyObject *, PyObject *); +static PyObject *python_kore_fatalx(PyObject *, PyObject *); +static PyObject *python_kore_suspend(PyObject *, PyObject *); +static PyObject *python_kore_shutdown(PyObject *, PyObject *); +static PyObject *python_kore_bind_unix(PyObject *, PyObject *); +static PyObject *python_kore_task_create(PyObject *, PyObject *); +static PyObject *python_kore_socket_wrap(PyObject *, PyObject *); +static PyObject *python_kore_gather(PyObject *, PyObject *, PyObject *); + +#if defined(KORE_USE_PGSQL) +static PyObject *python_kore_pgsql_register(PyObject *, PyObject *); +#endif + +#if defined(KORE_USE_CURL) +static PyObject *python_kore_httpclient(PyObject *, + PyObject *, PyObject *); +#endif + +static PyObject *python_websocket_broadcast(PyObject *, PyObject *); + +#define METHOD(n, c, a) { n, (PyCFunction)c, a, NULL } +#define GETTER(n, g) { n, (getter)g, NULL, NULL, NULL } +#define SETTER(n, s) { n, NULL, (setter)g, NULL, NULL } +#define GETSET(n, g, s) { n, (getter)g, (setter)s, NULL, NULL } + +static struct PyMethodDef pykore_methods[] = { + METHOD("log", python_kore_log, METH_VARARGS), + METHOD("time", python_kore_time, METH_NOARGS), + METHOD("lock", python_kore_lock, METH_NOARGS), + METHOD("proc", python_kore_proc, METH_VARARGS), + METHOD("bind", python_kore_bind, METH_VARARGS), + METHOD("timer", python_kore_timer, METH_VARARGS), + METHOD("queue", python_kore_queue, METH_VARARGS), + METHOD("tracer", python_kore_tracer, METH_VARARGS), + METHOD("gather", python_kore_gather, METH_VARARGS | METH_KEYWORDS), + METHOD("fatal", python_kore_fatal, METH_VARARGS), + METHOD("fatalx", python_kore_fatalx, METH_VARARGS), + METHOD("suspend", python_kore_suspend, METH_VARARGS), + METHOD("shutdown", python_kore_shutdown, METH_NOARGS), + METHOD("bind_unix", python_kore_bind_unix, METH_VARARGS), + METHOD("task_create", python_kore_task_create, METH_VARARGS), + METHOD("socket_wrap", python_kore_socket_wrap, METH_VARARGS), + METHOD("websocket_broadcast", python_websocket_broadcast, METH_VARARGS), +#if defined(KORE_USE_PGSQL) + METHOD("register_database", python_kore_pgsql_register, METH_VARARGS), +#endif +#if defined(KORE_USE_CURL) + METHOD("httpclient", python_kore_httpclient, + METH_VARARGS | METH_KEYWORDS), +#endif + { NULL, NULL, 0, NULL } +}; + +static struct PyModuleDef pykore_module = { + PyModuleDef_HEAD_INIT, "kore", NULL, -1, pykore_methods +}; + +#define PYSUSPEND_OP_INIT 1 +#define PYSUSPEND_OP_WAIT 2 +#define PYSUSPEND_OP_CONTINUE 3 + +struct pysuspend_op { + PyObject_HEAD + int state; + int delay; + struct python_coro *coro; + struct kore_timer *timer; +}; + +static void pysuspend_op_dealloc(struct pysuspend_op *); + +static PyObject *pysuspend_op_await(PyObject *); +static PyObject *pysuspend_op_iternext(struct pysuspend_op *); + +static PyAsyncMethods pysuspend_op_async = { + (unaryfunc)pysuspend_op_await, + NULL, + NULL +}; + +static PyTypeObject pysuspend_op_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.suspend", + .tp_doc = "suspension operation", + .tp_as_async = &pysuspend_op_async, + .tp_iternext = (iternextfunc)pysuspend_op_iternext, + .tp_basicsize = sizeof(struct pysuspend_op), + .tp_dealloc = (destructor)pysuspend_op_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pytimer { + PyObject_HEAD + int flags; + struct kore_timer *run; + PyObject *callable; +}; + +static PyObject *pytimer_close(struct pytimer *, PyObject *); + +static PyMethodDef pytimer_methods[] = { + METHOD("close", pytimer_close, METH_NOARGS), + METHOD(NULL, NULL, -1) +}; + +static void pytimer_dealloc(struct pytimer *); + +static PyTypeObject pytimer_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.timer", + .tp_doc = "kore timer implementation", + .tp_methods = pytimer_methods, + .tp_basicsize = sizeof(struct pytimer), + .tp_dealloc = (destructor)pytimer_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +/* XXX */ +struct pysocket; +struct pysocket_op; + +struct pysocket_event { + struct kore_event evt; + struct pysocket *s; +}; + +struct pysocket { + PyObject_HEAD + int fd; + int family; + int protocol; + int scheduled; + PyObject *socket; + socklen_t addr_len; + + struct pysocket_event event; + struct pysocket_op *recvop; + struct pysocket_op *sendop; + + union { + struct sockaddr_in ipv4; + struct sockaddr_un sun; + } addr; +}; + +static PyObject *pysocket_send(struct pysocket *, PyObject *); +static PyObject *pysocket_recv(struct pysocket *, PyObject *); +static PyObject *pysocket_close(struct pysocket *, PyObject *); +static PyObject *pysocket_accept(struct pysocket *, PyObject *); +static PyObject *pysocket_sendto(struct pysocket *, PyObject *); +static PyObject *pysocket_connect(struct pysocket *, PyObject *); +static PyObject *pysocket_recvfrom(struct pysocket *, PyObject *); + +static PyMethodDef pysocket_methods[] = { + METHOD("recv", pysocket_recv, METH_VARARGS), + METHOD("send", pysocket_send, METH_VARARGS), + METHOD("close", pysocket_close, METH_NOARGS), + METHOD("accept", pysocket_accept, METH_NOARGS), + METHOD("sendto", pysocket_sendto, METH_VARARGS), + METHOD("connect", pysocket_connect, METH_VARARGS), + METHOD("recvfrom", pysocket_recvfrom, METH_VARARGS), + METHOD(NULL, NULL, -1), +}; + +static void pysocket_dealloc(struct pysocket *); + +static PyTypeObject pysocket_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.socket", + .tp_doc = "kore socket implementation", + .tp_methods = pysocket_methods, + .tp_basicsize = sizeof(struct pysocket), + .tp_dealloc = (destructor)pysocket_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +#define PYSOCKET_TYPE_ACCEPT 1 +#define PYSOCKET_TYPE_CONNECT 2 +#define PYSOCKET_TYPE_RECV 3 +#define PYSOCKET_TYPE_SEND 4 +#define PYSOCKET_TYPE_RECVFROM 5 +#define PYSOCKET_TYPE_SENDTO 6 + +struct pysocket_op { + PyObject_HEAD + int eof; + int type; + void *self; + struct python_coro *coro; + int state; + size_t length; + struct kore_buf buffer; + struct pysocket *socket; + struct kore_timer *timer; + + union { + struct sockaddr_in ipv4; + struct sockaddr_un sun; + } sendaddr; +}; + +static void pysocket_op_dealloc(struct pysocket_op *); + +static PyObject *pysocket_op_await(PyObject *); +static PyObject *pysocket_op_iternext(struct pysocket_op *); + +static PyAsyncMethods pysocket_op_async = { + (unaryfunc)pysocket_op_await, + NULL, + NULL +}; + +static PyTypeObject pysocket_op_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.socketop", + .tp_doc = "socket operation", + .tp_as_async = &pysocket_op_async, + .tp_iternext = (iternextfunc)pysocket_op_iternext, + .tp_basicsize = sizeof(struct pysocket_op), + .tp_dealloc = (destructor)pysocket_op_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pyqueue_waiting { + struct python_coro *coro; + struct pyqueue_op *op; + TAILQ_ENTRY(pyqueue_waiting) list; +}; + +struct pyqueue_object { + PyObject *obj; + TAILQ_ENTRY(pyqueue_object) list; +}; + +struct pyqueue { + PyObject_HEAD + TAILQ_HEAD(, pyqueue_object) objects; + TAILQ_HEAD(, pyqueue_waiting) waiting; +}; + +static PyObject *pyqueue_pop(struct pyqueue *, PyObject *); +static PyObject *pyqueue_push(struct pyqueue *, PyObject *); +static PyObject *pyqueue_popnow(struct pyqueue *, PyObject *); + +static PyMethodDef pyqueue_methods[] = { + METHOD("pop", pyqueue_pop, METH_NOARGS), + METHOD("push", pyqueue_push, METH_VARARGS), + METHOD("popnow", pyqueue_popnow, METH_NOARGS), + METHOD(NULL, NULL, -1) +}; + +static void pyqueue_dealloc(struct pyqueue *); + +static PyTypeObject pyqueue_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.queue", + .tp_doc = "queue", + .tp_methods = pyqueue_methods, + .tp_basicsize = sizeof(struct pyqueue), + .tp_dealloc = (destructor)pyqueue_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pyqueue_op { + PyObject_HEAD + struct pyqueue *queue; + struct pyqueue_waiting *waiting; +}; + +static void pyqueue_op_dealloc(struct pyqueue_op *); + +static PyObject *pyqueue_op_await(PyObject *); +static PyObject *pyqueue_op_iternext(struct pyqueue_op *); + +static PyAsyncMethods pyqueue_op_async = { + (unaryfunc)pyqueue_op_await, + NULL, + NULL +}; + +static PyTypeObject pyqueue_op_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.queueop", + .tp_doc = "queue waitable", + .tp_as_async = &pyqueue_op_async, + .tp_iternext = (iternextfunc)pyqueue_op_iternext, + .tp_basicsize = sizeof(struct pyqueue_op), + .tp_dealloc = (destructor)pyqueue_op_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pylock { + PyObject_HEAD + struct python_coro *owner; + TAILQ_HEAD(, pylock_op) ops; +}; + +static PyObject *pylock_aexit(struct pylock *, PyObject *); +static PyObject *pylock_aenter(struct pylock *, PyObject *); + +static PyMethodDef pylock_methods[] = { + METHOD("__aexit__", pylock_aexit, METH_VARARGS), + METHOD("__aenter__", pylock_aenter, METH_NOARGS), + METHOD(NULL, NULL, -1) +}; + +static void pylock_dealloc(struct pylock *); + +static PyTypeObject pylock_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.lock", + .tp_doc = "locking mechanism", + .tp_methods = pylock_methods, + .tp_basicsize = sizeof(struct pylock), + .tp_dealloc = (destructor)pylock_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pylock_op { + PyObject_HEAD + int locking; + int active; + struct pylock *lock; + struct python_coro *coro; + TAILQ_ENTRY(pylock_op) list; +}; + +static void pylock_op_dealloc(struct pylock_op *); + +static PyObject *pylock_op_await(PyObject *); +static PyObject *pylock_op_iternext(struct pylock_op *); + +static PyAsyncMethods pylock_op_async = { + (unaryfunc)pylock_op_await, + NULL, + NULL +}; + +static PyTypeObject pylock_op_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.lockop", + .tp_doc = "lock awaitable", + .tp_as_async = &pylock_op_async, + .tp_iternext = (iternextfunc)pylock_op_iternext, + .tp_basicsize = sizeof(struct pylock_op), + .tp_dealloc = (destructor)pylock_op_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pyproc { + PyObject_HEAD + pid_t pid; + int reaped; + int status; + struct pysocket *in; + struct pysocket *out; + struct python_coro *coro; + struct kore_timer *timer; + TAILQ_ENTRY(pyproc) list; +}; + +static void pyproc_dealloc(struct pyproc *); + +static PyObject *pyproc_kill(struct pyproc *, PyObject *); +static PyObject *pyproc_reap(struct pyproc *, PyObject *); +static PyObject *pyproc_recv(struct pyproc *, PyObject *); +static PyObject *pyproc_send(struct pyproc *, PyObject *); +static PyObject *pyproc_close_stdin(struct pyproc *, PyObject *); + +static PyMethodDef pyproc_methods[] = { + METHOD("kill", pyproc_kill, METH_NOARGS), + METHOD("reap", pyproc_reap, METH_NOARGS), + METHOD("recv", pyproc_recv, METH_VARARGS), + METHOD("send", pyproc_send, METH_VARARGS), + METHOD("close_stdin", pyproc_close_stdin, METH_NOARGS), + METHOD(NULL, NULL, -1), +}; + +static PyTypeObject pyproc_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.proc", + .tp_doc = "async process", + .tp_methods = pyproc_methods, + .tp_basicsize = sizeof(struct pyproc), + .tp_dealloc = (destructor)pyproc_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pyproc_op { + PyObject_HEAD + struct pyproc *proc; +}; + +static void pyproc_op_dealloc(struct pyproc_op *); + +static PyObject *pyproc_op_await(PyObject *); +static PyObject *pyproc_op_iternext(struct pyproc_op *); + +static PyAsyncMethods pyproc_op_async = { + (unaryfunc)pyproc_op_await, + NULL, + NULL +}; + +static PyTypeObject pyproc_op_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.proc_op", + .tp_doc = "proc reaper awaitable", + .tp_as_async = &pyproc_op_async, + .tp_iternext = (iternextfunc)pyproc_op_iternext, + .tp_basicsize = sizeof(struct pyproc_op), + .tp_dealloc = (destructor)pyproc_op_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pygather_coro { + struct python_coro *coro; + PyObject *result; + TAILQ_ENTRY(pygather_coro) list; +}; + +struct pygather_result { + PyObject *obj; + TAILQ_ENTRY(pygather_result) list; +}; + +struct pygather_op { + PyObject_HEAD + int count; + int running; + int concurrency; + struct python_coro *coro; + TAILQ_HEAD(, pygather_result) results; + TAILQ_HEAD(, pygather_coro) coroutines; +}; + +static void pygather_op_dealloc(struct pygather_op *); + +static PyObject *pygather_op_await(PyObject *); +static PyObject *pygather_op_iternext(struct pygather_op *); + +static PyAsyncMethods pygather_op_async = { + (unaryfunc)pygather_op_await, + NULL, + NULL +}; + +static PyTypeObject pygather_op_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.pygather_op", + .tp_doc = "coroutine gathering", + .tp_as_async = &pygather_op_async, + .tp_iternext = (iternextfunc)pygather_op_iternext, + .tp_basicsize = sizeof(struct pygather_op), + .tp_dealloc = (destructor)pygather_op_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pyconnection { + PyObject_HEAD + struct connection *c; +}; + +static PyObject *pyconnection_disconnect(struct pyconnection *, PyObject *); +static PyObject *pyconnection_websocket_send(struct pyconnection *, PyObject *); + +static PyMethodDef pyconnection_methods[] = { + METHOD("disconnect", pyconnection_disconnect, METH_NOARGS), + METHOD("websocket_send", pyconnection_websocket_send, METH_VARARGS), + METHOD(NULL, NULL, -1), +}; + +static PyObject *pyconnection_get_fd(struct pyconnection *, void *); +static PyObject *pyconnection_get_addr(struct pyconnection *, void *); + +#if !defined(KORE_NO_TLS) +static PyObject *pyconnection_get_peer_x509(struct pyconnection *, void *); +#endif + +static PyGetSetDef pyconnection_getset[] = { + GETTER("fd", pyconnection_get_fd), + GETTER("addr", pyconnection_get_addr), +#if !defined(KORE_NO_TLS) + GETTER("x509", pyconnection_get_peer_x509), +#endif + GETTER(NULL, NULL), +}; + +static void pyconnection_dealloc(struct pyconnection *); + +static PyTypeObject pyconnection_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.connection", + .tp_doc = "struct connection", + .tp_getset = pyconnection_getset, + .tp_methods = pyconnection_methods, + .tp_basicsize = sizeof(struct pyconnection), + .tp_dealloc = (destructor)pyconnection_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pyhttp_request { + PyObject_HEAD + struct http_request *req; + PyObject *data; +}; + +struct pyhttp_iterobj { + int remove; + PyObject *iterator; + struct connection *connection; + struct kore_buf buf; +}; + +struct pyhttp_file { + PyObject_HEAD + struct http_file *file; +}; + +static void pyhttp_dealloc(struct pyhttp_request *); +static void pyhttp_file_dealloc(struct pyhttp_file *); + +#if defined(KORE_USE_PGSQL) +static PyObject *pyhttp_pgsql(struct pyhttp_request *, PyObject *, PyObject *); +#endif +static PyObject *pyhttp_cookie(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_response(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_argument(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_body_read(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_file_lookup(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_populate_get(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_populate_post(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_populate_multi(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_populate_cookies(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_request_header(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_response_header(struct pyhttp_request *, PyObject *); +static PyObject *pyhttp_websocket_handshake(struct pyhttp_request *, + PyObject *); + +static PyMethodDef pyhttp_request_methods[] = { +#if defined(KORE_USE_PGSQL) + METHOD("pgsql", pyhttp_pgsql, METH_VARARGS | METH_KEYWORDS), +#endif + METHOD("cookie", pyhttp_cookie, METH_VARARGS), + METHOD("response", pyhttp_response, METH_VARARGS), + METHOD("argument", pyhttp_argument, METH_VARARGS), + METHOD("body_read", pyhttp_body_read, METH_VARARGS), + METHOD("file_lookup", pyhttp_file_lookup, METH_VARARGS), + METHOD("populate_get", pyhttp_populate_get, METH_NOARGS), + METHOD("populate_post", pyhttp_populate_post, METH_NOARGS), + METHOD("populate_multi", pyhttp_populate_multi, METH_NOARGS), + METHOD("populate_cookies", pyhttp_populate_cookies, METH_NOARGS), + METHOD("request_header", pyhttp_request_header, METH_VARARGS), + METHOD("response_header", pyhttp_response_header, METH_VARARGS), + METHOD("websocket_handshake", pyhttp_websocket_handshake, METH_VARARGS), + METHOD(NULL, NULL, -1) +}; + +static PyObject *pyhttp_get_host(struct pyhttp_request *, void *); +static PyObject *pyhttp_get_path(struct pyhttp_request *, void *); +static PyObject *pyhttp_get_body(struct pyhttp_request *, void *); +static PyObject *pyhttp_get_agent(struct pyhttp_request *, void *); +static PyObject *pyhttp_get_method(struct pyhttp_request *, void *); +static PyObject *pyhttp_get_body_path(struct pyhttp_request *, void *); +static PyObject *pyhttp_get_connection(struct pyhttp_request *, void *); + +static PyGetSetDef pyhttp_request_getset[] = { + GETTER("host", pyhttp_get_host), + GETTER("path", pyhttp_get_path), + GETTER("body", pyhttp_get_body), + GETTER("agent", pyhttp_get_agent), + GETTER("method", pyhttp_get_method), + GETTER("body_path", pyhttp_get_body_path), + GETTER("connection", pyhttp_get_connection), + GETTER(NULL, NULL) +}; + +static PyTypeObject pyhttp_request_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.http_request", + .tp_doc = "struct http_request", + .tp_getset = pyhttp_request_getset, + .tp_methods = pyhttp_request_methods, + .tp_dealloc = (destructor)pyhttp_dealloc, + .tp_basicsize = sizeof(struct pyhttp_request), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +static PyObject *pyhttp_file_read(struct pyhttp_file *, PyObject *); + +static PyMethodDef pyhttp_file_methods[] = { + METHOD("read", pyhttp_file_read, METH_VARARGS), + METHOD(NULL, NULL, -1) +}; + +static PyObject *pyhttp_file_get_name(struct pyhttp_file *, void *); +static PyObject *pyhttp_file_get_filename(struct pyhttp_file *, void *); + +static PyGetSetDef pyhttp_file_getset[] = { + GETTER("name", pyhttp_file_get_name), + GETTER("filename", pyhttp_file_get_filename), + GETTER(NULL, NULL) +}; + +static PyTypeObject pyhttp_file_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.http_file", + .tp_doc = "struct http_file", + .tp_getset = pyhttp_file_getset, + .tp_methods = pyhttp_file_methods, + .tp_dealloc = (destructor)pyhttp_file_dealloc, + .tp_basicsize = sizeof(struct pyhttp_file), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +#if defined(KORE_USE_CURL) +struct pyhttp_client { + PyObject_HEAD + char *url; + char *unix; + char *tlskey; + char *tlscert; + char *cabundle; + int tlsverify; +}; + +#define PYHTTP_CLIENT_OP_RUN 1 +#define PYHTTP_CLIENT_OP_RESULT 2 + +struct pyhttp_client_op { + PyObject_HEAD + int state; + int headers; + struct kore_curl curl; + struct python_coro *coro; +}; + +static PyObject *pyhttp_client_op_await(PyObject *); +static PyObject *pyhttp_client_op_iternext(struct pyhttp_client_op *); + +static void pyhttp_client_dealloc(struct pyhttp_client *); +static void pyhttp_client_op_dealloc(struct pyhttp_client_op *); + +static PyObject *pyhttp_client_get(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_put(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_post(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_head(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_patch(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_delete(struct pyhttp_client *, + PyObject *, PyObject *); +static PyObject *pyhttp_client_options(struct pyhttp_client *, + PyObject *, PyObject *); + +static PyMethodDef pyhttp_client_methods[] = { + METHOD("get", pyhttp_client_get, METH_VARARGS | METH_KEYWORDS), + METHOD("put", pyhttp_client_put, METH_VARARGS | METH_KEYWORDS), + METHOD("post", pyhttp_client_post, METH_VARARGS | METH_KEYWORDS), + METHOD("head", pyhttp_client_head, METH_VARARGS | METH_KEYWORDS), + METHOD("patch", pyhttp_client_patch, METH_VARARGS | METH_KEYWORDS), + METHOD("delete", pyhttp_client_delete, METH_VARARGS | METH_KEYWORDS), + METHOD("options", pyhttp_client_options, METH_VARARGS | METH_KEYWORDS), + METHOD(NULL, NULL, -1) +}; + +static PyTypeObject pyhttp_client_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.httpclient", + .tp_doc = "An asynchronous HTTP client", + .tp_methods = pyhttp_client_methods, + .tp_basicsize = sizeof(struct pyhttp_client), + .tp_dealloc = (destructor)pyhttp_client_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +static PyAsyncMethods pyhttp_client_op_async = { + (unaryfunc)pyhttp_client_op_await, + NULL, + NULL +}; + +static PyTypeObject pyhttp_client_op_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.httpclientop", + .tp_doc = "Asynchronous HTTP client operation", + .tp_as_async = &pyhttp_client_op_async, + .tp_iternext = (iternextfunc)pyhttp_client_op_iternext, + .tp_basicsize = sizeof(struct pyhttp_client_op), + .tp_dealloc = (destructor)pyhttp_client_op_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; +#endif + +#if defined(KORE_USE_PGSQL) + +#define PYKORE_PGSQL_PREINIT 1 +#define PYKORE_PGSQL_INITIALIZE 2 +#define PYKORE_PGSQL_QUERY 3 +#define PYKORE_PGSQL_WAIT 4 + +struct pykore_pgsql { + PyObject_HEAD + int state; + int binary; + struct kore_pgsql sql; + + char *db; + struct http_request *req; + char *query; + PyObject *result; + struct { + int count; + const char **values; + int *lengths; + int *formats; + PyObject **objs; + } param; +}; + +static void pykore_pgsql_dealloc(struct pykore_pgsql *); +static int pykore_pgsql_result(struct pykore_pgsql *); +static int pykore_pgsql_params(struct pykore_pgsql *, PyObject *); + +static PyObject *pykore_pgsql_await(PyObject *); +static PyObject *pykore_pgsql_iternext(struct pykore_pgsql *); + +static PyAsyncMethods pykore_pgsql_async = { + (unaryfunc)pykore_pgsql_await, + NULL, + NULL +}; + +static PyTypeObject pykore_pgsql_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.pgsql", + .tp_doc = "struct kore_pgsql", + .tp_as_async = &pykore_pgsql_async, + .tp_iternext = (iternextfunc)pykore_pgsql_iternext, + .tp_basicsize = sizeof(struct pykore_pgsql), + .tp_dealloc = (destructor)pykore_pgsql_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; +#endif diff -Nru kore-2.0.0/include/kore/tasks.h kore-3.3.1/include/kore/tasks.h --- kore-2.0.0/include/kore/tasks.h 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/include/kore/tasks.h 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2014 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef __H_KORE_TASKS +#define __H_KORE_TASKS + +#include + +#define KORE_TASK_STATE_CREATED 1 +#define KORE_TASK_STATE_RUNNING 2 +#define KORE_TASK_STATE_FINISHED 3 +#define KORE_TASK_STATE_ABORT 4 + +#define KORE_TASK_THREADS 2 + +#if defined(__cplusplus) +extern "C" { +#endif + +#if !defined(KORE_NO_HTTP) +struct http_request; +#endif + +struct kore_task { + struct kore_event evt; + int state; + int result; + pthread_rwlock_t lock; + +#if !defined(KORE_NO_HTTP) + struct http_request *req; +#endif + + int fds[2]; + int (*entry)(struct kore_task *); + void (*cb)(struct kore_task *); + + struct kore_task_thread *thread; + + TAILQ_ENTRY(kore_task) list; + LIST_ENTRY(kore_task) rlist; +}; + +struct kore_task_thread { + u_int8_t idx; + pthread_t tid; + pthread_mutex_t lock; + pthread_cond_t cond; + TAILQ_HEAD(, kore_task) tasks; + + TAILQ_ENTRY(kore_task_thread) list; +}; + +void kore_task_init(void); +void kore_task_handle(void *, int); +void kore_task_run(struct kore_task *); +void kore_task_finish(struct kore_task *); +void kore_task_destroy(struct kore_task *); +int kore_task_finished(struct kore_task *); + +#if !defined(KORE_NO_HTTP) +void kore_task_bind_request(struct kore_task *, + struct http_request *); +#endif +void kore_task_bind_callback(struct kore_task *, + void (*cb)(struct kore_task *)); +void kore_task_create(struct kore_task *, + int (*entry)(struct kore_task *)); + +u_int32_t kore_task_channel_read(struct kore_task *, void *, u_int32_t); +void kore_task_channel_write(struct kore_task *, void *, u_int32_t); + +void kore_task_set_state(struct kore_task *, int); +void kore_task_set_result(struct kore_task *, int); + +int kore_task_state(struct kore_task *); +int kore_task_result(struct kore_task *); + +extern u_int16_t kore_task_threads; + +#if defined(__cplusplus) +} +#endif + +#endif diff -Nru kore-2.0.0/includes/http.h kore-3.3.1/includes/http.h --- kore-2.0.0/includes/http.h 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/includes/http.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,296 +0,0 @@ -/* - * Copyright (c) 2013 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#if !defined(KORE_NO_HTTP) - -#ifndef __H_HTTP_H -#define __H_HTTP_H - -#if defined(__cplusplus) -extern "C" { -#endif - -#define HTTP_KEEPALIVE_TIME 20 -#define HTTP_HSTS_ENABLE 31536000 -#define HTTP_HEADER_MAX_LEN 4096 -#define HTTP_BODY_MAX_LEN 1024000 -#define HTTP_URI_LEN 2000 -#define HTTP_USERAGENT_LEN 256 -#define HTTP_REQ_HEADER_MAX 25 -#define HTTP_MAX_QUERY_ARGS 20 -#define HTTP_MAX_COOKIES 10 -#define HTTP_REQUEST_LIMIT 1000 -#define HTTP_BODY_DISK_PATH "tmp_files" -#define HTTP_BODY_DISK_OFFLOAD 0 -#define HTTP_BODY_PATH_MAX 256 -#define HTTP_BOUNDARY_MAX 80 - -#define HTTP_ARG_TYPE_RAW 0 -#define HTTP_ARG_TYPE_BYTE 1 -#define HTTP_ARG_TYPE_INT16 2 -#define HTTP_ARG_TYPE_UINT16 3 -#define HTTP_ARG_TYPE_INT32 4 -#define HTTP_ARG_TYPE_UINT32 5 -#define HTTP_ARG_TYPE_STRING 6 -#define HTTP_ARG_TYPE_INT64 7 -#define HTTP_ARG_TYPE_UINT64 8 - -#define HTTP_STATE_ERROR 0 -#define HTTP_STATE_CONTINUE 1 -#define HTTP_STATE_COMPLETE 2 -#define HTTP_STATE_RETRY 3 - -struct http_header { - char *header; - char *value; - - TAILQ_ENTRY(http_header) list; -}; - -struct http_arg { - char *name; - char *s_value; - - TAILQ_ENTRY(http_arg) list; -}; - -#define COPY_ARG_TYPE(v, t) \ - do { \ - *(t *)nout = v; \ - } while (0) - -#define COPY_ARG_INT64(type, sign) \ - do { \ - int err; \ - type nval; \ - nval = (type)kore_strtonum64(q->s_value, sign, &err); \ - if (err != KORE_RESULT_OK) \ - return (KORE_RESULT_ERROR); \ - COPY_ARG_TYPE(nval, type); \ - } while (0) - -#define COPY_ARG_INT(min, max, type) \ - do { \ - int err; \ - int64_t nval; \ - nval = kore_strtonum(q->s_value, 10, min, max, &err); \ - if (err != KORE_RESULT_OK) \ - return (KORE_RESULT_ERROR); \ - COPY_ARG_TYPE(nval, type); \ - } while (0) - -#define COPY_AS_INTTYPE_64(type, sign) \ - do { \ - if (nout == NULL) \ - return (KORE_RESULT_ERROR); \ - COPY_ARG_INT64(type, sign); \ - } while (0) - -#define COPY_AS_INTTYPE(min, max, type) \ - do { \ - if (nout == NULL) \ - return (KORE_RESULT_ERROR); \ - COPY_ARG_INT(min, max, type); \ - } while (0) - -#define http_argument_type(r, n, so, no, t) \ - http_argument_get(r, n, so, no, t) - -#define http_argument_get_string(r, n, o) \ - http_argument_type(r, n, (void **)o, NULL, HTTP_ARG_TYPE_STRING) - -#define http_argument_get_byte(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_BYTE) - -#define http_argument_get_uint16(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT16) - -#define http_argument_get_int16(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT16) - -#define http_argument_get_uint32(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT32) - -#define http_argument_get_int32(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT32) - -#define http_argument_get_uint64(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT64) - -#define http_argument_get_int64(r, n, o) \ - http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT64) - - -struct http_file { - char *name; - char *filename; - size_t position; - size_t offset; - size_t length; - struct http_request *req; - TAILQ_ENTRY(http_file) list; -}; - -#define HTTP_METHOD_GET 0 -#define HTTP_METHOD_POST 1 -#define HTTP_METHOD_PUT 2 -#define HTTP_METHOD_DELETE 3 -#define HTTP_METHOD_HEAD 4 - -#define HTTP_REQUEST_COMPLETE 0x0001 -#define HTTP_REQUEST_DELETE 0x0002 -#define HTTP_REQUEST_SLEEPING 0x0004 -#define HTTP_REQUEST_PGSQL_QUEUE 0x0010 -#define HTTP_REQUEST_EXPECT_BODY 0x0020 -#define HTTP_REQUEST_RETAIN_EXTRA 0x0040 -#define HTTP_REQUEST_NO_CONTENT_LENGTH 0x0080 -#define HTTP_REQUEST_AUTHED 0x0100 - -struct kore_task; - -struct http_request { - u_int8_t method; - u_int8_t fsm_state; - u_int16_t flags; - u_int16_t status; - u_int64_t start; - u_int64_t end; - u_int64_t total; - char *host; - char *path; - char *agent; - struct connection *owner; - struct kore_buf *http_body; - int http_body_fd; - char *http_body_path; - size_t http_body_length; - size_t http_body_offset; - size_t content_length; - void *hdlr_extra; - char *query_string; - struct kore_module_handle *hdlr; - - LIST_HEAD(, kore_task) tasks; - LIST_HEAD(, kore_pgsql) pgsqls; - - TAILQ_HEAD(, http_header) req_headers; - TAILQ_HEAD(, http_header) resp_headers; - TAILQ_HEAD(, http_arg) arguments; - TAILQ_HEAD(, http_file) files; - TAILQ_ENTRY(http_request) list; - TAILQ_ENTRY(http_request) olist; -}; - -struct http_state { - const char *name; - int (*cb)(struct http_request *); -}; - -extern int http_request_count; -extern u_int16_t http_header_max; -extern u_int64_t http_body_max; -extern u_int64_t http_hsts_enable; -extern u_int16_t http_keepalive_time; -extern u_int32_t http_request_limit; -extern u_int64_t http_body_disk_offload; -extern char *http_body_disk_path; - -void kore_accesslog(struct http_request *); - -void http_init(void); -void http_cleanup(void); -void http_process(void); -const char *http_status_text(int); -const char *http_method_text(int); -time_t http_date_to_time(char *); -void http_request_free(struct http_request *); -void http_request_sleep(struct http_request *); -void http_request_wakeup(struct http_request *); -void http_process_request(struct http_request *); -ssize_t http_body_read(struct http_request *, void *, size_t); -void http_response(struct http_request *, int, const void *, size_t); -void http_response_stream(struct http_request *, int, void *, - size_t, int (*cb)(struct netbuf *), void *); -int http_request_header(struct http_request *, - const char *, char **); -void http_response_header(struct http_request *, - const char *, const char *); -int http_request_new(struct connection *, const char *, - const char *, const char *, const char *, - struct http_request **); -int http_state_run(struct http_state *, u_int8_t, - struct http_request *); - -int http_argument_urldecode(char *); -int http_header_recv(struct netbuf *); -void http_populate_get(struct http_request *); -void http_populate_post(struct http_request *); -void http_populate_multipart_form(struct http_request *); -int http_argument_get(struct http_request *, - const char *, void **, void *, int); - -void http_file_rewind(struct http_file *); -ssize_t http_file_read(struct http_file *, void *, size_t); -struct http_file *http_file_lookup(struct http_request *, const char *); - -enum http_status_code { - HTTP_STATUS_CONTINUE = 100, - HTTP_STATUS_SWITCHING_PROTOCOLS = 101, - HTTP_STATUS_OK = 200, - HTTP_STATUS_CREATED = 201, - HTTP_STATUS_ACCEPTED = 202, - HTTP_STATUS_NON_AUTHORITATIVE = 203, - HTTP_STATUS_NO_CONTENT = 204, - HTTP_STATUS_RESET_CONTENT = 205, - HTTP_STATUS_PARTIAL_CONTENT = 206, - HTTP_STATUS_MULTIPLE_CHOICES = 300, - HTTP_STATUS_MOVED_PERMANENTLY = 301, - HTTP_STATUS_FOUND = 302, - HTTP_STATUS_SEE_OTHER = 303, - HTTP_STATUS_NOT_MODIFIED = 304, - HTTP_STATUS_USE_PROXY = 305, - HTTP_STATUS_TEMPORARY_REDIRECT = 307, - HTTP_STATUS_BAD_REQUEST = 400, - HTTP_STATUS_UNAUTHORIZED = 401, - HTTP_STATUS_PAYMENT_REQUIRED = 402, - HTTP_STATUS_FORBIDDEN = 403, - HTTP_STATUS_NOT_FOUND = 404, - HTTP_STATUS_METHOD_NOT_ALLOWED = 405, - HTTP_STATUS_NOT_ACCEPTABLE = 406, - HTTP_STATUS_PROXY_AUTH_REQUIRED = 407, - HTTP_STATUS_REQUEST_TIMEOUT = 408, - HTTP_STATUS_CONFLICT = 409, - HTTP_STATUS_GONE = 410, - HTTP_STATUS_LENGTH_REQUIRED = 411, - HTTP_STATUS_PRECONDITION_FAILED = 412, - HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE = 413, - HTTP_STATUS_REQUEST_URI_TOO_LARGE = 414, - HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE = 415, - HTTP_STATUS_REQUEST_RANGE_INVALID = 416, - HTTP_STATUS_EXPECTATION_FAILED = 417, - HTTP_STATUS_INTERNAL_ERROR = 500, - HTTP_STATUS_NOT_IMPLEMENTED = 501, - HTTP_STATUS_BAD_GATEWAY = 502, - HTTP_STATUS_SERVICE_UNAVAILABLE = 503, - HTTP_STATUS_GATEWAY_TIMEOUT = 504, - HTTP_STATUS_BAD_VERSION = 505 -}; -#if defined(__cplusplus) -} -#endif -#endif /* !__H_HTTP_H */ - -#endif /* ! KORE_NO_HTTP */ diff -Nru kore-2.0.0/includes/jsonrpc.h kore-3.3.1/includes/jsonrpc.h --- kore-2.0.0/includes/jsonrpc.h 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/includes/jsonrpc.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2016 Raphaël Monrouzeau - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#if !defined(KORE_NO_HTTP) - -#ifndef __H_JSONRPC_H -#define __H_JSONRPC_H - -#if defined(__cplusplus) -extern "C" { -#endif - -/* JSON RPC request handling log entry. */ -struct jsonrpc_log -{ - char *msg; - struct jsonrpc_log *next, *prev; - int lvl; -}; - -/* JSON RPC request. */ -struct jsonrpc_request -{ - struct jsonrpc_log log; - struct kore_buf buf; - struct http_request *http; - yajl_gen gen; - yajl_val json; - yajl_val id; - char *method; - yajl_val params; - unsigned int flags; - int log_levels; -}; - -#define YAJL_GEN_CONST_STRING(CTX, STR) \ - yajl_gen_string((CTX), (unsigned char *)(STR), sizeof (STR) - 1) - -#define YAJL_GEN_CONST_NUMBER(CTX, STR) \ - yajl_gen_number((CTX), (unsigned char *)(STR), sizeof (STR) - 1) - -#define YAJL_GEN_KO(OPERATION) \ - ((OPERATION) != yajl_gen_status_ok) - -enum jsonrpc_error_code -{ -#define JSONRPC_PARSE_ERROR_MSG "Parse error" - JSONRPC_PARSE_ERROR = -32700, -#define JSONRPC_INVALID_REQUEST_MSG "Invalid Request" - JSONRPC_INVALID_REQUEST = -32600, -#define JSONRPC_METHOD_NOT_FOUND_MSG "Method not found" - JSONRPC_METHOD_NOT_FOUND = -32601, -#define JSONRPC_INVALID_PARAMS_MSG "Invalid params" - JSONRPC_INVALID_PARAMS = -32602, -#define JSONRPC_INTERNAL_ERROR_MSG "Internal error" - JSONRPC_INTERNAL_ERROR = -32603, -#define JSONRPC_SERVER_ERROR_MSG "Server error" - JSONRPC_SERVER_ERROR = -32000, -#define JSONRPC_LIMIT_REACHED_MSG "Limit reached" - JSONRPC_LIMIT_REACHED = -31997 -}; - -void jsonrpc_log(struct jsonrpc_request *, int, const char *, ...); -int jsonrpc_read_request(struct http_request *, struct jsonrpc_request *); -void jsonrpc_destroy_request(struct jsonrpc_request *); -int jsonrpc_error(struct jsonrpc_request *, int, const char *); -int jsonrpc_result(struct jsonrpc_request *, - int (*)(struct jsonrpc_request *, void *), void *); -#if defined(__cplusplus) -} -#endif -#endif /* !__H_JSONRPC_H */ - -#endif /* ! KORE_NO_HTTP */ diff -Nru kore-2.0.0/includes/kore.h kore-3.3.1/includes/kore.h --- kore-2.0.0/includes/kore.h 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/includes/kore.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,649 +0,0 @@ -/* - * Copyright (c) 2013-2016 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#ifndef __H_KORE_H -#define __H_KORE_H - -#if defined(__APPLE__) -#define daemon portability_is_king -#endif - -#include -#include -#include - -#include -#include - -#if !defined(KORE_NO_TLS) -#include -#include -#include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__cplusplus) -extern "C" { -#endif - -#if defined(__APPLE__) -#undef daemon -extern int daemon(int, int); -#endif - -#define KORE_RESULT_ERROR 0 -#define KORE_RESULT_OK 1 -#define KORE_RESULT_RETRY 2 - -#define KORE_VERSION_MAJOR 2 -#define KORE_VERSION_MINOR 0 -#define KORE_VERSION_PATCH 0 -#define KORE_VERSION_STATE "release" - -#define KORE_TLS_VERSION_1_2 0 -#define KORE_TLS_VERSION_1_0 1 -#define KORE_TLS_VERSION_BOTH 2 - -#define errno_s strerror(errno) -#define ssl_errno_s ERR_error_string(ERR_get_error(), NULL) - -#define KORE_DOMAINNAME_LEN 255 -#define KORE_PIDFILE_DEFAULT "kore.pid" -#define KORE_DEFAULT_CIPHER_LIST "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!3DES:!MD5:!PSK:!kRSA:!kDSA" - -#if defined(KORE_DEBUG) -#define kore_debug(...) \ - if (kore_debug) \ - kore_debug_internal(__FILE__, __LINE__, __VA_ARGS__) -#else -#define kore_debug(...) -#endif - -#define NETBUF_RECV 0 -#define NETBUF_SEND 1 -#define NETBUF_SEND_PAYLOAD_MAX 8192 - -#define NETBUF_LAST_CHAIN 0 -#define NETBUF_BEFORE_CHAIN 1 - -#define NETBUF_CALL_CB_ALWAYS 0x01 -#define NETBUF_FORCE_REMOVE 0x02 -#define NETBUF_MUST_RESEND 0x04 -#define NETBUF_IS_STREAM 0x10 - -#define X509_GET_CN(c, o, l) \ - X509_NAME_get_text_by_NID(X509_get_subject_name(c), \ - NID_commonName, o, l) - -#define X509_CN_LENGTH (ub_common_name + 1) - -/* XXX hackish. */ -#if !defined(KORE_NO_HTTP) -struct http_request; -#endif - -struct netbuf { - u_int8_t *buf; - size_t s_off; - size_t b_len; - size_t m_len; - u_int8_t type; - u_int8_t flags; - - void *owner; - - void *extra; - int (*cb)(struct netbuf *); - - TAILQ_ENTRY(netbuf) list; -}; - -TAILQ_HEAD(netbuf_head, netbuf); - -#define KORE_TYPE_LISTENER 1 -#define KORE_TYPE_CONNECTION 2 -#define KORE_TYPE_PGSQL_CONN 3 -#define KORE_TYPE_TASK 4 - -#define CONN_STATE_UNKNOWN 0 -#define CONN_STATE_SSL_SHAKE 1 -#define CONN_STATE_ESTABLISHED 2 -#define CONN_STATE_DISCONNECTING 3 - -#define CONN_PROTO_UNKNOWN 0 -#define CONN_PROTO_HTTP 1 -#define CONN_PROTO_WEBSOCKET 2 -#define CONN_PROTO_MSG 3 - -#define CONN_READ_POSSIBLE 0x01 -#define CONN_WRITE_POSSIBLE 0x02 -#define CONN_WRITE_BLOCK 0x04 -#define CONN_IDLE_TIMER_ACT 0x10 -#define CONN_READ_BLOCK 0x20 -#define CONN_CLOSE_EMPTY 0x40 - -#define KORE_IDLE_TIMER_MAX 20000 - -#define WEBSOCKET_OP_CONT 0x00 -#define WEBSOCKET_OP_TEXT 0x01 -#define WEBSOCKET_OP_BINARY 0x02 -#define WEBSOCKET_OP_CLOSE 0x08 -#define WEBSOCKET_OP_PING 0x09 -#define WEBSOCKET_OP_PONG 0x10 - -#define WEBSOCKET_BROADCAST_LOCAL 1 -#define WEBSOCKET_BROADCAST_GLOBAL 2 - -#define KORE_TIMER_ONESHOT 0x01 - -#define KORE_CONNECTION_PRUNE_DISCONNECT 0 -#define KORE_CONNECTION_PRUNE_ALL 1 - -struct connection { - u_int8_t type; - int fd; - u_int8_t state; - u_int8_t proto; - void *owner; -#if !defined(KORE_NO_TLS) - X509 *cert; - SSL *ssl; - int tls_reneg; -#endif - u_int8_t flags; - void *hdlr_extra; - - int (*handle)(struct connection *); - void (*disconnect)(struct connection *); - int (*read)(struct connection *, int *); - int (*write)(struct connection *, int, int *); - - u_int8_t addrtype; - union { - struct sockaddr_in ipv4; - struct sockaddr_in6 ipv6; - } addr; - - struct { - u_int64_t length; - u_int64_t start; - } idle_timer; - - struct netbuf_head send_queue; - struct netbuf *snb; - struct netbuf *rnb; - -#if !defined(KORE_NO_HTTP) - void *wscbs; - TAILQ_HEAD(, http_request) http_requests; -#endif - - TAILQ_ENTRY(connection) list; -}; - -TAILQ_HEAD(connection_list, connection); -extern struct connection_list connections; -extern struct connection_list disconnected; - -struct listener { - u_int8_t type; - u_int8_t addrtype; - int fd; - void (*connect)(struct connection *); - - union { - struct sockaddr_in ipv4; - struct sockaddr_in6 ipv6; - } addr; - - LIST_ENTRY(listener) list; -}; - -LIST_HEAD(listener_head, listener); - -#if !defined(KORE_NO_HTTP) - -struct kore_handler_params { - char *name; - u_int8_t method; - struct kore_validator *validator; - - TAILQ_ENTRY(kore_handler_params) list; -}; - -#define KORE_AUTH_TYPE_COOKIE 1 -#define KORE_AUTH_TYPE_HEADER 2 -#define KORE_AUTH_TYPE_REQUEST 3 - -struct kore_auth { - u_int8_t type; - char *name; - char *value; - char *redirect; - struct kore_validator *validator; - - TAILQ_ENTRY(kore_auth) list; -}; - -#define HANDLER_TYPE_STATIC 1 -#define HANDLER_TYPE_DYNAMIC 2 - -#endif - -#define KORE_MODULE_LOAD 1 -#define KORE_MODULE_UNLOAD 2 - -struct kore_module { - void *handle; - char *path; - char *onload; - int (*ocb)(int); - - time_t mtime; - - TAILQ_ENTRY(kore_module) list; -}; - -struct kore_module_handle { - char *path; - char *func; - void *addr; - int type; - int errors; - regex_t rctx; - struct kore_domain *dom; -#if !defined(KORE_NO_HTTP) - struct kore_auth *auth; - TAILQ_HEAD(, kore_handler_params) params; -#endif - TAILQ_ENTRY(kore_module_handle) list; -}; - -struct kore_worker { - u_int8_t id; - u_int8_t cpu; - pid_t pid; - int pipe[2]; - struct connection *msg[2]; - u_int8_t has_lock; - struct kore_module_handle *active_hdlr; -}; - -struct kore_domain { - char *domain; - int accesslog; -#if !defined(KORE_NO_TLS) - char *cafile; - char *crlfile; - char *certfile; - char *certkey; - SSL_CTX *ssl_ctx; -#endif - TAILQ_HEAD(, kore_module_handle) handlers; - TAILQ_ENTRY(kore_domain) list; -}; - -TAILQ_HEAD(kore_domain_h, kore_domain); - -#if !defined(KORE_NO_HTTP) - -#define KORE_VALIDATOR_TYPE_REGEX 1 -#define KORE_VALIDATOR_TYPE_FUNCTION 2 - -struct kore_validator { - u_int8_t type; - char *name; - char *arg; - regex_t rctx; - int (*func)(struct http_request *, char *); - - TAILQ_ENTRY(kore_validator) list; -}; -#endif - -#define KORE_BUF_OWNER_API 0x0001 - -struct kore_buf { - u_int8_t *data; - int flags; - size_t length; - size_t offset; -}; - -struct kore_pool_region { - void *start; - size_t length; - LIST_ENTRY(kore_pool_region) list; -}; - -struct kore_pool_entry { - u_int8_t state; - struct kore_pool_region *region; - LIST_ENTRY(kore_pool_entry) list; -}; - -struct kore_pool { - size_t elen; - size_t slen; - size_t elms; - size_t inuse; - volatile int lock; - char *name; - - LIST_HEAD(, kore_pool_region) regions; - LIST_HEAD(, kore_pool_entry) freelist; -}; - -struct kore_wscbs { - void (*connect)(struct connection *); - void (*message)(struct connection *, u_int8_t, - void *, size_t); - void (*disconnect)(struct connection *); -}; - -struct kore_timer { - u_int64_t nextrun; - u_int64_t interval; - int flags; - void *arg; - void (*cb)(void *, u_int64_t); - - TAILQ_ENTRY(kore_timer) list; -}; - -#define KORE_WORKER_KEYMGR 0 - -/* Reserved message ids, registered on workers. */ -#define KORE_MSG_ACCESSLOG 1 -#define KORE_MSG_WEBSOCKET 2 -#define KORE_MSG_KEYMGR_REQ 3 -#define KORE_MSG_KEYMGR_RESP 4 - -/* Predefined message targets. */ -#define KORE_MSG_PARENT 1000 -#define KORE_MSG_WORKER_ALL 1001 - -struct kore_msg { - u_int8_t id; - u_int16_t src; - u_int16_t dst; - u_int32_t length; -}; - -#if !defined(KORE_NO_TLS) -struct kore_keyreq { - int padding; - char domain[KORE_DOMAINNAME_LEN]; - u_int8_t domain_len; - u_int16_t data_len; - u_int8_t data[]; -}; -#endif - -#if !defined(KORE_SINGLE_BINARY) -extern char *config_file; -#endif - -extern pid_t kore_pid; -extern int foreground; -extern int kore_debug; -extern int skip_chroot; -extern char *chroot_path; -extern int skip_runas; -extern char *runas_user; -extern char *kore_pidfile; -extern char *kore_tls_cipher_list; -extern int tls_version; - -#if !defined(KORE_NO_TLS) -extern DH *tls_dhparam; -#endif - -extern u_int8_t nlisteners; -extern u_int16_t cpu_count; -extern u_int8_t worker_count; -extern u_int8_t worker_set_affinity; -extern u_int32_t worker_rlimit_nofiles; -extern u_int32_t worker_max_connections; -extern u_int32_t worker_active_connections; -extern u_int32_t worker_accept_threshold; -extern u_int64_t kore_websocket_maxframe; -extern u_int64_t kore_websocket_timeout; -extern u_int32_t kore_socket_backlog; - -extern struct listener_head listeners; -extern struct kore_worker *worker; -extern struct kore_domain_h domains; -extern struct kore_domain *primary_dom; -extern struct kore_pool nb_pool; - -void kore_cli_usage(int); -int kore_cli_main(int, char **); - -void kore_signal(int); -void kore_worker_wait(int); -void kore_worker_init(void); -void kore_worker_shutdown(void); -void kore_worker_privdrop(void); -void kore_worker_dispatch_signal(int); -void kore_worker_spawn(u_int16_t, u_int16_t); -void kore_worker_entry(struct kore_worker *); - -struct kore_worker *kore_worker_data(u_int8_t); - -void kore_platform_init(void); -void kore_platform_event_init(void); -void kore_platform_event_cleanup(void); -void kore_platform_proctitle(char *); -void kore_platform_disable_read(int); -void kore_platform_enable_accept(void); -void kore_platform_disable_accept(void); -int kore_platform_event_wait(u_int64_t); -void kore_platform_event_all(int, void *); -void kore_platform_schedule_read(int, void *); -void kore_platform_schedule_write(int, void *); -void kore_platform_event_schedule(int, int, int, void *); -void kore_platform_worker_setcpu(struct kore_worker *); - -void kore_accesslog_init(void); -void kore_accesslog_worker_init(void); -int kore_accesslog_write(const void *, u_int32_t); - -#if !defined(KORE_NO_HTTP) -int kore_auth_run(struct http_request *, struct kore_auth *); -void kore_auth_init(void); -int kore_auth_new(const char *); -struct kore_auth *kore_auth_lookup(const char *); -#endif - -void kore_timer_init(void); -u_int64_t kore_timer_run(u_int64_t); -void kore_timer_remove(struct kore_timer *); -struct kore_timer *kore_timer_add(void (*cb)(void *, u_int64_t), - u_int64_t, void *, int); - -void kore_listener_cleanup(void); -int kore_server_bind(const char *, const char *, const char *); -#if !defined(KORE_NO_TLS) -int kore_tls_sni_cb(SSL *, int *, void *); -void kore_tls_info_callback(const SSL *, int, int); -#endif - -void kore_connection_init(void); -void kore_connection_cleanup(void); -void kore_connection_prune(int); -struct connection *kore_connection_new(void *); -void kore_connection_check_timeout(void); -int kore_connection_nonblock(int, int); -int kore_connection_handle(struct connection *); -void kore_connection_remove(struct connection *); -void kore_connection_disconnect(struct connection *); -void kore_connection_start_idletimer(struct connection *); -void kore_connection_stop_idletimer(struct connection *); -void kore_connection_check_idletimer(u_int64_t, - struct connection *); -int kore_connection_accept(struct listener *, - struct connection **); - -u_int64_t kore_time_ms(void); -void kore_log_init(void); - -void *kore_malloc(size_t); -void kore_parse_config(void); -void *kore_calloc(size_t, size_t); -void *kore_realloc(void *, size_t); -void kore_free(void *); -void kore_mem_init(void); - -void *kore_pool_get(struct kore_pool *); -void kore_pool_put(struct kore_pool *, void *); -void kore_pool_init(struct kore_pool *, const char *, - size_t, size_t); -void kore_pool_cleanup(struct kore_pool *); - -time_t kore_date_to_time(char *); -char *kore_time_to_date(time_t); -char *kore_strdup(const char *); -void kore_log(int, const char *, ...); -u_int64_t kore_strtonum64(const char *, int, int *); -size_t kore_strlcpy(char *, const char *, const size_t); -void kore_server_disconnect(struct connection *); -int kore_split_string(char *, char *, char **, size_t); -void kore_strip_chars(char *, const char, char **); -int kore_snprintf(char *, size_t, int *, const char *, ...); -long long kore_strtonum(const char *, int, long long, long long, int *); -int kore_base64_encode(u_int8_t *, size_t, char **); -int kore_base64_decode(char *, u_int8_t **, size_t *); -void *kore_mem_find(void *, size_t, void *, size_t); -char *kore_text_trim(char *, size_t); -char *kore_read_line(FILE *, char *, size_t); - -#if !defined(KORE_NO_HTTP) -void kore_websocket_handshake(struct http_request *, - struct kore_wscbs *); -void kore_websocket_send(struct connection *, - u_int8_t, const void *, size_t); -void kore_websocket_broadcast(struct connection *, - u_int8_t, const void *, size_t, int); -#endif - -void kore_msg_init(void); -void kore_msg_worker_init(void); -void kore_msg_parent_init(void); -void kore_msg_parent_add(struct kore_worker *); -void kore_msg_parent_remove(struct kore_worker *); -void kore_msg_send(u_int16_t, u_int8_t, const void *, u_int32_t); -int kore_msg_register(u_int8_t, - void (*cb)(struct kore_msg *, const void *)); - -void kore_domain_init(void); -void kore_domain_cleanup(void); -int kore_domain_new(char *); -void kore_domain_free(struct kore_domain *); -void kore_module_init(void); -void kore_module_cleanup(void); -void kore_module_reload(int); -void kore_module_onload(void); -int kore_module_loaded(void); -void kore_domain_closelogs(void); -void *kore_module_getsym(const char *); -void kore_domain_load_crl(void); -void kore_domain_keymgr_init(void); -void kore_module_load(const char *, const char *); -void kore_domain_sslstart(struct kore_domain *); -void kore_domain_callback(void (*cb)(struct kore_domain *)); -int kore_module_handler_new(const char *, const char *, - const char *, const char *, int); -void kore_module_handler_free(struct kore_module_handle *); - -struct kore_domain *kore_domain_lookup(const char *); -struct kore_module_handle *kore_module_handler_find(const char *, - const char *); - -#if !defined(KORE_NO_HTTP) -void kore_validator_init(void); -void kore_validator_reload(void); -int kore_validator_add(const char *, u_int8_t, const char *); -int kore_validator_run(struct http_request *, const char *, char *); -int kore_validator_check(struct http_request *, - struct kore_validator *, void *); -struct kore_validator *kore_validator_lookup(const char *); -#endif - -void fatal(const char *, ...) __attribute__((noreturn)); -void kore_debug_internal(char *, int, const char *, ...); - -u_int16_t net_read16(u_int8_t *); -u_int32_t net_read32(u_int8_t *); -u_int64_t net_read64(u_int8_t *); -void net_write16(u_int8_t *, u_int16_t); -void net_write32(u_int8_t *, u_int32_t); -void net_write64(u_int8_t *, u_int64_t); - -void net_init(void); -void net_cleanup(void); -int net_send(struct connection *); -int net_send_flush(struct connection *); -int net_recv_flush(struct connection *); -int net_read(struct connection *, int *); -int net_read_ssl(struct connection *, int *); -int net_write(struct connection *, int, int *); -int net_write_ssl(struct connection *, int, int *); -void net_recv_reset(struct connection *, size_t, - int (*cb)(struct netbuf *)); -void net_remove_netbuf(struct netbuf_head *, struct netbuf *); -void net_recv_queue(struct connection *, size_t, int, - int (*cb)(struct netbuf *)); -void net_recv_expand(struct connection *c, size_t, - int (*cb)(struct netbuf *)); -void net_send_queue(struct connection *, const void *, size_t); -void net_send_stream(struct connection *, void *, - size_t, int (*cb)(struct netbuf *), struct netbuf **); - -void kore_buf_free(struct kore_buf *); -struct kore_buf *kore_buf_alloc(size_t); -void kore_buf_init(struct kore_buf *, size_t); -void kore_buf_append(struct kore_buf *, const void *, size_t); -u_int8_t *kore_buf_release(struct kore_buf *, size_t *); -void kore_buf_reset(struct kore_buf *); -void kore_buf_cleanup(struct kore_buf *); - -char *kore_buf_stringify(struct kore_buf *, size_t *); -void kore_buf_appendf(struct kore_buf *, const char *, ...); -void kore_buf_appendv(struct kore_buf *, const char *, va_list); -void kore_buf_replace_string(struct kore_buf *, char *, void *, size_t); - -void kore_keymgr_run(void); -void kore_keymgr_cleanup(void); - -#if defined(KORE_SINGLE_BINARY) -void kore_onload(void); -#endif - -#if defined(__cplusplus) -} -#endif - -#endif /* !__H_KORE_H */ diff -Nru kore-2.0.0/includes/pgsql.h kore-3.3.1/includes/pgsql.h --- kore-2.0.0/includes/pgsql.h 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/includes/pgsql.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2014 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#ifndef _H_KORE_PGSQL -#define _H_KORE_PGSQL - -#include - -#define KORE_PGSQL_FORMAT_TEXT 0 -#define KORE_PGSQL_FORMAT_BINARY 1 - -#define KORE_PGSQL_SYNC 0x0001 -#define KORE_PGSQL_ASYNC 0x0002 - -#if defined(__cplusplus) -extern "C" { -#endif - -struct pgsql_conn { - u_int8_t type; - u_int8_t flags; - char *name; - - PGconn *db; - struct pgsql_job *job; - TAILQ_ENTRY(pgsql_conn) list; -}; - -struct pgsql_db { - char *name; - char *conn_string; - - LIST_ENTRY(pgsql_db) rlist; -}; - -struct kore_pgsql { - u_int8_t state; - int flags; - char *error; - PGresult *result; - struct pgsql_conn *conn; - - LIST_ENTRY(kore_pgsql) rlist; -}; - -extern u_int16_t pgsql_conn_max; - -void kore_pgsql_init(void); -int kore_pgsql_query_init(struct kore_pgsql *, struct http_request *, - const char *, int); -void kore_pgsql_handle(void *, int); -void kore_pgsql_cleanup(struct kore_pgsql *); -void kore_pgsql_continue(struct http_request *, struct kore_pgsql *); -int kore_pgsql_query(struct kore_pgsql *, const char *); -int kore_pgsql_query_params(struct kore_pgsql *, - const char *, int, u_int8_t, ...); -int kore_pgsql_v_query_params(struct kore_pgsql *, - const char *, int, u_int8_t, va_list); -int kore_pgsql_register(const char *, const char *); -int kore_pgsql_ntuples(struct kore_pgsql *); -void kore_pgsql_logerror(struct kore_pgsql *); -void kore_pgsql_queue_remove(struct http_request *); -char *kore_pgsql_getvalue(struct kore_pgsql *, int, int); -int kore_pgsql_getlength(struct kore_pgsql *, int, int); - -#if defined(__cplusplus) -} -#endif - -#define KORE_PGSQL_STATE_INIT 1 -#define KORE_PGSQL_STATE_WAIT 2 -#define KORE_PGSQL_STATE_RESULT 3 -#define KORE_PGSQL_STATE_ERROR 4 -#define KORE_PGSQL_STATE_DONE 5 -#define KORE_PGSQL_STATE_COMPLETE 6 - -#endif diff -Nru kore-2.0.0/includes/tasks.h kore-3.3.1/includes/tasks.h --- kore-2.0.0/includes/tasks.h 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/includes/tasks.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2014 Joris Vink - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#ifndef __H_KORE_TASKS -#define __H_KORE_TASKS - -#include - -#define KORE_TASK_STATE_CREATED 1 -#define KORE_TASK_STATE_RUNNING 2 -#define KORE_TASK_STATE_FINISHED 3 -#define KORE_TASK_STATE_ABORT 4 - -#define KORE_TASK_THREADS 2 - -#if defined(__cplusplus) -extern "C" { -#endif - -#if !defined(KORE_NO_HTTP) -struct http_request; -#endif - -struct kore_task { - u_int8_t type; - int state; - int result; - pthread_rwlock_t lock; - -#if !defined(KORE_NO_HTTP) - struct http_request *req; -#endif - - int fds[2]; - int (*entry)(struct kore_task *); - void (*cb)(struct kore_task *); - - struct kore_task_thread *thread; - - TAILQ_ENTRY(kore_task) list; - LIST_ENTRY(kore_task) rlist; -}; - -struct kore_task_thread { - u_int8_t idx; - pthread_t tid; - pthread_mutex_t lock; - pthread_cond_t cond; - TAILQ_HEAD(, kore_task) tasks; - - TAILQ_ENTRY(kore_task_thread) list; -}; - -void kore_task_init(void); -void kore_task_run(struct kore_task *); -void kore_task_finish(struct kore_task *); -void kore_task_destroy(struct kore_task *); -int kore_task_finished(struct kore_task *); -void kore_task_handle(struct kore_task *, int); - -#if !defined(KORE_NO_HTTP) -void kore_task_bind_request(struct kore_task *, - struct http_request *); -#endif -void kore_task_bind_callback(struct kore_task *, - void (*cb)(struct kore_task *)); -void kore_task_create(struct kore_task *, - int (*entry)(struct kore_task *)); - -u_int32_t kore_task_channel_read(struct kore_task *, void *, u_int32_t); -void kore_task_channel_write(struct kore_task *, void *, u_int32_t); - -void kore_task_set_state(struct kore_task *, int); -void kore_task_set_result(struct kore_task *, int); - -int kore_task_state(struct kore_task *); -int kore_task_result(struct kore_task *); - -extern u_int16_t kore_task_threads; - -#if defined(__cplusplus) -} -#endif - -#endif diff -Nru kore-2.0.0/kodev/Makefile kore-3.3.1/kodev/Makefile --- kore-2.0.0/kodev/Makefile 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/kodev/Makefile 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,53 @@ +# kodev Makefile + +CC?=cc +PREFIX?=/usr/local +OBJDIR?=obj +KODEV=kodev +INSTALL_DIR=$(PREFIX)/bin + +S_SRC= ../src/cli.c + +CFLAGS+=-Wall -Werror -Wstrict-prototypes -Wmissing-prototypes +CFLAGS+=-Wmissing-declarations -Wshadow -Wpointer-arith -Wcast-qual +CFLAGS+=-Wsign-compare -Iincludes -std=c99 -pedantic +CFLAGS+=-DPREFIX='"$(PREFIX)"' +LDFLAGS=-lcrypto + +ifneq ("$(NOOPT)", "") + CFLAGS+=-O0 +else + CFLAGS+=-O2 +endif + +OSNAME=$(shell uname -s | sed -e 's/[-_].*//g' | tr A-Z a-z) +ifeq ("$(OSNAME)", "darwin") + CFLAGS+=-I/opt/local/include/ -I/usr/local/opt/openssl/include + LDFLAGS+=-L/opt/local/lib -L/usr/local/opt/openssl/lib +else ifeq ("$(OSNAME)", "linux") + CFLAGS+=-D_GNU_SOURCE=1 +endif + +S_OBJS= $(S_SRC:../src/%.c=$(OBJDIR)/%.o) + +$(KODEV): $(OBJDIR) $(S_OBJS) + $(CC) $(S_OBJS) $(LDFLAGS) -o $(KODEV) + +$(OBJDIR): + @mkdir -p $(OBJDIR) + +install: $(KODEV) + mkdir -p $(INSTALL_DIR) + install -m 555 $(KODEV) $(INSTALL_DIR)/$(KODEV) + +uninstall: + rm -f $(INSTALL_DIR)/$(KODEV) + +$(OBJDIR)/%.o: ../src/%.c + $(CC) $(CFLAGS) -c $< -o $@ + +clean: + find . -type f -name \*.o -exec rm {} \; + rm -rf $(KODEV) $(OBJDIR) + +.PHONY: all clean diff -Nru kore-2.0.0/LICENSE kore-3.3.1/LICENSE --- kore-2.0.0/LICENSE 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/LICENSE 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2018 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff -Nru kore-2.0.0/Makefile kore-3.3.1/Makefile --- kore-2.0.0/Makefile 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/Makefile 2019-06-03 13:29:24.000000000 +0000 @@ -1,24 +1,46 @@ # Kore Makefile -CC?=gcc +CC?=cc PREFIX?=/usr/local OBJDIR?=obj KORE=kore +KODEV=kodev/kodev +KORE_CRYPTO?=crypto INSTALL_DIR=$(PREFIX)/bin +MAN_DIR=$(PREFIX)/share/man +SHARE_DIR=$(PREFIX)/share/kore INCLUDE_DIR=$(PREFIX)/include/kore -S_SRC= src/kore.c src/buf.c src/cli.c src/config.c src/connection.c \ - src/domain.c src/mem.c src/msg.c src/module.c src/net.c \ - src/pool.c src/timer.c src/utils.c src/worker.c src/keymgr.c +VERSION=src/version.c + +S_SRC= src/kore.c src/buf.c src/config.c src/connection.c \ + src/domain.c src/filemap.c src/fileref.c src/mem.c src/msg.c \ + src/module.c src/net.c src/pool.c src/runtime.c src/timer.c \ + src/utils.c src/worker.c src/keymgr.c $(VERSION) + +FEATURES= +FEATURES_INC= CFLAGS+=-Wall -Werror -Wstrict-prototypes -Wmissing-prototypes CFLAGS+=-Wmissing-declarations -Wshadow -Wpointer-arith -Wcast-qual -CFLAGS+=-Wsign-compare -Iincludes -std=c99 -pedantic -CFLAGS+=-DPREFIX='"$(PREFIX)"' -LDFLAGS=-rdynamic -lssl -lcrypto +CFLAGS+=-Wsign-compare -Iinclude/kore -std=c99 -pedantic +CFLAGS+=-DPREFIX='"$(PREFIX)"' -fstack-protector-all + +ifneq ("$(OPENSSL_PATH)", "") +CFLAGS+=-I$(OPENSSL_PATH)/include +LDFLAGS=-rdynamic -L$(OPENSSL_PATH) -lssl -l$(KORE_CRYPTO) +else +LDFLAGS=-rdynamic -lssl -l$(KORE_CRYPTO) +endif + +ifneq ("$(KORE_SINGLE_BINARY)", "") + CFLAGS+=-DKORE_SINGLE_BINARY + FEATURES+=-DKORE_SINGLE_BINARY +endif ifneq ("$(DEBUG)", "") CFLAGS+=-DKORE_DEBUG -g + FEATURES+=-DKORE_DEBUG NOOPT=1 endif @@ -28,8 +50,13 @@ CFLAGS+=-O2 endif +ifneq ("$(NOSENDFILE)", "") + CFLAGS+=-DKORE_NO_SENDFILE +endif + ifneq ("$(NOHTTP)", "") CFLAGS+=-DKORE_NO_HTTP + FEATURES+=-DKORE_NO_HTTP else S_SRC+= src/auth.c src/accesslog.c src/http.c \ src/validator.c src/websocket.c @@ -37,10 +64,11 @@ ifneq ("$(NOTLS)", "") CFLAGS+=-DKORE_NO_TLS + FEATURES+=-DKORE_NO_TLS ifneq ("$(NOHTTP)", "") LDFLAGS=-rdynamic else - LDFLAGS=-rdynamic -lcrypto + LDFLAGS=-rdynamic -l$(KORE_CRYPTO) endif endif @@ -49,27 +77,61 @@ LDFLAGS+=-L$(shell pg_config --libdir) -lpq CFLAGS+=-I$(shell pg_config --includedir) -DKORE_USE_PGSQL \ -DPGSQL_INCLUDE_PATH="\"$(shell pg_config --includedir)\"" + FEATURES+=-DKORE_USE_PGSQL + FEATURES_INC+=-I$(shell pg_config --includedir) endif ifneq ("$(TASKS)", "") S_SRC+=src/tasks.c LDFLAGS+=-lpthread CFLAGS+=-DKORE_USE_TASKS + FEATURES+=-DKORE_USE_TASKS endif ifneq ("$(JSONRPC)", "") S_SRC+=src/jsonrpc.c LDFLAGS+=-lyajl CFLAGS+=-DKORE_USE_JSONRPC + FEATURES+=-DKORE_USE_JSONRPC +endif + +ifneq ("$(PYTHON)", "") + S_SRC+=src/python.c + KORE_PYTHON_LIB?=$(shell python3-config --ldflags) + KORE_PYTHON_INC?=$(shell python3-config --includes) + LDFLAGS+=$(KORE_PYTHON_LIB) + CFLAGS+=$(KORE_PYTHON_INC) -DKORE_USE_PYTHON + FEATURES+=-DKORE_USE_PYTHON + FEATURES_INC+=$(KORE_PYTHON_INC) endif OSNAME=$(shell uname -s | sed -e 's/[-_].*//g' | tr A-Z a-z) +ifeq ("$(OSNAME)", "freebsd") + KORE_CURL_LIB=-L/usr/local/lib -lcurl + KORE_CURL_INC=-I/usr/local/include +endif + +ifneq ("$(CURL)", "") + S_SRC+=src/curl.c + KORE_CURL_LIB?=$(shell pkg-config --libs libcurl) + KORE_CURL_INC?=$(shell pkg-config --cflags libcurl) + LDFLAGS+=$(KORE_CURL_LIB) + CFLAGS+=$(KORE_CURL_INC) -DKORE_USE_CURL + FEATURES+=-DKORE_USE_CURL + FEATURES_INC+=$(KORE_CURL_INC) +endif + +ifneq ("$(SANITIZE)", "") + CFLAGS+=-fsanitize=$(SANITIZE) + LDFLAGS+=-fsanitize=$(SANITIZE) +endif + ifeq ("$(OSNAME)", "darwin") CFLAGS+=-I/opt/local/include/ -I/usr/local/opt/openssl/include LDFLAGS+=-L/opt/local/lib -L/usr/local/opt/openssl/lib S_SRC+=src/bsd.c else ifeq ("$(OSNAME)", "linux") - CFLAGS+=-D_GNU_SOURCE=1 + CFLAGS+=-D_GNU_SOURCE=1 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 LDFLAGS+=-ldl S_SRC+=src/linux.c else @@ -82,31 +144,68 @@ S_OBJS= $(S_SRC:src/%.c=$(OBJDIR)/%.o) +all: $(VERSION) $(KORE) $(KODEV) + +$(VERSION): force + @if [ -d .git ]; then \ + GIT_REVISION=`git rev-parse --short=8 HEAD`; \ + GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`; \ + rm -f $(VERSION); \ + printf "const char *kore_version = \"%s-%s\";\n" \ + $$GIT_BRANCH $$GIT_REVISION > $(VERSION); \ + elif [ -f RELEASE ]; then \ + printf "const char *kore_version = \"%s\";\n" \ + `cat RELEASE` > $(VERSION); \ + else \ + echo "No version information found (no .git or RELEASE)"; \ + exit 1; \ + fi + +$(KODEV): + $(MAKE) -C kodev + $(KORE): $(OBJDIR) $(S_OBJS) $(CC) $(S_OBJS) $(LDFLAGS) -o $(KORE) + @echo $(FEATURES) $(FEATURES_INC) > kore.features objects: $(OBJDIR) $(S_OBJS) - -all: $(KORE) + @echo $(LDFLAGS) > $(OBJDIR)/ldflags + @echo "$(FEATURES) $(FEATURES_INC)" > $(OBJDIR)/features $(OBJDIR): @mkdir -p $(OBJDIR) install: + mkdir -p $(SHARE_DIR) mkdir -p $(INCLUDE_DIR) mkdir -p $(INSTALL_DIR) + mkdir -p $(MAN_DIR)/man1 + install -m 644 share/man/kodev.1 $(MAN_DIR)/man1/kodev.1 install -m 555 $(KORE) $(INSTALL_DIR)/$(KORE) - install -m 644 includes/*.h $(INCLUDE_DIR) + install -m 644 kore.features $(SHARE_DIR)/features + install -m 644 include/kore/*.h $(INCLUDE_DIR) + $(MAKE) -C kodev install uninstall: rm -f $(INSTALL_DIR)/$(KORE) rm -rf $(INCLUDE_DIR) + rm -rf $(SHARE_DIR) + $(MAKE) -C kodev uninstall $(OBJDIR)/%.o: src/%.c $(CC) $(CFLAGS) -c $< -o $@ clean: + rm -f $(VERSION) find . -type f -name \*.o -exec rm {} \; - rm -rf $(KORE) $(OBJDIR) + rm -rf $(KORE) $(OBJDIR) kore.features + $(MAKE) -C kodev clean + +releng-build-examples: + rm -rf /tmp/kore_releng + $(MAKE) clean + $(MAKE) PYTHON=1 PGSQL=1 TASKS=1 PREFIX=/tmp/kore_releng + $(MAKE) install PREFIX=/tmp/kore_releng + $(MAKE) -C examples -.PHONY: all clean +.PHONY: all clean force diff -Nru kore-2.0.0/minisign.pub kore-3.3.1/minisign.pub --- kore-2.0.0/minisign.pub 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/minisign.pub 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,2 @@ +untrusted comment: Kore minisign public key +RWSxkEDc2y+whfKTmvhqs/YaFmEwblmvar7l6RXMjnu6o9tZW3LC0Hc9 diff -Nru kore-2.0.0/README.md kore-3.3.1/README.md --- kore-2.0.0/README.md 2016-08-01 08:00:45.000000000 +0000 +++ kore-3.3.1/README.md 2019-06-03 13:29:24.000000000 +0000 @@ -1,48 +1,62 @@ About ----- -[![Build Status](https://travis-ci.org/jorisvink/kore.svg?branch=master)](https://travis-ci.org/jorisvink/kore) -Kore (https://kore.io) is an easy to use web application framework for +Kore (https://kore.io) is an easy to use web application platform for writing scalable web APIs in C. Its main goals are security, scalability and allowing rapid development and deployment of such APIs. Because of this Kore is an ideal candidate for building robust, scalable and secure web things. -Features --------- +Key Features +------------ * Supports SNI * Supports HTTP/1.1 * Websocket support * Privseps by default -* Lightweight background tasks +* TLS enabled by default +* Optional background tasks * Built-in parameter validation -* Only HTTPS connections allowed -* Built-in asynchronous PostgreSQL support +* Optional asynchronous PostgreSQL support +* Optional support for page handlers in Python +* Reload private keys and certificates on-the-fly * Private keys isolated in separate process (RSA and ECDSA) * Default sane TLS ciphersuites (PFS in all major browsers) * Modules can be reloaded on-the-fly, even while serving content -* Event driven (epoll/kqueue) architecture with per CPU core workers +* Event driven (epoll/kqueue) architecture with per CPU worker processes * Build your web application as a precompiled dynamic library or single binary +And loads more. + License ------- * Kore is licensed under the ISC license +Documentation +-------------- +[Read the documentation](https://docs.kore.io/3.2.0/) + +Performance +----------- +Read the [benchmarks](https://blog.kore.io/posts/benchmarks) blog post. + Platforms supported ------------------- * Linux * OpenBSD * FreeBSD -* OSX - -See https://kore.io/doc/#requirements for more information. +* MacOS Building Kore ------------- +Clone this repository or get the latest release at [https://kore.io/releases/3.2.0](https://kore.io/releases/3.2.0). Requirements -* openssl (latest) +* openssl (1.0.2, 1.1.0 or 1.1.1) (note: this requirement drops away when building with NOTLS=1 NOHTTP=1) + (note: libressl works as a replacement) + +Requirement for asynchronous curl (optional) +* libcurl Requirements for background tasks (optional) * pthreads @@ -50,17 +64,21 @@ Requirements for pgsql (optional) * libpq +Requirements for python (optional) +* Python 3.6+ + Normal compilation and installation: ``` -# cd kore -# make +$ cd kore +$ make # make install ``` If you would like to build a specific flavor, you can enable those by setting a shell environment variable before running **_make_**. +* CURL=1 (compiles in asynchronous curl support) * TASKS=1 (compiles in task support) * PGSQL=1 (compiles in pgsql support) * DEBUG=1 (enables use of -d for debug) @@ -68,21 +86,34 @@ * NOHTTP=1 (compiles Kore without HTTP support) * NOOPT=1 (disable compiler optimizations) * JSONRPC=1 (compiles in JSONRPC support) +* PYTHON=1 (compiles in the Python support) + +Note that certain build flavors cannot be mixed together and you will just +be met with compilation errors. Example applications ----------------- - You can find example applications under **_examples/_**. The examples contain a README file with instructions on how to build or use them. -Bugs, contributions and more ----------------------------- +Mailing lists +------------- + +**patches@kore.io** - Send patches here, preferably inline. + +**users@kore.io** - Questions regarding kore. + + +If you want to signup to those mailing lists send an empty email to + listname+subscribe@kore.io + + +Other mailboxes (these are **not** mailing lists): -If you run into any bugs, have suggestions or patches please -contact me at joris@coders.se. +**security@kore.io** - Mail this email if you think you found a security problem. -If you feel like hanging out or just chatting there is an [IRC chatroom (#kore-dev@irc.freenode.org)](https://webchat.freenode.net?channels=kore-dev). +**sponsor@kore.io** - If your company would like to sponsor part of Kore development. More information can be found on https://kore.io/ diff -Nru kore-2.0.0/RELEASE kore-3.3.1/RELEASE --- kore-2.0.0/RELEASE 2016-08-01 07:57:38.000000000 +0000 +++ kore-3.3.1/RELEASE 2019-06-03 13:29:24.000000000 +0000 @@ -1,6 +1 @@ -This is the official release for Kore 2.0.0. - -If you want the current version, use git to clone the following: - https://github.com/jorisvink/kore/ - -See the README file for more information. +3.3.1-release diff -Nru kore-2.0.0/share/man/kodev.1 kore-3.3.1/share/man/kodev.1 --- kore-2.0.0/share/man/kodev.1 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/share/man/kodev.1 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,278 @@ +.TH KODEV 1 +.SH NAME +kodev \- Kore project management tool + +.SH SYNOPSIS +.BR kodev +[\fIOPTION\fR] ... + +.SH DESCRIPTION +This documentation describes the application management tools for building and +running a Kore project. For information regarding the technical interface and +C bindings, look to +.BR kore(3) +where these functions and structures are described. +.BR Kore +projects may be managed using the following OPTIONS; + +.BR create +.RS +Create a new application skeleton with the name that is passed to it. This will +create a new directory with all the files required to begin hacking. See the +\fBGENERATED FILES\fR section for more information. +.RE + +.BR build +.RS +Build the application. See the \fBBUILDING\fR section for more information. +.RE + +.BR run +.RS +Start the application in the foreground. See the \fBRUNNING\fR section for +more information. +.RE + +.BR reload +.RS +Reload the application. This is a shortcut to sending SIGHUP to the parent +process (see kore_pid). +.RE + +.BR info +.RS +Show information about the application configuration. Namely; active flavor, +output type, Kore features, Kore source and Kore binary. +.RE + +.BR clean +.RS +Cleanup the build files. +.RE + +.BR flavor +.RS +Switch between build flavors with the argument being the new flavor. +.RE + +.BR help +.RS +Show the help synopsis. +.RE + +.SH GENERATED FILES +Executing the +.BR create +command will generate several new files under the directory matching the +application name specified. + +These files are: + +.RS +.BR conf/build.conf +.RS +The build configuration. +.RE + +.BR conf/app.conf +.RS +The Kore application configuration. +.RE + +.BR src/app.c +.RS +The initial placeholder source code. +.RE + +.BR cert/server.pem +.RS +The self-signed auto-generated x509 certificate. +.RE + +.BR cert/key.pem +.RS +The key matching the self-signed x509 certificate. +.RE + +.BR dh2048.pem +.RS +The 2048-bit DH parameters used by TLS. +.RE +.RE + +Those files are: + +.RS +.BR kore.conf +.RS +The Kore application configuration. +.RE +.RE + +.RS +.BR handlers.py +.RS +The initial placeholder python page handler. +.RE +.RE + +.RS +.BR __init__.py +.RS +The python initialization code. Sets up the kore listener, etc. +.RE +.RE + +.RS +.BR cert/server.pem +.RS +The self-signed auto-generated x509 certificate. +.RE +.RE + +.RS +.BR cert/key.pem +.RS +The key matching the self-signed x509 certificate. +.RE +.RE + +.RS +.BR dh2048.pem +.RS +The 2048-bit DH parameters used by TLS. +.RE +.RE + + +.SH BUILDING +Executing the +.BR build +command will build your application. How this happens is instructed by +the +.BR conf/build.conf +configuration file. This file supports the following directives: + +.RS +.BR single_binary +[yes|no] +.RS +If set to \fByes\fR the build system will produce a single binary containing +both your application code and the Kore code allowing you to distribute +your application more easily. + +If set to \fBno\fR the build system will produce a standard dynamicly +linked library that will be loaded into Kore at runtime. +.RE +.RE + +.RS +.BR kore_source +[path] +.RS +Must be set to point to the a Kore source code directory. Used only if +.BR single_binary +option is set to \fByes\fR. + +.BR Example: +kore_source=/home/joris/src/kore +.RE +.RE +.RE + +.RS +.BR kore_flavor +[build options] +.RS +Defines the build arguments for building Kore. Used only if +.BR single_binary +option is set to \fByes\fR. + +.BR Example: +kore_flavor=NOTLS=1 +.RE +.RE + +.RS +.BR cflags +.RS +Standard +.BR CFLAGS +used when compiling the application source code. +.RE +.RE + +.RS +.BR ldflags +.RS +Standard +.BR LDFLAGS +used when linking the application source code. +.RE +.RE + +Note that the +.BR build +command obeys the environment variables +.BR CC +and +.BR CXX + +.SH RUNNING +Executing the +.BR run +command will start your application in the foreground. + +What binary it executes depends +on whether or not the +.BR single_binary +flag was set in build configuration. If the +.BR single_binary +flag was enabled the +.BR run +command will execute the binary produced by the build system. If the +.BR single_binary +flag was not enabled the +.BR run +command will execute the +.BR $PREFIX/bin/kore +binary. +In both cases the +.BR run +command will pass the \fB\-fnr\fR command line options to the binary. +.RE + +.SH EXAMPLES +Changing flavor of the build; + +.RS +$ kodev flavor osx +.RE + +Building your application; + +.RS +$ kodev build +.RE + +.SH REPORTING BUGS, CONTRIBUTING && MORE +If you run into any bugs, have suggestions or patches, please contact me at +.BR + +More information can be found at +.BR + +.SH AUTHOR +.BR Kore +developed by Joris Vink +.BR + +Manpage authored by Guy Nankivell +.BR + +.SH LICENCE +Usage of this software is provided under the +.BR ISC +license which may be found, with the source, at +.BR + diff -Nru kore-2.0.0/src/accesslog.c kore-3.3.1/src/accesslog.c --- kore-2.0.0/src/accesslog.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/accesslog.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,34 +14,45 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include -#include #include +#include #include "kore.h" #include "http.h" -struct kore_log_packet { - u_int8_t method; - int status; - u_int16_t time_req; - u_int16_t worker_id; - u_int16_t worker_cpu; - u_int8_t addrtype; - u_int8_t addr[sizeof(struct in6_addr)]; - char host[KORE_DOMAINNAME_LEN]; - char path[HTTP_URI_LEN]; - char agent[HTTP_USERAGENT_LEN]; +/* + * The worker will write accesslogs to its worker data structure which is + * held in shared memory. + * + * Each accesslog is prefixed with the internal domain ID (2 bytes) and + * the length of the log entry (2 bytes) (packed in kore_alog_header). + * + * The parent will every 10ms fetch the produced accesslogs from the workers + * and copy them to its own log buffer. Once this log buffer becomes full + * or 1 second has passed the parent will parse the logs and append them + * to the correct domain logbuffer which is eventually flushed to disk. + */ + +#define LOGBUF_SIZE (KORE_ACCESSLOG_BUFLEN * worker_count) +#define DOMAIN_LOGBUF_LEN (1024 * 1024) +#define LOG_ENTRY_MINSIZE_GUESS 90 + +static void accesslog_lock(struct kore_worker *); +static void accesslog_unlock(struct kore_worker *); +static void accesslog_flush_cb(struct kore_domain *); +static void accesslog_flush(struct kore_domain *, u_int64_t, int); + +static u_int64_t time_cache = 0; +static char tbuf[128] = { '\0' }; + #if !defined(KORE_NO_TLS) - char cn[X509_CN_LENGTH]; +char cnbuf[1024] = { '\0' }; #endif -}; -void -kore_accesslog_init(void) -{ -} +static struct kore_buf *logbuf = NULL; void kore_accesslog_worker_init(void) @@ -49,30 +60,20 @@ kore_domain_closelogs(); } -int -kore_accesslog_write(const void *data, u_int32_t len) +void +kore_accesslog(struct http_request *req) { - int l; - time_t now; - ssize_t sent; - struct kore_domain *dom; - struct kore_log_packet logpacket; + struct timespec ts; + struct tm *tm; + u_int64_t now; + struct kore_alog_header *hdr; + size_t avail; + time_t curtime; + int len, attempts; char addr[INET6_ADDRSTRLEN]; - char *method, *buf, *tbuf, *cn; - - if (len != sizeof(struct kore_log_packet)) - return (KORE_RESULT_ERROR); + const char *ptr, *method, *cn, *referer; - (void)memcpy(&logpacket, data, sizeof(logpacket)); - - if ((dom = kore_domain_lookup(logpacket.host)) == NULL) { - kore_log(LOG_WARNING, - "got accesslog packet for unknown domain: %s", - logpacket.host); - return (KORE_RESULT_OK); - } - - switch (logpacket.method) { + switch (req->method) { case HTTP_METHOD_GET: method = "GET"; break; @@ -88,96 +89,245 @@ case HTTP_METHOD_HEAD: method = "HEAD"; break; + case HTTP_METHOD_PATCH: + method = "PATCH"; + break; default: method = "UNKNOWN"; break; } - cn = "none"; + if (req->referer != NULL) + referer = req->referer; + else + referer = "-"; + + cn = "-"; #if !defined(KORE_NO_TLS) - if (logpacket.cn[0] != '\0') - cn = logpacket.cn; + if (req->owner->cert != NULL) { + if (X509_GET_CN(req->owner->cert, cnbuf, sizeof(cnbuf)) != -1) + cn = cnbuf; + } #endif - if (inet_ntop(logpacket.addrtype, &(logpacket.addr), - addr, sizeof(addr)) == NULL) - (void)kore_strlcpy(addr, "unknown", sizeof(addr)); - - time(&now); - tbuf = kore_time_to_date(now); - l = asprintf(&buf, "[%s] %s %d %s %s (w#%d) (%dms) (%s) (%s)\n", - tbuf, addr, logpacket.status, method, logpacket.path, - logpacket.worker_id, logpacket.time_req, cn, logpacket.agent); - if (l == -1) { - kore_log(LOG_WARNING, - "kore_accesslog_write(): asprintf() == -1"); - return (KORE_RESULT_ERROR); - } - - sent = write(dom->accesslog, buf, l); - if (sent == -1) { - free(buf); - kore_log(LOG_WARNING, - "kore_accesslog_write(): write(): %s", errno_s); - return (KORE_RESULT_ERROR); + switch (req->owner->family) { + case AF_INET: + ptr = inet_ntop(req->owner->family, + &(req->owner->addr.ipv4.sin_addr), addr, sizeof(addr)); + break; + case AF_INET6: + ptr = inet_ntop(req->owner->family, + &(req->owner->addr.ipv6.sin6_addr), addr, sizeof(addr)); + break; + case AF_UNIX: + ptr = NULL; + break; + default: + fatal("unknown family %d", req->owner->family); + } + + if (ptr == NULL) { + addr[0] = '-'; + addr[1] = '\0'; } - if (sent != l) - kore_log(LOG_NOTICE, "accesslog: %s", buf); + now = kore_time_ms(); + if ((now - time_cache) >= 1000) { + time(&curtime); + tm = localtime(&curtime); + (void)strftime(tbuf, sizeof(tbuf), "%d/%b/%Y:%H:%M:%S %z", tm); + time_cache = now; + } + + attempts = 0; + ts.tv_sec = 0; + ts.tv_nsec = 1000000; + + for (;;) { + if (attempts++ > 1000) { + if (getppid() == 1) { + if (kill(worker->pid, SIGQUIT) == -1) + fatal("failed to shutdown"); + return; + } + + attempts = 0; + } + + accesslog_lock(worker); + + avail = KORE_ACCESSLOG_BUFLEN - worker->lb.offset; + if (avail < sizeof(*hdr) + LOG_ENTRY_MINSIZE_GUESS) { + accesslog_unlock(worker); + nanosleep(&ts, NULL); + continue; + } + + hdr = (struct kore_alog_header *) + (worker->lb.buf + worker->lb.offset); + worker->lb.offset += sizeof(*hdr); + + len = snprintf(worker->lb.buf + worker->lb.offset, avail, + "%s - %s [%s] \"%s %s HTTP/1.1\" %d %zu \"%s\" \"%s\"\n", + addr, cn, tbuf, method, req->path, req->status, + req->content_length, referer, req->agent); + if (len == -1) + fatal("failed to create log entry"); + + if ((size_t)len >= avail) { + worker->lb.offset -= sizeof(*hdr); + accesslog_unlock(worker); + nanosleep(&ts, NULL); + continue; + } + + if ((size_t)len > USHRT_MAX) { + kore_log(LOG_WARNING, + "log entry length exceeds limit (%d)", len); + worker->lb.offset -= sizeof(*hdr); + break; + } + + hdr->loglen = len; + hdr->domain = req->hdlr->dom->id; - free(buf); - return (KORE_RESULT_OK); + worker->lb.offset += (size_t)len; + break; + } + + accesslog_unlock(worker); } void -kore_accesslog(struct http_request *req) +kore_accesslog_gather(void *arg, u_int64_t now, int force) { - struct kore_log_packet logpacket; + int id; + struct kore_worker *kw; + struct kore_alog_header *hdr; + struct kore_domain *dom; + size_t off, remain; + + if (logbuf == NULL) + logbuf = kore_buf_alloc(LOGBUF_SIZE); + + for (id = 0; id < worker_count; id++) { + kw = kore_worker_data(id); + + accesslog_lock(kw); + + if (force || kw->lb.offset >= KORE_ACCESSLOG_SYNC) { + kore_buf_append(logbuf, kw->lb.buf, kw->lb.offset); + kw->lb.offset = 0; + } - logpacket.addrtype = req->owner->addrtype; - if (logpacket.addrtype == AF_INET) { - memcpy(logpacket.addr, - &(req->owner->addr.ipv4.sin_addr), - sizeof(req->owner->addr.ipv4.sin_addr)); - } else { - memcpy(logpacket.addr, - &(req->owner->addr.ipv6.sin6_addr), - sizeof(req->owner->addr.ipv6.sin6_addr)); - } - - logpacket.status = req->status; - logpacket.method = req->method; - logpacket.worker_id = worker->id; - logpacket.worker_cpu = worker->cpu; - logpacket.time_req = req->total; - - if (kore_strlcpy(logpacket.host, - req->host, sizeof(logpacket.host)) >= sizeof(logpacket.host)) - kore_log(LOG_NOTICE, "kore_accesslog: host truncated"); - - if (kore_strlcpy(logpacket.path, - req->path, sizeof(logpacket.path)) >= sizeof(logpacket.path)) - kore_log(LOG_NOTICE, "kore_accesslog: path truncated"); - - if (req->agent != NULL) { - if (kore_strlcpy(logpacket.agent, req->agent, - sizeof(logpacket.agent)) >= sizeof(logpacket.agent)) - kore_log(LOG_NOTICE, "kore_accesslog: agent truncated"); - } else { - (void)kore_strlcpy(logpacket.agent, "unknown", - sizeof(logpacket.agent)); + accesslog_unlock(kw); } -#if !defined(KORE_NO_TLS) - memset(logpacket.cn, '\0', sizeof(logpacket.cn)); - if (req->owner->cert != NULL) { - if (X509_GET_CN(req->owner->cert, - logpacket.cn, sizeof(logpacket.cn)) == -1) { - kore_log(LOG_WARNING, "client cert without a CN?"); + if (force || logbuf->offset >= LOGBUF_SIZE) { + off = 0; + remain = logbuf->offset; + + while (remain > 0) { + if (remain < sizeof(*hdr)) { + kore_log(LOG_ERR, + "invalid log buffer: (%zu remain)", remain); + break; + } + + hdr = (struct kore_alog_header *)(logbuf->data + off); + off += sizeof(*hdr); + remain -= sizeof(*hdr); + + if (hdr->loglen > remain) { + kore_log(LOG_ERR, + "invalid log header: %u (%zu remain)", + hdr->loglen, remain); + break; + } + + if ((dom = kore_domain_byid(hdr->domain)) == NULL) + fatal("unknown domain id %u", hdr->domain); + + if (dom->logbuf == NULL) + dom->logbuf = kore_buf_alloc(DOMAIN_LOGBUF_LEN); + + kore_buf_append(dom->logbuf, &logbuf->data[off], + hdr->loglen); + + off += hdr->loglen; + remain -= hdr->loglen; + + accesslog_flush(dom, now, force); } + + kore_buf_reset(logbuf); } -#endif - kore_msg_send(KORE_MSG_PARENT, - KORE_MSG_ACCESSLOG, &logpacket, sizeof(logpacket)); + if (force) + kore_domain_callback(accesslog_flush_cb); +} + +void +kore_accesslog_run(void *arg, u_int64_t now) +{ + static int ticks = 0; + + kore_accesslog_gather(arg, now, ticks++ % 100 ? 0 : 1); +} + +static void +accesslog_flush_cb(struct kore_domain *dom) +{ + accesslog_flush(dom, 0, 1); +} + +static void +accesslog_flush(struct kore_domain *dom, u_int64_t now, int force) +{ + ssize_t written; + + if (force && dom->logbuf == NULL) + return; + + if (force || dom->logbuf->offset >= DOMAIN_LOGBUF_LEN) { + written = write(dom->accesslog, dom->logbuf->data, + dom->logbuf->offset); + if (written == -1) { + if (errno == EINTR) + return; + if (dom->logwarn == 0 || + errno != dom->logerr) { + kore_log(LOG_NOTICE, + "error writing log for %s (%s)", + dom->domain, errno_s); + dom->logwarn = now; + dom->logerr = errno; + } + kore_buf_reset(dom->logbuf); + return; + } + + if ((size_t)written != dom->logbuf->offset) { + kore_log(LOG_ERR, "partial accesslog write for %s", + dom->domain); + } + + kore_buf_reset(dom->logbuf); + } +} + +static void +accesslog_lock(struct kore_worker *kw) +{ + for (;;) { + if (__sync_bool_compare_and_swap(&kw->lb.lock, 0, 1)) + break; + } +} + +static void +accesslog_unlock(struct kore_worker *kw) +{ + if (!__sync_bool_compare_and_swap(&kw->lb.lock, 1, 0)) + fatal("accesslog_unlock: failed to release"); } diff -Nru kore-2.0.0/src/auth.c kore-3.3.1/src/auth.c --- kore-2.0.0/src/auth.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/auth.c 2019-06-03 13:29:24.000000000 +0000 @@ -15,6 +15,7 @@ */ #include +#include #include @@ -23,10 +24,6 @@ TAILQ_HEAD(, kore_auth) auth_list; -static int kore_auth_cookie(struct http_request *, struct kore_auth *); -static int kore_auth_header(struct http_request *, struct kore_auth *); -static int kore_auth_request(struct http_request *, struct kore_auth *); - void kore_auth_init(void) { @@ -103,22 +100,23 @@ return (KORE_RESULT_ERROR); } -static int +int kore_auth_cookie(struct http_request *req, struct kore_auth *auth) { + const char *hdr; int i, v; size_t len, slen; char *value, *c, *cookie, *cookies[HTTP_MAX_COOKIES]; - if (!http_request_header(req, "cookie", &c)) + if (!http_request_header(req, "cookie", &hdr)) return (KORE_RESULT_ERROR); - cookie = kore_strdup(c); + cookie = kore_strdup(hdr); slen = strlen(auth->value); v = kore_split_string(cookie, ";", cookies, HTTP_MAX_COOKIES); for (i = 0; i < v; i++) { - for (c = cookies[i]; isspace(*c); c++) + for (c = cookies[i]; isspace(*(unsigned char *)c); c++) ; len = MIN(slen, strlen(cookies[i])); @@ -143,10 +141,10 @@ return (i); } -static int +int kore_auth_header(struct http_request *req, struct kore_auth *auth) { - char *header; + const char *header; if (!http_request_header(req, auth->value, &header)) return (KORE_RESULT_ERROR); @@ -154,10 +152,16 @@ return (kore_validator_check(req, auth->validator, header)); } -static int +int kore_auth_request(struct http_request *req, struct kore_auth *auth) { - return (kore_validator_check(req, auth->validator, req)); + int ret; + + req->flags |= HTTP_VALIDATOR_IS_REQUEST; + ret = kore_validator_check(req, auth->validator, req); + req->flags &= ~HTTP_VALIDATOR_IS_REQUEST; + + return (ret); } struct kore_auth * diff -Nru kore-2.0.0/src/bsd.c kore-3.3.1/src/bsd.c --- kore-2.0.0/src/bsd.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/bsd.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -17,6 +17,8 @@ #include #include #include +#include +#include #if defined(__FreeBSD_version) #include @@ -36,13 +38,16 @@ #endif static int kfd = -1; -static struct kevent *events; +static struct kevent *events = NULL; static u_int32_t event_count = 0; +#if defined(KORE_USE_PLATFORM_PLEDGE) +static char pledges[256] = { "stdio rpath inet error" }; +#endif + void kore_platform_init(void) { -#if defined(__MACH__) || defined(__FreeBSD_version) long n; size_t len = sizeof(n); int mib[] = { CTL_HW, HW_NCPU }; @@ -53,9 +58,6 @@ } else { cpu_count = (u_int16_t)n; } -#else - cpu_count = 0; -#endif /* __MACH__ || __FreeBSD_version */ } void @@ -70,7 +72,6 @@ -1, sizeof(cpuset), &cpuset) == -1) { fatal("failed: %s", errno_s); } - #endif /* __FreeBSD_version */ } @@ -79,6 +80,11 @@ { struct listener *l; + if (kfd != -1) + close(kfd); + if (events != NULL) + kore_free(events); + if ((kfd = kqueue()) == -1) fatal("kqueue(): %s", errno_s); @@ -108,110 +114,51 @@ } } -int +void kore_platform_event_wait(u_int64_t timer) { u_int32_t r; - struct listener *l; - struct connection *c; - u_int8_t type; - struct timespec timeo; + struct kore_event *evt; int n, i; + struct timespec timeo, *ts; + + if (timer == KORE_WAIT_INFINITE) { + ts = NULL; + } else { + timeo.tv_sec = timer / 1000; + timeo.tv_nsec = (timer % 1000) * 1000000; + ts = &timeo; + } - timeo.tv_sec = timer / 1000; - timeo.tv_nsec = (timer % 1000) * 1000000; - n = kevent(kfd, NULL, 0, events, event_count, &timeo); + n = kevent(kfd, NULL, 0, events, event_count, ts); if (n == -1) { if (errno == EINTR) - return (0); + return; fatal("kevent(): %s", errno_s); } if (n > 0) kore_debug("main(): %d sockets available", n); - r = 0; for (i = 0; i < n; i++) { - if (events[i].udata == NULL) - fatal("events[%d].udata == NULL", i); + evt = (struct kore_event *)events[i].udata; - type = *(u_int8_t *)events[i].udata; + if (evt == NULL) + fatal("evt == NULL"); - if (events[i].flags & EV_EOF || - events[i].flags & EV_ERROR) { - switch (type) { - case KORE_TYPE_LISTENER: - fatal("error on server socket"); - /* NOTREACHED */ -#if defined(KORE_USE_PGSQL) - case KORE_TYPE_PGSQL_CONN: - kore_pgsql_handle(events[i].udata, 1); - break; -#endif -#if defined(KORE_USE_TASKS) - case KORE_TYPE_TASK: - kore_task_handle(events[i].udata, 1); - break; -#endif - default: - c = (struct connection *)events[i].udata; - kore_connection_disconnect(c); - break; - } + r = 0; - continue; - } + if (events[i].filter == EVFILT_READ) + evt->flags |= KORE_EVENT_READ; - switch (type) { - case KORE_TYPE_LISTENER: - l = (struct listener *)events[i].udata; - - while (worker_active_connections < - worker_max_connections) { - if (worker_accept_threshold != 0 && - r >= worker_accept_threshold) - break; - - if (!kore_connection_accept(l, &c)) { - r = 1; - break; - } - - if (c == NULL) - break; - - r++; - kore_platform_event_all(c->fd, c); - } - break; - case KORE_TYPE_CONNECTION: - c = (struct connection *)events[i].udata; - if (events[i].filter == EVFILT_READ && - !(c->flags & CONN_READ_BLOCK)) - c->flags |= CONN_READ_POSSIBLE; - if (events[i].filter == EVFILT_WRITE && - !(c->flags & CONN_WRITE_BLOCK)) - c->flags |= CONN_WRITE_POSSIBLE; - - if (c->handle != NULL && !c->handle(c)) - kore_connection_disconnect(c); - break; -#if defined(KORE_USE_PGSQL) - case KORE_TYPE_PGSQL_CONN: - kore_pgsql_handle(events[i].udata, 0); - break; -#endif -#if defined(KORE_USE_TASKS) - case KORE_TYPE_TASK: - kore_task_handle(events[i].udata, 0); - break; -#endif - default: - fatal("wrong type in event %d", type); - } - } + if (events[i].filter == EVFILT_WRITE) + evt->flags |= KORE_EVENT_WRITE; - return (r); + if (events[i].flags & EV_EOF || events[i].flags & EV_ERROR) + r = 1; + + evt->handle(evt, r); + } } void @@ -252,13 +199,13 @@ void kore_platform_schedule_read(int fd, void *data) { - kore_platform_event_schedule(fd, EVFILT_READ, EV_ADD, data); + kore_platform_event_schedule(fd, EVFILT_READ, EV_ADD | EV_CLEAR, data); } void kore_platform_schedule_write(int fd, void *data) { - kore_platform_event_schedule(fd, EVFILT_WRITE, EV_ADD, data); + kore_platform_event_schedule(fd, EVFILT_WRITE, EV_ADD | EV_CLEAR, data); } void @@ -268,9 +215,82 @@ } void +kore_platform_disable_write(int fd) +{ + kore_platform_event_schedule(fd, EVFILT_WRITE, EV_DELETE, NULL); +} + +void kore_platform_proctitle(char *title) { -#ifndef __MACH__ +#ifdef __MACH__ + kore_proctitle(title); +#else setproctitle("%s", title); #endif } + +#if defined(KORE_USE_PLATFORM_SENDFILE) +int +kore_platform_sendfile(struct connection *c, struct netbuf *nb) +{ + int ret; + off_t len, smin; + + smin = nb->fd_len - nb->fd_off; + len = MIN(SENDFILE_PAYLOAD_MAX, smin); + +#if defined(__MACH__) + ret = sendfile(nb->file_ref->fd, c->fd, nb->fd_off, &len, NULL, 0); +#else + ret = sendfile(nb->file_ref->fd, c->fd, nb->fd_off, len, NULL, &len, 0); +#endif + + if (ret == -1) { + if (errno == EAGAIN) { + nb->fd_off += len; + c->evt.flags &= ~KORE_EVENT_WRITE; + return (KORE_RESULT_OK); + } + + if (errno == EINTR) { + nb->fd_off += len; + return (KORE_RESULT_OK); + } + + return (KORE_RESULT_ERROR); + } + + nb->fd_off += len; + + if (len == 0 || nb->fd_off == nb->fd_len) { + net_remove_netbuf(c, nb); + c->snb = NULL; + } + + return (KORE_RESULT_OK); +} +#endif + +#if defined(KORE_USE_PLATFORM_PLEDGE) +void +kore_platform_pledge(void) +{ + if (pledge(pledges, NULL) == -1) + fatal("failed to pledge process"); +} + +void +kore_platform_add_pledge(const char *pledge) +{ + size_t len; + + len = strlcat(pledges, " ", sizeof(pledges)); + if (len >= sizeof(pledges)) + fatal("truncation on pledges"); + + len = strlcat(pledges, pledge, sizeof(pledges)); + if (len >= sizeof(pledges)) + fatal("truncation on pledges (%s)", pledge); +} +#endif diff -Nru kore-2.0.0/src/buf.c kore-3.3.1/src/buf.c --- kore-2.0.0/src/buf.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/buf.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,8 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include + #include #include #include @@ -64,7 +66,7 @@ } void -kore_buf_append(struct kore_buf *buf, const void *d, size_t len) +kore_buf_append(struct kore_buf *buf, const void *data, size_t len) { if ((buf->offset + len) < len) fatal("overflow in kore_buf_append"); @@ -74,7 +76,7 @@ buf->data = kore_realloc(buf->data, buf->length); } - memcpy((buf->data + buf->offset), d, len); + memcpy((buf->data + buf->offset), data, len); buf->offset += len; } @@ -82,14 +84,17 @@ kore_buf_appendv(struct kore_buf *buf, const char *fmt, va_list args) { int l; + va_list copy; char *b, sb[BUFSIZ]; + va_copy(copy, args); + l = vsnprintf(sb, sizeof(sb), fmt, args); if (l == -1) fatal("kore_buf_appendv(): vsnprintf error"); if ((size_t)l >= sizeof(sb)) { - l = vasprintf(&b, fmt, args); + l = vasprintf(&b, fmt, copy); if (l == -1) fatal("kore_buf_appendv(): error or truncation"); } else { @@ -99,6 +104,8 @@ kore_buf_append(buf, b, l); if (b != sb) free(b); + + va_end(copy); } void @@ -140,7 +147,8 @@ } void -kore_buf_replace_string(struct kore_buf *b, char *src, void *dst, size_t len) +kore_buf_replace_string(struct kore_buf *b, const char *src, + const void *dst, size_t len) { char *key, *end, *tmp, *p; size_t blen, off, off2, nlen, klen; diff -Nru kore-2.0.0/src/cli.c kore-3.3.1/src/cli.c --- kore-2.0.0/src/cli.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/cli.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Joris Vink + * Copyright (c) 2014-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -33,13 +34,15 @@ #include #include #include +#include #include #include #include #include #include -#include "kore.h" +#define errno_s strerror(errno) +#define ssl_errno_s ERR_error_string(ERR_get_error(), NULL) #if defined(OpenBSD) || defined(__FreeBSD_version) || \ defined(NetBSD) || defined(__DragonFly_version) @@ -58,22 +61,37 @@ #define PRI_TIME_T "ld" #endif -#define LD_FLAGS_MAX 30 -#define CFLAGS_MAX 30 +#define LD_FLAGS_MAX 300 +#define CFLAGS_MAX 300 #define CXXFLAGS_MAX CFLAGS_MAX #define BUILD_NOBUILD 0 #define BUILD_C 1 #define BUILD_CXX 2 +struct cli_buf { + u_int8_t *data; + size_t length; + size_t offset; +}; + +struct mime_type { + char *ext; + char *type; + TAILQ_ENTRY(mime_type) list; +}; + +TAILQ_HEAD(mime_list, mime_type); + struct buildopt { char *name; char *kore_source; char *kore_flavor; + int flavor_nohttp; int single_binary; - struct kore_buf *cflags; - struct kore_buf *cxxflags; - struct kore_buf *ldflags; + struct cli_buf *cflags; + struct cli_buf *cxxflags; + struct cli_buf *ldflags; TAILQ_ENTRY(buildopt) list; }; @@ -100,12 +118,31 @@ TAILQ_HEAD(cfile_list, cfile); -static void cli_fatal(const char *, ...) __attribute__((noreturn)); +static struct cli_buf *cli_buf_alloc(size_t); +static void cli_buf_free(struct cli_buf *); +static char *cli_buf_stringify(struct cli_buf *, size_t *); +static void cli_buf_append(struct cli_buf *, const void *, size_t); +static void cli_buf_appendf(struct cli_buf *, const char *, ...); +static void cli_buf_appendv(struct cli_buf *, const char *, + va_list); + +static void *cli_malloc(size_t); +static char *cli_strdup(const char *); +static void *cli_realloc(void *, size_t); + +static char *cli_text_trim(char *, size_t); +static char *cli_read_line(FILE *, char *, size_t); +static long long cli_strtonum(const char *, long long, long long); +static int cli_split_string(char *, const char *, char **, size_t); + +static void usage(void) __attribute__((noreturn)); +static void fatal(const char *, ...) __attribute__((noreturn)); + static void cli_file_close(int); static void cli_run_kore(void); static void cli_generate_certs(void); -static void cli_link_library(void *); static void cli_compile_kore(void *); +static void cli_link_application(void *); static void cli_compile_source_file(void *); static void cli_mkdir(const char *, int); static int cli_dir_exists(const char *); @@ -114,6 +151,7 @@ static void cli_build_cflags(struct buildopt *); static void cli_build_cxxflags(struct buildopt *); static void cli_build_ldflags(struct buildopt *); +static void cli_file_read(int, char **, size_t *); static void cli_file_writef(int, const char *, ...); static void cli_file_open(const char *, int, int *); static void cli_file_remove(char *, struct dirent *); @@ -121,7 +159,8 @@ static void cli_file_write(int, const void *, size_t); static int cli_vasprintf(char **, const char *, ...); static void cli_spawn_proc(void (*cb)(void *), void *); -static void cli_write_asset(const char *, const char *); +static void cli_write_asset(const char *, const char *, + struct buildopt *); static void cli_register_kore_file(char *, struct dirent *); static void cli_register_source_file(char *, struct dirent *); static void cli_file_create(const char *, const char *, size_t); @@ -145,17 +184,24 @@ const char *); static void cli_buildopt_kore_flavor(struct buildopt *, const char *); +static void cli_buildopt_mime(struct buildopt *, const char *); static void cli_flavor_load(void); static void cli_flavor_change(const char *); +static void cli_kore_features(struct buildopt *, + char **, size_t *); static void cli_run(int, char **); static void cli_help(int, char **); +static void cli_info(int, char **); static void cli_build(int, char **); static void cli_clean(int, char **); static void cli_create(int, char **); +static void cli_reload(int, char **); static void cli_flavor(int, char **); +static void cli_create_help(void); + static void file_create_src(void); static void file_create_config(void); static void file_create_gitignore(void); @@ -163,10 +209,12 @@ static struct cmd cmds[] = { { "help", "this help text", cli_help }, { "run", "run an application (-fnr implied)", cli_run }, + { "reload", "reload the application (SIGHUP)", cli_reload }, + { "info", "show info on kore on this system", cli_info }, { "build", "build an application", cli_build }, { "clean", "cleanup the build files", cli_clean }, { "create", "create a new application skeleton", cli_create }, - { "flavor", "switch build flavor", cli_flavor }, + { "flavor", "switch between build flavors", cli_flavor }, { NULL, NULL, NULL } }; @@ -179,14 +227,21 @@ static const char *gen_dirs[] = { "src", -#if !defined(KORE_NO_TLS) "cert", -#endif "conf", "assets", NULL }; +static const char *http_serveable_function = + "int\n" + "asset_serve_%s_%s(struct http_request *req)\n" + "{\n" + " http_serveable(req, asset_%s_%s, asset_len_%s_%s,\n" + " asset_sha256_%s_%s, \"%s\");\n" + " return (KORE_RESULT_OK);\n" + "}\n"; + static const char *src_data = "#include \n" "#include \n" @@ -205,26 +260,23 @@ "\n" "bind\t\t127.0.0.1 8888\n" "load\t\t./%s.so\n" -#if !defined(KORE_NO_TLS) + "\n" "tls_dhparam\tdh2048.pem\n" -#endif "\n" "domain * {\n" -#if !defined(KORE_NO_TLS) - "\tcertfile\tcert/server.crt\n" - "\tcertkey\t\tcert/server.key\n" -#endif + "\tcertfile\tcert/server.pem\n" + "\tcertkey\t\tcert/key.pem\n" + "\n" "\tstatic\t/\tpage\n" "}\n"; static const char *build_data = "# %s build config\n" - "# You can switch flavors using: kore flavor [newflavor]\n" + "# You can switch flavors using: kodev flavor [newflavor]\n" "\n" "# Set to yes if you wish to produce a single binary instead\n" "# of a dynamic library. If you set this to yes you must also\n" - "# set kore_source together with kore_flavor and update ldflags\n" - "# to include the appropriate libraries you will be linking with.\n" + "# set kore_source together with kore_flavor.\n" "#single_binary=no\n" "#kore_source=/home/joris/src/kore\n" "#kore_flavor=\n" @@ -237,6 +289,11 @@ "cxxflags=-Wall -Wmissing-declarations -Wshadow\n" "cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare\n" "\n" + "# Mime types for assets served via the builtin asset_serve_*\n" + "#mime_add=txt:text/plain; charset=utf-8\n" + "#mime_add=png:image/png\n" + "#mime_add=html:text/html; charset=utf-8\n" + "\n" "dev {\n" " # These flags are added to the shared ones when\n" " # you build the \"dev\" flavor.\n" @@ -249,7 +306,6 @@ "# included if you build with the \"prod\" flavor.\n" "#}\n"; -#if !defined(KORE_NO_TLS) static const char *dh2048_data = "-----BEGIN DH PARAMETERS-----\n" "MIIBCAKCAQEAn4f4Qn5SudFjEYPWTbUaOTLUH85YWmmPFW1+b5bRa9ygr+1wfamv\n" @@ -259,17 +315,17 @@ "Bzy9fYgnUlJ82g/bziCI83R2xAdtH014fR63MpElkqdNeChb94pPbEdFlNUvYIBN\n" "xx2vTUQMqRbB4UdG2zuzzr5j98HDdblQ+wIBAg==\n" "-----END DH PARAMETERS-----"; -#endif static const char *gitignore = "*.o\n.flavor\n.objs\n%s.so\nassets.h\ncert\n"; static int s_fd = -1; static char *appl = NULL; static int run_after = 0; -static char *rootdir = NULL; -static char *compiler_c = "gcc"; -static char *compiler_cpp = "g++"; -static char *compiler_ld = "gcc"; +static char *compiler_c = "cc"; +static char *compiler_cpp = "c++"; +static char *compiler_ld = "cc"; +static const char *prefix = PREFIX; +static struct mime_list mime_types; static struct cfile_list source_files; static struct buildopt_list build_options; static int source_files_count; @@ -279,49 +335,56 @@ static int cxxflags_count = 0; static int ldflags_count = 0; static char *flavor = NULL; +static char *out_dir = "."; +static char *object_dir = ".objs"; static char *cflags[CFLAGS_MAX]; static char *cxxflags[CXXFLAGS_MAX]; static char *ldflags[LD_FLAGS_MAX]; -void -kore_cli_usage(int local) +static void +usage(void) { int i; - if (local) - fprintf(stderr, "Usage: kore [command]\n"); - + fprintf(stderr, "Usage: kodev [command]\n"); fprintf(stderr, "\nAvailable commands:\n"); + for (i = 0; cmds[i].name != NULL; i++) printf("\t%s\t%s\n", cmds[i].name, cmds[i].descr); - fprintf(stderr, "\nThe commands mostly exist for your convenience\n"); - fprintf(stderr, "when hacking on your Kore applications.\n"); - fprintf(stderr, "\n"); - fprintf(stderr, "Production servers should be started using "); - fprintf(stderr, "the options.\n"); - fprintf(stderr, "\nFind more information on https://kore.io\n"); exit(1); } int -kore_cli_main(int argc, char **argv) +main(int argc, char **argv) { int i; + char *env; - if (argc < 1) - kore_cli_usage(1); + if (argc < 2) + usage(); - (void)umask(S_IWGRP|S_IWOTH); + argc--; + argv++; - if ((flavor = strchr(argv[0], ':')) != NULL) - *(flavor)++ = '\0'; + if ((env = getenv("KORE_PREFIX")) != NULL) + prefix = env; + + if ((env = getenv("KORE_OBJDIR")) != NULL) + object_dir = env; + + if ((env = getenv("KODEV_OUTPUT")) != NULL) + out_dir = env; + + (void)umask(S_IWGRP | S_IWOTH); for (i = 0; cmds[i].name != NULL; i++) { if (!strcmp(argv[0], cmds[i].name)) { - argc--; - argv++; + if (strcmp(argv[0], "create")) { + argc--; + argv++; + } command = &cmds[i]; cmds[i].cb(argc, argv); break; @@ -329,8 +392,8 @@ } if (cmds[i].name == NULL) { - fprintf(stderr, "No such command: %s\n", argv[0]); - kore_cli_usage(1); + fprintf(stderr, "unknown command: %s\n", argv[0]); + usage(); } return (0); @@ -339,38 +402,68 @@ static void cli_help(int argc, char **argv) { - kore_cli_usage(1); + usage(); +} + +static void +cli_create_help(void) +{ + printf("Usage: kodev create [-p] [name]\n"); + printf("Synopsis:\n"); + printf(" Create a new application skeleton directory structure.\n"); + printf("\n"); + + exit(1); } static void cli_create(int argc, char **argv) { - int i; - char *fpath; + int i, ch; + char *fpath; + const char **dirs; + struct filegen *files; + + while ((ch = getopt(argc, argv, "hp")) != -1) { + switch (ch) { + case 'h': + cli_create_help(); + break; + default: + cli_create_help(); + break; + } + } + + argc -= optind; + argv += optind; if (argc != 1) - cli_fatal("missing application name"); + cli_create_help(); appl = argv[0]; cli_mkdir(appl, 0755); - rootdir = appl; - for (i = 0; gen_dirs[i] != NULL; i++) { - (void)cli_vasprintf(&fpath, "%s/%s", appl, gen_dirs[i]); + dirs = gen_dirs; + files = gen_files; + + for (i = 0; dirs[i] != NULL; i++) { + (void)cli_vasprintf(&fpath, "%s/%s", appl, dirs[i]); cli_mkdir(fpath, 0755); free(fpath); } - for (i = 0; gen_files[i].cb != NULL; i++) - gen_files[i].cb(); + for (i = 0; files[i].cb != NULL; i++) + files[i].cb(); + + if (chdir(appl) == -1) + fatal("chdir(%s): %s", appl, errno_s); cli_generate_certs(); printf("%s created successfully!\n", appl); - -#if !defined(KORE_NO_TLS) - printf("note: do NOT use the created DH parameters/certificates in production\n"); -#endif + printf("WARNING: DO NOT USE THE GENERATED DH PARAMETERS " + "AND CERTIFICATES IN PRODUCTION\n"); } static void @@ -380,15 +473,16 @@ char pwd[MAXPATHLEN], *conf; if (getcwd(pwd, sizeof(pwd)) == NULL) - cli_fatal("could not get cwd: %s", errno_s); + fatal("could not get cwd: %s", errno_s); appl = basename(pwd); (void)cli_vasprintf(&conf, "conf/%s.conf", appl); if (!cli_dir_exists("conf") || !cli_file_exists(conf)) - cli_fatal("%s doesn't appear to be a kore app", appl); + fatal("%s doesn't appear to be a kore app", appl); free(conf); TAILQ_INIT(&build_options); + TAILQ_INIT(&mime_types); (void)cli_buildopt_new("_default"); cli_buildopt_parse("conf/build.conf"); @@ -421,34 +515,38 @@ char *build_path; int requires_relink, l; char *sofile, *config, *data; - char *assets_path, *p, *obj_path, *cpath; - char pwd[PATH_MAX], *src_path, *assets_header; + char *assets_path, *p, *src_path; + char pwd[PATH_MAX], *assets_header; if (getcwd(pwd, sizeof(pwd)) == NULL) - cli_fatal("could not get cwd: %s", errno_s); + fatal("could not get cwd: %s", errno_s); - rootdir = "."; appl = basename(pwd); - if ((p = getenv("CC")) != NULL) + if ((p = getenv("CC")) != NULL) { compiler_c = p; + compiler_ld = p; + } - if ((p = getenv("CXX")) != NULL) + if ((p = getenv("CXX")) != NULL) { compiler_cpp = p; + compiler_ld = p; + } source_files_count = 0; cxx_files_count = 0; TAILQ_INIT(&source_files); TAILQ_INIT(&build_options); + TAILQ_INIT(&mime_types); - (void)cli_vasprintf(&src_path, "%s/src", rootdir); - (void)cli_vasprintf(&assets_path, "%s/assets", rootdir); - (void)cli_vasprintf(&config, "%s/conf/%s.conf", rootdir, appl); - (void)cli_vasprintf(&assets_header, "%s/src/assets.h", rootdir); - (void)cli_vasprintf(&build_path, "%s/conf/build.conf", rootdir); + (void)cli_vasprintf(&src_path, "src"); + (void)cli_vasprintf(&assets_path, "assets"); + (void)cli_vasprintf(&config, "conf/%s.conf", appl); + (void)cli_vasprintf(&build_path, "conf/build.conf"); + (void)cli_vasprintf(&assets_header, "%s/assets.h", object_dir); if (!cli_dir_exists(src_path) || !cli_file_exists(config)) - cli_fatal("%s doesn't appear to be a kore app", appl); + fatal("%s doesn't appear to be a kore app", appl); cli_flavor_load(); bopt = cli_buildopt_new("_default"); @@ -464,14 +562,12 @@ cli_buildopt_parse(build_path); free(build_path); - (void)cli_vasprintf(&obj_path, "%s/.objs", rootdir); - if (!cli_dir_exists(obj_path)) - cli_mkdir(obj_path, 0755); - free(obj_path); + if (!cli_dir_exists(object_dir)) + cli_mkdir(object_dir, 0755); if (bopt->single_binary) { if (bopt->kore_source == NULL) - cli_fatal("single_binary set but not kore_source"); + fatal("single_binary set but not kore_source"); printf("building kore (%s)\n", bopt->kore_source); cli_spawn_proc(cli_compile_kore, bopt); @@ -533,15 +629,15 @@ requires_relink++; } - (void)unlink(assets_header); free(assets_header); - (void)cli_vasprintf(&cpath, "%s/cert", rootdir); - if (!cli_dir_exists(cpath)) { - cli_mkdir(cpath, 0700); - cli_generate_certs(); + if (bopt->kore_flavor == NULL || + !strstr(bopt->kore_flavor, "NOTLS=1")) { + if (!cli_dir_exists("cert")) { + cli_mkdir("cert", 0700); + cli_generate_certs(); + } } - free(cpath); if (bopt->single_binary) { requires_relink++; @@ -550,12 +646,13 @@ (void)cli_vasprintf(&sofile, "%s.so", appl); } - if (!cli_file_exists(sofile)) + if (!cli_file_exists(sofile) && source_files_count > 0) requires_relink++; + free(sofile); if (requires_relink) { - cli_spawn_proc(cli_link_library, bopt); + cli_spawn_proc(cli_link_application, bopt); printf("%s built successfully!\n", appl); } else { printf("nothing to be done!\n"); @@ -570,11 +667,11 @@ { char pwd[PATH_MAX], *sofile; - if (cli_dir_exists(".objs")) - cli_cleanup_files(".objs"); + if (cli_dir_exists(object_dir)) + cli_cleanup_files(object_dir); if (getcwd(pwd, sizeof(pwd)) == NULL) - cli_fatal("could not get cwd: %s", errno_s); + fatal("could not get cwd: %s", errno_s); appl = basename(pwd); (void)cli_vasprintf(&sofile, "%s.so", appl); @@ -590,9 +687,6 @@ run_after = 1; cli_build(argc, argv); - if (chdir(rootdir) == -1) - cli_fatal("couldn't change directory to %s", rootdir); - /* * We are exec()'ing kore again, while we could technically set * the right cli options manually and just continue running. @@ -601,11 +695,65 @@ } static void +cli_reload(int argc, char **argv) +{ + int fd; + size_t len; + pid_t pid; + char *buf; + + cli_file_open("kore.pid", O_RDONLY, &fd); + cli_file_read(fd, &buf, &len); + cli_file_close(fd); + + if (len == 0) + fatal("reload: pid file is empty"); + + buf[len - 1] = '\0'; + + pid = cli_strtonum(buf, 0, UINT_MAX); + + if (kill(pid, SIGHUP) == -1) + fatal("failed to reload: %s", errno_s); + + printf("reloaded application\n"); +} + +static void +cli_info(int argc, char **argv) +{ + size_t len; + struct buildopt *bopt; + char *features; + + TAILQ_INIT(&mime_types); + TAILQ_INIT(&build_options); + + cli_flavor_load(); + bopt = cli_buildopt_new("_default"); + cli_buildopt_parse("conf/build.conf"); + + printf("active flavor\t %s\n", flavor); + printf("output type \t %s\n", + (bopt->single_binary) ? "binary" : "dso"); + + if (bopt->single_binary) { + printf("kore features\t %s\n", bopt->kore_flavor); + printf("kore source \t %s\n", bopt->kore_source); + } else { + cli_kore_features(bopt, &features, &len); + printf("kore binary \t %s/bin/kore\n", prefix); + printf("kore features\t %.*s\n", (int)len, features); + free(features); + } +} + +static void file_create_src(void) { char *name; - (void)cli_vasprintf(&name, "src/%s.c", appl); + (void)cli_vasprintf(&name, "%s/src/%s.c", appl, appl); cli_file_create(name, src_data, strlen(src_data)); free(name); } @@ -616,14 +764,16 @@ int l; char *name, *data; - (void)cli_vasprintf(&name, "conf/%s.conf", appl); + (void)cli_vasprintf(&name, "%s/conf/%s.conf", appl, appl); l = cli_vasprintf(&data, config_data, appl, appl); cli_file_create(name, data, l); free(name); free(data); + (void)cli_vasprintf(&name, "%s/conf/build.conf", appl); l = cli_vasprintf(&data, build_data, appl); - cli_file_create("conf/build.conf", data, l); + cli_file_create(name, data, l); + free(name); free(data); } @@ -631,10 +781,12 @@ file_create_gitignore(void) { int l; - char *data; + char *name, *data; + (void)cli_vasprintf(&name, "%s/.gitignore", appl); l = cli_vasprintf(&data, gitignore, appl); - cli_file_create(".gitignore", data, l); + cli_file_create(name, data, l); + free(name); free(data); } @@ -642,7 +794,7 @@ cli_mkdir(const char *fpath, int mode) { if (mkdir(fpath, mode) == -1) - cli_fatal("cli_mkdir(%s): %s", fpath, errno_s); + fatal("cli_mkdir(%s): %s", fpath, errno_s); } static int @@ -667,7 +819,7 @@ if (stat(opath, &ost) == -1) { if (errno == ENOENT) return (1); - cli_fatal("stat(%s): %s", opath, errno_s); + fatal("stat(%s): %s", opath, errno_s); } return (fst->st_mtime != ost.st_mtime); @@ -691,7 +843,43 @@ cli_file_open(const char *fpath, int flags, int *fd) { if ((*fd = open(fpath, flags, 0644)) == -1) - cli_fatal("cli_file_open(%s): %s", fpath, errno_s); + fatal("cli_file_open(%s): %s", fpath, errno_s); +} + +static void +cli_file_read(int fd, char **buf, size_t *len) +{ + struct stat st; + char *p; + ssize_t ret; + size_t offset, bytes; + + if (fstat(fd, &st) == -1) + fatal("fstat(): %s", errno_s); + + if (st.st_size > USHRT_MAX) + fatal("cli_file_read: way too big"); + + offset = 0; + bytes = st.st_size; + p = cli_malloc(bytes); + + while (offset != bytes) { + ret = read(fd, p + offset, bytes - offset); + if (ret == -1) { + if (errno == EINTR) + continue; + fatal("read(): %s", errno_s); + } + + if (ret == 0) + fatal("unexpected EOF"); + + offset += (size_t)ret; + } + + *buf = p; + *len = bytes; } static void @@ -713,7 +901,7 @@ va_end(args); if (l == -1) - cli_fatal("cli_file_writef"); + fatal("cli_file_writef"); cli_file_write(fd, buf, l); free(buf); @@ -733,7 +921,7 @@ if (r == -1) { if (errno == EINTR) continue; - cli_fatal("cli_file_write: %s", errno_s); + fatal("cli_file_write: %s", errno_s); } written += r; @@ -744,41 +932,56 @@ cli_file_create(const char *name, const char *data, size_t len) { int fd; - char *fpath; - - (void)cli_vasprintf(&fpath, "%s/%s", rootdir, name); - cli_file_open(fpath, O_CREAT | O_TRUNC | O_WRONLY, &fd); + cli_file_open(name, O_CREAT | O_TRUNC | O_WRONLY, &fd); cli_file_write(fd, data, len); cli_file_close(fd); - printf("created %s\n", fpath); - free(fpath); + printf("created %s\n", name); } static void -cli_write_asset(const char *n, const char *e) +cli_write_asset(const char *n, const char *e, struct buildopt *bopt) { - cli_file_writef(s_fd, "extern u_int8_t asset_%s_%s[];\n", n, e); - cli_file_writef(s_fd, "extern u_int32_t asset_len_%s_%s;\n", n, e); - cli_file_writef(s_fd, "extern time_t asset_mtime_%s_%s;\n", n, e); + cli_file_writef(s_fd, "extern const u_int8_t asset_%s_%s[];\n", n, e); + cli_file_writef(s_fd, "extern const u_int32_t asset_len_%s_%s;\n", n, e); + cli_file_writef(s_fd, "extern const time_t asset_mtime_%s_%s;\n", n, e); + cli_file_writef(s_fd, "extern const char *asset_sha256_%s_%s;\n", n, e); + + if (bopt->flavor_nohttp == 0) { + cli_file_writef(s_fd, + "int asset_serve_%s_%s(struct http_request *);\n", n, e); + } } static void cli_build_asset(char *fpath, struct dirent *dp) { struct stat st; - u_int8_t *d; + SHA256_CTX sctx; off_t off; void *base; - int in, out; + struct mime_type *mime; + struct buildopt *bopt; + const char *mime_type; + int in, out, i, len; + u_int8_t *d, digest[SHA256_DIGEST_LENGTH]; char *cpath, *ext, *opath, *p, *name; + char hash[(SHA256_DIGEST_LENGTH * 2) + 1]; - name = kore_strdup(dp->d_name); + bopt = cli_buildopt_default(); + + /* Ignore hidden files and some editor files */ + if (dp->d_name[0] == '.' || + strrchr(dp->d_name, '~') || strrchr(dp->d_name, '#')) { + return; + } + + name = cli_strdup(dp->d_name); /* Grab the extension as we're using it in the symbol name. */ if ((ext = strrchr(name, '.')) == NULL) - cli_fatal("couldn't find ext in %s", name); + fatal("couldn't find ext in %s", name); /* Replace dots, spaces, etc etc with underscores. */ for (p = name; *p != '\0'; p++) { @@ -788,25 +991,26 @@ /* Grab inode information. */ if (stat(fpath, &st) == -1) - cli_fatal("stat: %s %s", fpath, errno_s); + fatal("stat: %s %s", fpath, errno_s); /* If this file was empty, skip it. */ if (st.st_size == 0) { printf("skipping empty asset %s\n", name); + free(name); return; } - (void)cli_vasprintf(&opath, "%s/.objs/%s.o", rootdir, name); - (void)cli_vasprintf(&cpath, "%s/.objs/%s.c", rootdir, name); + (void)cli_vasprintf(&opath, "%s/%s.o", object_dir, name); + (void)cli_vasprintf(&cpath, "%s/%s.c", object_dir, name); /* Check if the file needs to be built. */ if (!cli_file_requires_build(&st, opath)) { *(ext)++ = '\0'; - cli_write_asset(name, ext); + cli_write_asset(name, ext, bopt); *ext = '_'; cli_add_source_file(name, cpath, opath, &st, BUILD_NOBUILD); - kore_free(name); + free(name); return; } @@ -816,7 +1020,7 @@ /* mmap our in file. */ if ((base = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, in, 0)) == MAP_FAILED) - cli_fatal("mmap: %s %s", fpath, errno_s); + fatal("mmap: %s %s", fpath, errno_s); /* Create the c file where we will write too. */ cli_file_open(cpath, O_CREAT | O_TRUNC | O_WRONLY, &out); @@ -828,9 +1032,12 @@ /* Start generating the file. */ cli_file_writef(out, "/* Auto generated */\n"); cli_file_writef(out, "#include \n\n"); + cli_file_writef(out, "#include \n"); + cli_file_writef(out, "#include \n\n"); + cli_file_writef(out, "#include \"assets.h\"\n\n"); /* Write the file data as a byte array. */ - cli_file_writef(out, "u_int8_t asset_%s_%s[] = {\n", name, ext); + cli_file_writef(out, "const u_int8_t asset_%s_%s[] = {\n", name, ext); d = base; for (off = 0; off < st.st_size; off++) cli_file_writef(out, "0x%02x,", *d++); @@ -842,19 +1049,51 @@ */ cli_file_writef(out, "0x00"); + /* Calculate the SHA256 digest of the contents. */ + (void)SHA256_Init(&sctx); + (void)SHA256_Update(&sctx, base, st.st_size); + (void)SHA256_Final(digest, &sctx); + + for (i = 0; i < (int)sizeof(digest); i++) { + len = snprintf(hash + (i * 2), sizeof(hash) - (i * 2), + "%02x", digest[i]); + if (len == -1 || (size_t)len >= sizeof(hash)) + fatal("failed to convert SHA256 digest to hex"); + } + + mime = NULL; + TAILQ_FOREACH(mime, &mime_types, list) { + if (!strcasecmp(mime->ext, ext)) + break; + } + + if (mime != NULL) + mime_type = mime->type; + else + mime_type = "text/plain"; + /* Add the meta data. */ cli_file_writef(out, "};\n\n"); - cli_file_writef(out, "u_int32_t asset_len_%s_%s = %" PRIu32 ";\n", + cli_file_writef(out, "const u_int32_t asset_len_%s_%s = %" PRIu32 ";\n", name, ext, (u_int32_t)st.st_size); - cli_file_writef(out, "time_t asset_mtime_%s_%s = %" PRI_TIME_T ";\n", + cli_file_writef(out, + "const time_t asset_mtime_%s_%s = %" PRI_TIME_T ";\n", name, ext, st.st_mtime); + if (bopt->flavor_nohttp == 0) { + cli_file_writef(out, + "const char *asset_sha256_%s_%s = \"\\\"%s\\\"\";\n", + name, ext, hash); + cli_file_writef(out, http_serveable_function, + name, ext, name, ext, name, ext, name, ext, mime_type); + } + /* Write the file symbols into assets.h so they can be used. */ - cli_write_asset(name, ext); + cli_write_asset(name, ext, bopt); /* Cleanup static file source. */ if (munmap(base, st.st_size) == -1) - cli_fatal("munmap: %s %s", fpath, errno_s); + fatal("munmap: %s %s", fpath, errno_s); /* Cleanup fds */ cli_file_close(in); @@ -865,7 +1104,7 @@ /* Register the .c file now (cpath is free'd later). */ cli_add_source_file(name, cpath, opath, &st, BUILD_C); - kore_free(name); + free(name); } static void @@ -875,13 +1114,13 @@ struct cfile *cf; source_files_count++; - cf = kore_malloc(sizeof(*cf)); + cf = cli_malloc(sizeof(*cf)); cf->st = *st; cf->build = build; cf->fpath = fpath; cf->opath = opath; - cf->name = kore_strdup(name); + cf->name = cli_strdup(name); TAILQ_INSERT_TAIL(&source_files, cf, list); } @@ -898,12 +1137,12 @@ return; if (stat(fpath, &st) == -1) - cli_fatal("stat(%s): %s", fpath, errno_s); + fatal("stat(%s): %s", fpath, errno_s); if (!strcmp(ext, ".cpp")) cxx_files_count++; - (void)cli_vasprintf(&opath, "%s/.objs/%s.o", rootdir, dp->d_name); + (void)cli_vasprintf(&opath, "%s/%s.o", object_dir, dp->d_name); if (!cli_file_requires_build(&st, opath)) { build = BUILD_NOBUILD; } else if (!strcmp(ext, ".cpp")) { @@ -925,13 +1164,13 @@ return; if (stat(fpath, &st) == -1) - cli_fatal("stat(%s): %s", fpath, errno_s); + fatal("stat(%s): %s", fpath, errno_s); *ext = '\0'; if ((fname = basename(fpath)) == NULL) - cli_fatal("basename failed"); + fatal("basename failed"); - (void)cli_vasprintf(&opath, "%s/.objs/%s.o", rootdir, fname); + (void)cli_vasprintf(&opath, "%s/%s.o", object_dir, fname); /* Silently ignore non existing object files for kore source files. */ if (stat(opath, &ost) == -1) { @@ -958,7 +1197,7 @@ char *fpath; if ((d = opendir(path)) == NULL) - cli_fatal("cli_find_files: opendir(%s): %s", path, errno_s); + fatal("cli_find_files: opendir(%s): %s", path, errno_s); while ((dp = readdir(d)) != NULL) { if (!strcmp(dp->d_name, ".") || @@ -989,7 +1228,6 @@ static void cli_generate_certs(void) { -#if !defined(KORE_NO_TLS) BIGNUM *e; FILE *fp; time_t now; @@ -997,106 +1235,101 @@ EVP_PKEY *pkey; X509 *x509; RSA *kpair; - char *fpath, issuer[64]; + char issuer[64]; /* Write out DH parameters. */ cli_file_create("dh2048.pem", dh2048_data, strlen(dh2048_data)); /* Create new certificate. */ if ((x509 = X509_new()) == NULL) - cli_fatal("X509_new(): %s", ssl_errno_s); + fatal("X509_new(): %s", ssl_errno_s); /* Generate version 3. */ if (!X509_set_version(x509, 2)) - cli_fatal("X509_set_version(): %s", ssl_errno_s); + fatal("X509_set_version(): %s", ssl_errno_s); /* Generate RSA keys. */ if ((pkey = EVP_PKEY_new()) == NULL) - cli_fatal("EVP_PKEY_new(): %s", ssl_errno_s); + fatal("EVP_PKEY_new(): %s", ssl_errno_s); if ((kpair = RSA_new()) == NULL) - cli_fatal("RSA_new(): %s", ssl_errno_s); + fatal("RSA_new(): %s", ssl_errno_s); if ((e = BN_new()) == NULL) - cli_fatal("BN_new(): %s", ssl_errno_s); + fatal("BN_new(): %s", ssl_errno_s); if (!BN_set_word(e, 65537)) - cli_fatal("BN_set_word(): %s", ssl_errno_s); + fatal("BN_set_word(): %s", ssl_errno_s); if (!RSA_generate_key_ex(kpair, 2048, e, NULL)) - cli_fatal("RSA_generate_key_ex(): %s", ssl_errno_s); + fatal("RSA_generate_key_ex(): %s", ssl_errno_s); BN_free(e); if (!EVP_PKEY_assign_RSA(pkey, kpair)) - cli_fatal("EVP_PKEY_assign_RSA(): %s", ssl_errno_s); + fatal("EVP_PKEY_assign_RSA(): %s", ssl_errno_s); /* Set serial number to current timestamp. */ time(&now); if (!ASN1_INTEGER_set(X509_get_serialNumber(x509), now)) - cli_fatal("ASN1_INTEGER_set(): %s", ssl_errno_s); + fatal("ASN1_INTEGER_set(): %s", ssl_errno_s); /* Not before and not after dates. */ if (!X509_gmtime_adj(X509_get_notBefore(x509), 0)) - cli_fatal("X509_gmtime_adj(): %s", ssl_errno_s); + fatal("X509_gmtime_adj(): %s", ssl_errno_s); if (!X509_gmtime_adj(X509_get_notAfter(x509), (long)60 * 60 * 24 * 3000)) - cli_fatal("X509_gmtime_adj(): %s", ssl_errno_s); + fatal("X509_gmtime_adj(): %s", ssl_errno_s); /* Attach the pkey to the certificate. */ if (!X509_set_pubkey(x509, pkey)) - cli_fatal("X509_set_pubkey(): %s", ssl_errno_s); + fatal("X509_set_pubkey(): %s", ssl_errno_s); /* Set certificate information. */ if ((name = X509_get_subject_name(x509)) == NULL) - cli_fatal("X509_get_subject_name(): %s", ssl_errno_s); + fatal("X509_get_subject_name(): %s", ssl_errno_s); (void)snprintf(issuer, sizeof(issuer), "kore autogen: %s", appl); if (!X509_NAME_add_entry_by_txt(name, "C", MBSTRING_ASC, (const unsigned char *)"SE", -1, -1, 0)) - cli_fatal("X509_NAME_add_entry_by_txt(): C %s", ssl_errno_s); + fatal("X509_NAME_add_entry_by_txt(): C %s", ssl_errno_s); if (!X509_NAME_add_entry_by_txt(name, "O", MBSTRING_ASC, (const unsigned char *)issuer, -1, -1, 0)) - cli_fatal("X509_NAME_add_entry_by_txt(): O %s", ssl_errno_s); + fatal("X509_NAME_add_entry_by_txt(): O %s", ssl_errno_s); if (!X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC, (const unsigned char *)"localhost", -1, -1, 0)) - cli_fatal("X509_NAME_add_entry_by_txt(): CN %s", ssl_errno_s); + fatal("X509_NAME_add_entry_by_txt(): CN %s", ssl_errno_s); if (!X509_set_issuer_name(x509, name)) - cli_fatal("X509_set_issuer_name(): %s", ssl_errno_s); + fatal("X509_set_issuer_name(): %s", ssl_errno_s); if (!X509_sign(x509, pkey, EVP_sha256())) - cli_fatal("X509_sign(): %s", ssl_errno_s); - - (void)cli_vasprintf(&fpath, "%s/cert/server.key", rootdir); - if ((fp = fopen(fpath, "w")) == NULL) - cli_fatal("fopen(%s): %s", fpath, errno_s); - free(fpath); + fatal("X509_sign(): %s", ssl_errno_s); + if ((fp = fopen("cert/key.pem", "w")) == NULL) + fatal("fopen(cert/key.pem): %s", errno_s); if (!PEM_write_PrivateKey(fp, pkey, NULL, NULL, 0, NULL, NULL)) - cli_fatal("PEM_write_PrivateKey(): %s", ssl_errno_s); + fatal("PEM_write_PrivateKey(): %s", ssl_errno_s); fclose(fp); - (void)cli_vasprintf(&fpath, "%s/cert/server.crt", rootdir); - if ((fp = fopen(fpath, "w")) == NULL) - cli_fatal("fopen(%s): %s", fpath, errno_s); - free(fpath); - + if ((fp = fopen("cert/server.pem", "w")) == NULL) + fatal("fopen(cert/server.pem): %s", errno_s); if (!PEM_write_X509(fp, x509)) - cli_fatal("PEM_write_X509(%s)", errno_s); + fatal("PEM_write_X509(%s)", errno_s); fclose(fp); EVP_PKEY_free(pkey); X509_free(x509); -#endif } static void cli_compile_source_file(void *arg) { - int idx, i; - struct cfile *cf = arg; - char *args[32 + CFLAGS_MAX]; - char *compiler; - char **flags; - int flags_count; + struct cfile *cf; + int idx, i; + char **flags; + char *compiler; + int flags_count; + char *args[34 + CFLAGS_MAX]; + + cf = arg; switch (cf->build) { case BUILD_C: @@ -1110,7 +1343,7 @@ flags_count = cxxflags_count; break; default: - cli_fatal("cli_compile_file: unexpected file type: %d", + fatal("cli_compile_file: unexpected file type: %d", cf->build); break; } @@ -1121,6 +1354,8 @@ for (i = 0; i < flags_count; i++) args[idx++] = flags[i]; + args[idx++] = "-I"; + args[idx++] = object_dir; args[idx++] = "-c"; args[idx++] = cf->fpath; args[idx++] = "-o"; @@ -1128,11 +1363,11 @@ args[idx] = NULL; execvp(compiler, args); - cli_fatal("failed to start '%s': %s", compiler, errno_s); + fatal("failed to start '%s': %s", compiler, errno_s); } static void -cli_link_library(void *arg) +cli_link_application(void *arg) { struct cfile *cf; struct buildopt *bopt; @@ -1143,9 +1378,9 @@ bopt = arg; if (bopt->single_binary) - (void)cli_vasprintf(&output, "%s/%s", rootdir, appl); + (void)cli_vasprintf(&output, "%s/%s", out_dir, appl); else - (void)cli_vasprintf(&output, "%s/%s.so", rootdir, appl); + (void)cli_vasprintf(&output, "%s/%s.so", out_dir, appl); idx = 0; args[idx++] = compiler_ld; @@ -1156,19 +1391,12 @@ for (i = 0; i < ldflags_count; i++) args[idx++] = ldflags[i]; - if (bopt->single_binary) { - args[idx++] = "-rdynamic"; -#if defined(__linux__) - args[idx++] = "-ldl"; -#endif - } - args[idx++] = "-o"; args[idx++] = output; args[idx] = NULL; execvp(compiler_ld, args); - cli_fatal("failed to start '%s': %s", compiler_ld, errno_s); + fatal("failed to start '%s': %s", compiler_ld, errno_s); } static void @@ -1176,20 +1404,20 @@ { struct buildopt *bopt = arg; int idx, i, fcnt; - char *obj, *args[16], pwd[MAXPATHLEN], *flavors[7]; - - if (getcwd(pwd, sizeof(pwd)) == NULL) - cli_fatal("could not get cwd: %s", errno_s); + char pwd[MAXPATHLEN], *obj, *args[20], *flavors[7]; - (void)cli_vasprintf(&obj, "OBJDIR=%s/.objs", pwd); + if (object_dir[0] != '/') { + if (getcwd(pwd, sizeof(pwd)) == NULL) + fatal("could not get cwd: %s", errno_s); + (void)cli_vasprintf(&obj, "OBJDIR=%s/%s", pwd, object_dir); + } else { + (void)cli_vasprintf(&obj, "OBJDIR=%s", object_dir); + } if (putenv(obj) != 0) - cli_fatal("cannot set OBJDIR for building kore"); + fatal("cannot set OBJDIR for building kore"); - if (putenv("CFLAGS=-DKORE_SINGLE_BINARY") != 0) - cli_fatal("cannot set CFLAGS for building kore"); - - fcnt = kore_split_string(bopt->kore_flavor, " ", flavors, 7); + fcnt = cli_split_string(bopt->kore_flavor, " ", flavors, 7); #if defined(OpenBSD) || defined(__FreeBSD_version) || \ defined(NetBSD) || defined(__DragonFly_version) @@ -1209,10 +1437,11 @@ args[idx++] = flavors[i]; } + args[idx++] = "KORE_SINGLE_BINARY=1"; args[idx] = NULL; execvp(args[0], args); - cli_fatal("failed to start '%s': %s", args[0], errno_s); + fatal("failed to start '%s': %s", args[0], errno_s); } static void @@ -1228,8 +1457,8 @@ flags = "-fnr"; (void)cli_vasprintf(&cmd, "./%s", appl); } else { - cmd = "kore"; flags = "-fnrc"; + (void)cli_vasprintf(&cmd, "%s/bin/kore", prefix); (void)cli_vasprintf(&cpath, "conf/%s.conf", appl); } @@ -1244,22 +1473,23 @@ } execvp(args[0], args); - cli_fatal("failed to start '%s': %s", args[0], errno_s); + fatal("failed to start '%s': %s", args[0], errno_s); } static void cli_buildopt_parse(const char *path) { FILE *fp; + const char *env; struct buildopt *bopt; char buf[BUFSIZ], *p, *t; if ((fp = fopen(path, "r")) == NULL) - cli_fatal("cli_buildopt_parse: fopen(%s): %s", path, errno_s); + fatal("cli_buildopt_parse: fopen(%s): %s", path, errno_s); bopt = NULL; - while ((p = kore_read_line(fp, buf, sizeof(buf))) != NULL) { + while ((p = cli_read_line(fp, buf, sizeof(buf))) != NULL) { if (strlen(p) == 0) continue; @@ -1272,10 +1502,10 @@ if ((t = strchr(p, '=')) != NULL) goto parse_option; if ((t = strchr(p, ' ')) == NULL) - cli_fatal("unexpected '%s'", p); + fatal("unexpected '%s'", p); *(t)++ = '\0'; if (strcmp(t, "{")) - cli_fatal("expected '{', got '%s'", t); + fatal("expected '{', got '%s'", t); bopt = cli_buildopt_new(p); continue; } @@ -1288,8 +1518,8 @@ parse_option: *(t)++ = '\0'; - p = kore_text_trim(p, strlen(p)); - t = kore_text_trim(t, strlen(t)); + p = cli_text_trim(p, strlen(p)); + t = cli_text_trim(t, strlen(t)); if (!strcasecmp(p, "cflags")) { cli_buildopt_cflags(bopt, t); @@ -1303,10 +1533,20 @@ cli_buildopt_kore_source(bopt, t); } else if (!strcasecmp(p, "kore_flavor")) { cli_buildopt_kore_flavor(bopt, t); + } else if (!strcasecmp(p, "mime_add")) { + cli_buildopt_mime(bopt, t); } else { printf("ignoring unknown option '%s'\n", p); } } + + fclose(fp); + + if ((env = getenv("KORE_SOURCE")) != NULL) + cli_buildopt_kore_source(NULL, env); + + if ((env = getenv("KORE_FLAVOR")) != NULL) + cli_buildopt_kore_flavor(NULL, env); } static struct buildopt * @@ -1314,14 +1554,15 @@ { struct buildopt *bopt; - bopt = kore_malloc(sizeof(*bopt)); + bopt = cli_malloc(sizeof(*bopt)); bopt->cflags = NULL; bopt->cxxflags = NULL; bopt->ldflags = NULL; + bopt->flavor_nohttp = 0; bopt->single_binary = 0; bopt->kore_source = NULL; bopt->kore_flavor = NULL; - bopt->name = kore_strdup(name); + bopt->name = cli_strdup(name); TAILQ_INSERT_TAIL(&build_options, bopt, list); return (bopt); @@ -1355,22 +1596,31 @@ cli_buildopt_cleanup(void) { struct buildopt *bopt, *next; + struct mime_type *mime, *mnext; for (bopt = TAILQ_FIRST(&build_options); bopt != NULL; bopt = next) { next = TAILQ_NEXT(bopt, list); TAILQ_REMOVE(&build_options, bopt, list); if (bopt->cflags != NULL) - kore_buf_free(bopt->cflags); + cli_buf_free(bopt->cflags); if (bopt->cxxflags != NULL) - kore_buf_free(bopt->cxxflags); + cli_buf_free(bopt->cxxflags); if (bopt->ldflags != NULL) - kore_buf_free(bopt->ldflags); + cli_buf_free(bopt->ldflags); if (bopt->kore_source != NULL) - kore_free(bopt->kore_source); + free(bopt->kore_source); if (bopt->kore_flavor != NULL) - kore_free(bopt->kore_flavor); - kore_free(bopt); + free(bopt->kore_flavor); + free(bopt); + } + + for (mime = TAILQ_FIRST(&mime_types); mime != NULL; mime = mnext) { + mnext = TAILQ_NEXT(mime, list); + TAILQ_REMOVE(&mime_types, mime, list); + free(mime->type); + free(mime->ext); + free(mime); } } @@ -1381,9 +1631,9 @@ bopt = cli_buildopt_default(); if (bopt->cflags == NULL) - bopt->cflags = kore_buf_alloc(128); + bopt->cflags = cli_buf_alloc(128); - kore_buf_appendf(bopt->cflags, "%s ", string); + cli_buf_appendf(bopt->cflags, "%s ", string); } static void @@ -1393,9 +1643,9 @@ bopt = cli_buildopt_default(); if (bopt->cxxflags == NULL) - bopt->cxxflags = kore_buf_alloc(128); + bopt->cxxflags = cli_buf_alloc(128); - kore_buf_appendf(bopt->cxxflags, "%s ", string); + cli_buf_appendf(bopt->cxxflags, "%s ", string); } static void @@ -1405,9 +1655,9 @@ bopt = cli_buildopt_default(); if (bopt->ldflags == NULL) - bopt->ldflags = kore_buf_alloc(128); + bopt->ldflags = cli_buf_alloc(128); - kore_buf_appendf(bopt->ldflags, "%s ", string); + cli_buf_appendf(bopt->ldflags, "%s ", string); } static void @@ -1416,7 +1666,7 @@ if (bopt == NULL) bopt = cli_buildopt_default(); else - cli_fatal("single_binary only supported in global context"); + fatal("single_binary only supported in global context"); if (!strcmp(string, "yes")) bopt->single_binary = 1; @@ -1430,172 +1680,276 @@ if (bopt == NULL) bopt = cli_buildopt_default(); else - cli_fatal("kore_source only supported in global context"); + fatal("kore_source only supported in global context"); if (bopt->kore_source != NULL) - kore_free(bopt->kore_source); + free(bopt->kore_source); - bopt->kore_source = kore_strdup(string); + bopt->kore_source = cli_strdup(string); } static void cli_buildopt_kore_flavor(struct buildopt *bopt, const char *string) { + int cnt, i; + char *p, *copy, *flavors[10]; + if (bopt == NULL) bopt = cli_buildopt_default(); else - cli_fatal("kore_flavor only supported in global context"); + fatal("kore_flavor only supported in global context"); if (bopt->kore_flavor != NULL) - kore_free(bopt->kore_flavor); + free(bopt->kore_flavor); + + copy = cli_strdup(string); + cnt = cli_split_string(copy, " ", flavors, 10); + + for (i = 0; i < cnt; i++) { + if ((p = strchr(flavors[i], '=')) == NULL) + fatal("invalid flavor %s", string); + + *p = '\0'; + + if (!strcmp(flavors[i], "NOHTTP")) + bopt->flavor_nohttp = 1; + } - bopt->kore_flavor = kore_strdup(string); + bopt->kore_flavor = cli_strdup(string); + free(copy); } static void -cli_build_flags_common(struct kore_buf* buf) +cli_buildopt_mime(struct buildopt *bopt, const char *ext) { - kore_buf_appendf(buf, - "-fPIC -I%s/src -I%s/src/includes ", rootdir, rootdir); -#if defined(PREFIX) - kore_buf_appendf(buf, "-I%s/include ", PREFIX); -#else - kore_buf_appendf(buf, "-I/usr/local/include "); -#endif + struct mime_type *mime; + char *type; + + if (bopt == NULL) + bopt = cli_buildopt_default(); + else + fatal("mime_add only supported in global context"); + + if ((type = strchr(ext, ':')) == NULL) + fatal("no type given in %s", ext); + + *(type)++ = '\0'; + TAILQ_FOREACH(mime, &mime_types, list) { + if (!strcmp(mime->ext, ext)) + fatal("duplicate extension %s found", ext); + } + + mime = cli_malloc(sizeof(*mime)); + mime->ext = cli_strdup(ext); + mime->type = cli_strdup(type); + + TAILQ_INSERT_TAIL(&mime_types, mime, list); +} + +static void +cli_build_flags_common(struct buildopt *bopt, struct cli_buf *buf) +{ + size_t len; + char *data; + + cli_buf_appendf(buf, "-fPIC -Isrc -Isrc/includes "); + + if (bopt->single_binary == 0) + cli_buf_appendf(buf, "-I%s/include ", prefix); + else + cli_buf_appendf(buf, "-I%s/include ", bopt->kore_source); + #if defined(__MACH__) /* Add default openssl include path from homebrew / ports under OSX. */ - kore_buf_appendf(buf, "-I/opt/local/include "); - kore_buf_appendf(buf, "-I/usr/local/opt/openssl/include "); -#endif -#if defined(KORE_USE_PGSQL) - kore_buf_appendf(buf, "-I%s ", PGSQL_INCLUDE_PATH); -#endif -#if defined(KORE_NO_HTTP) - kore_buf_appendf(buf, "-DKORE_NO_HTTP "); -#endif -#if defined(KORE_NO_TLS) - kore_buf_appendf(buf, "-DKORE_NO_TLS "); + cli_buf_appendf(buf, "-I/opt/local/include "); + cli_buf_appendf(buf, "-I/usr/local/opt/openssl/include "); #endif + if (bopt->single_binary == 0) { + cli_kore_features(bopt, &data, &len); + cli_buf_append(buf, data, len); + cli_buf_appendf(buf, " "); + free(data); + } } static void cli_build_cflags(struct buildopt *bopt) { + size_t len; struct buildopt *obopt; - char *string; + char *string, *buf, *env; if ((obopt = cli_buildopt_find(flavor)) == NULL) - cli_fatal("no such build flavor: %s", flavor); + fatal("no such build flavor: %s", flavor); if (bopt->cflags == NULL) - bopt->cflags = kore_buf_alloc(128); + bopt->cflags = cli_buf_alloc(128); - cli_build_flags_common(bopt->cflags); + cli_build_flags_common(bopt, bopt->cflags); if (obopt != NULL && obopt->cflags != NULL) { - kore_buf_append(bopt->cflags, obopt->cflags->data, + cli_buf_append(bopt->cflags, obopt->cflags->data, obopt->cflags->offset); } - if (bopt->single_binary) - kore_buf_appendf(bopt->cflags, "-DKORE_SINGLE_BINARY"); + if (bopt->single_binary) { + cli_kore_features(bopt, &buf, &len); + cli_buf_append(bopt->cflags, buf, len); + cli_buf_appendf(bopt->cflags, " "); + free(buf); + } + + if ((env = getenv("CFLAGS")) != NULL) + cli_buf_appendf(bopt->cflags, "%s", env); - string = kore_buf_stringify(bopt->cflags, NULL); + string = cli_buf_stringify(bopt->cflags, NULL); printf("CFLAGS=%s\n", string); - cflags_count = kore_split_string(string, " ", cflags, CFLAGS_MAX); + cflags_count = cli_split_string(string, " ", cflags, CFLAGS_MAX); } static void cli_build_cxxflags(struct buildopt *bopt) { struct buildopt *obopt; - char *string; + char *string, *env; if ((obopt = cli_buildopt_find(flavor)) == NULL) - cli_fatal("no such build flavor: %s", flavor); + fatal("no such build flavor: %s", flavor); if (bopt->cxxflags == NULL) - bopt->cxxflags = kore_buf_alloc(128); + bopt->cxxflags = cli_buf_alloc(128); - cli_build_flags_common(bopt->cxxflags); + cli_build_flags_common(bopt, bopt->cxxflags); if (obopt != NULL && obopt->cxxflags != NULL) { - kore_buf_append(bopt->cxxflags, obopt->cxxflags->data, + cli_buf_append(bopt->cxxflags, obopt->cxxflags->data, obopt->cxxflags->offset); } - string = kore_buf_stringify(bopt->cxxflags, NULL); + if ((env = getenv("CXXFLAGS")) != NULL) + cli_buf_appendf(bopt->cxxflags, "%s", env); + + string = cli_buf_stringify(bopt->cxxflags, NULL); if (cxx_files_count > 0) printf("CXXFLAGS=%s\n", string); - cxxflags_count = kore_split_string(string, " ", cxxflags, CXXFLAGS_MAX); + cxxflags_count = cli_split_string(string, " ", cxxflags, CXXFLAGS_MAX); } static void cli_build_ldflags(struct buildopt *bopt) { + int fd; + size_t len; struct buildopt *obopt; - char *string; + char *string, *buf, *env, *path; if ((obopt = cli_buildopt_find(flavor)) == NULL) - cli_fatal("no such build flavor: %s", flavor); + fatal("no such build flavor: %s", flavor); if (bopt->ldflags == NULL) - bopt->ldflags = kore_buf_alloc(128); + bopt->ldflags = cli_buf_alloc(128); if (bopt->single_binary == 0) { #if defined(__MACH__) - kore_buf_appendf(bopt->ldflags, + cli_buf_appendf(bopt->ldflags, "-dynamiclib -undefined suppress -flat_namespace "); #else - kore_buf_appendf(bopt->ldflags, "-shared "); + cli_buf_appendf(bopt->ldflags, "-shared "); #endif + } else { + (void)cli_vasprintf(&path, "%s/ldflags", object_dir); + cli_file_open(path, O_RDONLY, &fd); + cli_file_read(fd, &buf, &len); + cli_file_close(fd); + if (len == 0) + fatal("ldflags is empty"); + len--; + + cli_buf_append(bopt->ldflags, buf, len); + cli_buf_appendf(bopt->ldflags, " "); + free(buf); } if (obopt != NULL && obopt->ldflags != NULL) { - kore_buf_append(bopt->ldflags, obopt->ldflags->data, + cli_buf_append(bopt->ldflags, obopt->ldflags->data, obopt->ldflags->offset); } - string = kore_buf_stringify(bopt->ldflags, NULL); + if ((env = getenv("LDFLAGS")) != NULL) + cli_buf_appendf(bopt->ldflags, "%s", env); + + string = cli_buf_stringify(bopt->ldflags, NULL); printf("LDFLAGS=%s\n", string); - ldflags_count = kore_split_string(string, " ", ldflags, LD_FLAGS_MAX); + ldflags_count = cli_split_string(string, " ", ldflags, LD_FLAGS_MAX); } static void cli_flavor_load(void) { FILE *fp; - char buf[BUFSIZ], pwd[MAXPATHLEN], *p, *conf; + char buf[BUFSIZ], pwd[MAXPATHLEN], *p, *conf, *env; + + if ((env = getenv("KORE_BUILD_FLAVOR")) != NULL) { + flavor = cli_strdup(env); + return; + } if (getcwd(pwd, sizeof(pwd)) == NULL) - cli_fatal("could not get cwd: %s", errno_s); + fatal("could not get cwd: %s", errno_s); appl = basename(pwd); if (appl == NULL) - cli_fatal("basename: %s", errno_s); - appl = kore_strdup(appl); + fatal("basename: %s", errno_s); + appl = cli_strdup(appl); (void)cli_vasprintf(&conf, "conf/%s.conf", appl); if (!cli_dir_exists("conf") || !cli_file_exists(conf)) - cli_fatal("%s doesn't appear to be a kore app", appl); + fatal("%s doesn't appear to be a kore app", appl); free(conf); if ((fp = fopen(".flavor", "r")) == NULL) { - flavor = kore_strdup("dev"); + flavor = cli_strdup("dev"); return; } if (fgets(buf, sizeof(buf), fp) == NULL) - cli_fatal("failed to read flavor from file"); + fatal("failed to read flavor from file"); if ((p = strchr(buf, '\n')) != NULL) *p = '\0'; - flavor = kore_strdup(buf); + flavor = cli_strdup(buf); (void)fclose(fp); } static void +cli_kore_features(struct buildopt *bopt, char **out, size_t *outlen) +{ + int fd; + size_t len; + char *path, *data; + + if (bopt->single_binary) { + (void)cli_vasprintf(&path, "%s/features", object_dir); + } else { + (void)cli_vasprintf(&path, "%s/share/kore/features", prefix); + } + + cli_file_open(path, O_RDONLY, &fd); + cli_file_read(fd, &data, &len); + cli_file_close(fd); + free(path); + + if (len == 0) + fatal("features is empty"); + + len--; + + *out = data; + *outlen = len; +} + +static void cli_flavor_change(const char *name) { FILE *fp; @@ -1603,19 +1957,19 @@ struct buildopt *bopt; if ((bopt = cli_buildopt_find(name)) == NULL) - cli_fatal("no such flavor: %s", name); + fatal("no such flavor: %s", name); if ((fp = fopen(".flavor.tmp", "w")) == NULL) - cli_fatal("failed to open temporary file to save flavor"); + fatal("failed to open temporary file to save flavor"); ret = fprintf(fp, "%s\n", name); if (ret == -1 || (size_t)ret != (strlen(name) + 1)) - cli_fatal("failed to write new build flavor"); + fatal("failed to write new build flavor"); (void)fclose(fp); if (rename(".flavor.tmp", ".flavor") == -1) - cli_fatal("failed to replace build flavor"); + fatal("failed to replace build flavor"); cli_clean(0, NULL); } @@ -1629,21 +1983,21 @@ pid = fork(); switch (pid) { case -1: - cli_fatal("cli_compile_cfile: fork() %s", errno_s); + fatal("cli_compile_cfile: fork() %s", errno_s); /* NOTREACHED */ case 0: cb(arg); - cli_fatal("cli_spawn_proc: %s", errno_s); + fatal("cli_spawn_proc: %s", errno_s); /* NOTREACHED */ default: break; } if (waitpid(pid, &status, 0) == -1) - cli_fatal("couldn't wait for child %d", pid); + fatal("couldn't wait for child %d", pid); if (WEXITSTATUS(status) || WTERMSIG(status) || WCOREDUMP(status)) - cli_fatal("subprocess trouble, check output"); + fatal("subprocess trouble, check output"); } static int @@ -1657,7 +2011,7 @@ va_end(args); if (l == -1) - cli_fatal("cli_vasprintf"); + fatal("cli_vasprintf"); return (l); } @@ -1671,8 +2025,226 @@ printf("couldn't rmdir %s\n", spath); } +static void * +cli_malloc(size_t len) +{ + void *ptr; + + if ((ptr = calloc(1, len)) == NULL) + fatal("calloc: %s", errno_s); + + return (ptr); +} + +static void * +cli_realloc(void *ptr, size_t len) +{ + void *nptr; + + if ((nptr = realloc(ptr, len)) == NULL) + fatal("realloc: %s", errno_s); + + return (nptr); +} + +static char * +cli_strdup(const char *string) +{ + char *copy; + + if ((copy = strdup(string)) == NULL) + fatal("strdup: %s", errno_s); + + return (copy); +} + +struct cli_buf * +cli_buf_alloc(size_t initial) +{ + struct cli_buf *buf; + + buf = cli_malloc(sizeof(*buf)); + + if (initial > 0) + buf->data = cli_malloc(initial); + else + buf->data = NULL; + + buf->length = initial; + buf->offset = 0; + + return (buf); +} + +void +cli_buf_free(struct cli_buf *buf) +{ + free(buf->data); + buf->data = NULL; + buf->offset = 0; + buf->length = 0; + free(buf); +} + +void +cli_buf_append(struct cli_buf *buf, const void *d, size_t len) +{ + if ((buf->offset + len) < len) + fatal("overflow in cli_buf_append"); + + if ((buf->offset + len) > buf->length) { + buf->length += len; + buf->data = cli_realloc(buf->data, buf->length); + } + + memcpy((buf->data + buf->offset), d, len); + buf->offset += len; +} + +void +cli_buf_appendv(struct cli_buf *buf, const char *fmt, va_list args) +{ + int l; + va_list copy; + char *b, sb[BUFSIZ]; + + va_copy(copy, args); + + l = vsnprintf(sb, sizeof(sb), fmt, args); + if (l == -1) + fatal("cli_buf_appendv(): vsnprintf error"); + + if ((size_t)l >= sizeof(sb)) { + l = vasprintf(&b, fmt, copy); + if (l == -1) + fatal("cli_buf_appendv(): error or truncation"); + } else { + b = sb; + } + + cli_buf_append(buf, b, l); + if (b != sb) + free(b); + + va_end(copy); +} + +void +cli_buf_appendf(struct cli_buf *buf, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + cli_buf_appendv(buf, fmt, args); + va_end(args); +} + +char * +cli_buf_stringify(struct cli_buf *buf, size_t *len) +{ + char c; + + if (len != NULL) + *len = buf->offset; + + c = '\0'; + cli_buf_append(buf, &c, sizeof(c)); + + return ((char *)buf->data); +} + +static int +cli_split_string(char *input, const char *delim, char **out, size_t ele) +{ + int count; + char **ap; + + if (ele == 0) + return (0); + + count = 0; + for (ap = out; ap < &out[ele - 1] && + (*ap = strsep(&input, delim)) != NULL;) { + if (**ap != '\0') { + ap++; + count++; + } + } + + *ap = NULL; + return (count); +} + +static char * +cli_read_line(FILE *fp, char *in, size_t len) +{ + char *p, *t; + + if (fgets(in, len, fp) == NULL) + return (NULL); + + p = in; + in[strcspn(in, "\n")] = '\0'; + + while (isspace(*(unsigned char *)p)) + p++; + + if (p[0] == '#' || p[0] == '\0') { + p[0] = '\0'; + return (p); + } + + for (t = p; *t != '\0'; t++) { + if (*t == '\t') + *t = ' '; + } + + return (p); +} + +static char * +cli_text_trim(char *string, size_t len) +{ + char *end; + + if (len == 0) + return (string); + + end = (string + len) - 1; + while (isspace(*(unsigned char *)string) && string < end) + string++; + + while (isspace(*(unsigned char *)end) && end > string) + *(end)-- = '\0'; + + return (string); +} + +static long long +cli_strtonum(const char *str, long long min, long long max) +{ + long long l; + char *ep; + + if (min > max) + fatal("cli_strtonum: min > max"); + + errno = 0; + l = strtoll(str, &ep, 10); + if (errno != 0 || str == ep || *ep != '\0') + fatal("strtoll(): %s", errno_s); + + if (l < min) + fatal("cli_strtonum: value < min"); + + if (l > max) + fatal("cli_strtonum: value > max"); + + return (l); +} + static void -cli_fatal(const char *fmt, ...) +fatal(const char *fmt, ...) { va_list args; char buf[2048]; diff -Nru kore-2.0.0/src/config.c kore-3.3.1/src/config.c --- kore-2.0.0/src/config.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/config.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,10 +15,12 @@ */ #include +#include #include #include #include +#include #include #include #include @@ -35,11 +37,19 @@ #include "tasks.h" #endif +#if defined(KORE_USE_PYTHON) +#include "python_api.h" +#endif + +#if defined(KORE_USE_CURL) +#include "curl.h" +#endif + /* XXX - This is becoming a clusterfuck. Fix it. */ -#if !defined(KORE_SINGLE_BINARY) static int configure_load(char *); -#else + +#if defined(KORE_SINGLE_BINARY) static FILE *config_file_write(void); extern u_int8_t asset_builtin_kore_conf[]; extern u_int32_t asset_len_builtin_kore_conf; @@ -47,35 +57,53 @@ static int configure_include(char *); static int configure_bind(char *); +static int configure_bind_unix(char *); static int configure_domain(char *); -static int configure_chroot(char *); +static int configure_root(char *); static int configure_runas(char *); static int configure_workers(char *); static int configure_pidfile(char *); static int configure_rlimit_nofiles(char *); static int configure_max_connections(char *); static int configure_accept_threshold(char *); +static int configure_death_policy(char *); static int configure_set_affinity(char *); static int configure_socket_backlog(char *); +#if defined(KORE_USE_PLATFORM_PLEDGE) +static int configure_add_pledge(char *); +#endif + #if !defined(KORE_NO_TLS) +static int configure_rand_file(char *); static int configure_certfile(char *); static int configure_certkey(char *); static int configure_tls_version(char *); static int configure_tls_cipher(char *); static int configure_tls_dhparam(char *); -static int configure_client_certificates(char *); +static int configure_keymgr_root(char *); +static int configure_keymgr_runas(char *); +static int configure_client_verify(char *); +static int configure_client_verify_depth(char *); #endif #if !defined(KORE_NO_HTTP) +static int configure_filemap(char *); +static int configure_restrict(char *); static int configure_handler(int, char *); static int configure_static_handler(char *); static int configure_dynamic_handler(char *); static int configure_accesslog(char *); static int configure_http_header_max(char *); +static int configure_http_header_timeout(char *); static int configure_http_body_max(char *); +static int configure_http_body_timeout(char *); +static int configure_filemap_ext(char *); +static int configure_filemap_index(char *); +static int configure_http_media_type(char *); static int configure_http_hsts_enable(char *); static int configure_http_keepalive_time(char *); +static int configure_http_request_ms(char *); static int configure_http_request_limit(char *); static int configure_http_body_disk_offload(char *); static int configure_http_body_disk_path(char *); @@ -93,14 +121,22 @@ #if defined(KORE_USE_PGSQL) static int configure_pgsql_conn_max(char *); +static int configure_pgsql_queue_limit(char *); #endif #if defined(KORE_USE_TASKS) static int configure_task_threads(char *); #endif -static void domain_sslstart(void); -static void kore_parse_config_file(const char *); +#if defined(KORE_USE_PYTHON) +static int configure_python_path(char *); +static int configure_python_import(char *); +#endif + +#if defined(KORE_USE_CURL) +static int configure_curl_timeout(char *); +static int configure_curl_recv_max(char *); +#endif static struct { const char *name; @@ -108,35 +144,55 @@ } config_names[] = { { "include", configure_include }, { "bind", configure_bind }, -#if !defined(KORE_SINGLE_BINARY) + { "bind_unix", configure_bind_unix }, { "load", configure_load }, +#if defined(KORE_USE_PYTHON) + { "python_path", configure_python_path }, + { "python_import", configure_python_import }, #endif + { "root", configure_root }, + { "chroot", configure_root }, { "domain", configure_domain }, - { "chroot", configure_chroot }, { "runas", configure_runas }, { "workers", configure_workers }, { "worker_max_connections", configure_max_connections }, { "worker_rlimit_nofiles", configure_rlimit_nofiles }, { "worker_accept_threshold", configure_accept_threshold }, + { "worker_death_policy", configure_death_policy }, { "worker_set_affinity", configure_set_affinity }, { "pidfile", configure_pidfile }, { "socket_backlog", configure_socket_backlog }, +#if defined(KORE_USE_PLATFORM_PLEDGE) + { "pledge", configure_add_pledge }, +#endif #if !defined(KORE_NO_TLS) { "tls_version", configure_tls_version }, { "tls_cipher", configure_tls_cipher }, { "tls_dhparam", configure_tls_dhparam }, + { "rand_file", configure_rand_file }, + { "keymgr_runas", configure_keymgr_runas }, + { "keymgr_root", configure_keymgr_root }, { "certfile", configure_certfile }, { "certkey", configure_certkey }, - { "client_certificates", configure_client_certificates }, + { "client_verify", configure_client_verify }, + { "client_verify_depth", configure_client_verify_depth }, #endif #if !defined(KORE_NO_HTTP) + { "filemap", configure_filemap }, + { "filemap_ext", configure_filemap_ext }, + { "filemap_index", configure_filemap_index }, { "static", configure_static_handler }, { "dynamic", configure_dynamic_handler }, { "accesslog", configure_accesslog }, + { "restrict", configure_restrict }, + { "http_media_type", configure_http_media_type }, { "http_header_max", configure_http_header_max }, + { "http_header_timeout", configure_http_header_timeout }, { "http_body_max", configure_http_body_max }, + { "http_body_timeout", configure_http_body_timeout }, { "http_hsts_enable", configure_http_hsts_enable }, { "http_keepalive_time", configure_http_keepalive_time }, + { "http_request_ms", configure_http_request_ms }, { "http_request_limit", configure_http_request_limit }, { "http_body_disk_offload", configure_http_body_disk_offload }, { "http_body_disk_path", configure_http_body_disk_path }, @@ -153,10 +209,15 @@ #endif #if defined(KORE_USE_PGSQL) { "pgsql_conn_max", configure_pgsql_conn_max }, + { "pgsql_queue_limit", configure_pgsql_queue_limit }, #endif #if defined(KORE_USE_TASKS) { "task_threads", configure_task_threads }, #endif +#if defined(KORE_USE_CURL) + { "curl_timeout", configure_curl_timeout }, + { "curl_recv_max", configure_curl_recv_max }, +#endif { NULL, NULL }, }; @@ -166,6 +227,7 @@ #if !defined(KORE_NO_HTTP) static u_int8_t current_method = 0; +static int current_flags = 0; static struct kore_auth *current_auth = NULL; static struct kore_module_handle *current_handler = NULL; #endif @@ -176,48 +238,80 @@ void kore_parse_config(void) { + FILE *fp; + char path[PATH_MAX]; + #if !defined(KORE_SINGLE_BINARY) - kore_parse_config_file(config_file); + if ((fp = fopen(config_file, "r")) == NULL) + fatal("configuration given cannot be opened: %s", config_file); #else - kore_parse_config_file(NULL); + fp = config_file_write(); #endif + kore_parse_config_file(fp); + (void)fclose(fp); + if (!kore_module_loaded()) fatal("no application module was loaded"); - if (skip_chroot != 1 && chroot_path == NULL) { - fatal("missing a chroot path"); + if (kore_root_path == NULL) { + if (getcwd(path, sizeof(path)) == NULL) + fatal("getcwd: %s", errno_s); + kore_root_path = kore_strdup(path); + + if (!kore_quiet) { + kore_log(LOG_NOTICE, "privsep: no root path set, " + "using working directory"); + } } if (getuid() != 0 && skip_chroot == 0) { fatal("cannot chroot, use -n to skip it"); } - if (skip_runas != 1 && runas_user == NULL) { + if (skip_runas != 1 && kore_runas_user == NULL) { fatal("missing runas user, use -r to skip it"); } if (getuid() != 0 && skip_runas == 0) { fatal("cannot drop privileges, use -r to skip it"); } + + if (skip_runas) { + if (!kore_quiet) + kore_log(LOG_WARNING, "privsep: will not change user"); + } else { +#if !defined(KORE_NO_TLS) + if (keymgr_runas_user == NULL) { + if (!kore_quiet) { + kore_log(LOG_NOTICE, "privsep: no keymgr_runas " + "set, using 'runas` user"); + } + keymgr_runas_user = kore_strdup(kore_runas_user); + } +#endif + } + +#if !defined(KORE_NO_TLS) + if (keymgr_root_path == NULL) { + if (!kore_quiet) { + kore_log(LOG_NOTICE, "privsep: no keymgr_root set, " + "using 'root` directory"); + } + keymgr_root_path = kore_strdup(kore_root_path); + } +#endif + + if (skip_chroot && !kore_quiet) + kore_log(LOG_WARNING, "privsep: will not chroot"); } -static void -kore_parse_config_file(const char *fpath) +void +kore_parse_config_file(FILE *fp) { - FILE *fp; int i, lineno; char buf[BUFSIZ], *p, *t; -#if !defined(KORE_SINGLE_BINARY) - if ((fp = fopen(fpath, "r")) == NULL) - fatal("configuration given cannot be opened: %s", fpath); -#else - fp = config_file_write(); -#endif - - kore_debug("parsing configuration file '%s'", fpath); - lineno = 1; while ((p = kore_read_line(fp, buf, sizeof(buf))) != NULL) { if (strlen(p) == 0) { @@ -228,6 +322,8 @@ #if !defined(KORE_NO_HTTP) if (!strcmp(p, "}") && current_handler != NULL) { lineno++; + current_flags = 0; + current_method = 0; current_handler = NULL; continue; } @@ -244,8 +340,16 @@ } #endif - if (!strcmp(p, "}") && current_domain != NULL) - domain_sslstart(); + if (!strcmp(p, "}") && current_domain != NULL) { +#if !defined(KORE_NO_TLS) + if (current_domain->certfile == NULL || + current_domain->certkey == NULL) { + fatal("incomplete TLS setup for '%s'", + current_domain->domain); + } +#endif + current_domain = NULL; + } if (!strcmp(p, "}")) { lineno++; @@ -280,14 +384,19 @@ printf("ignoring \"%s\" on line %d\n", p, lineno); lineno++; } - - fclose(fp); } static int configure_include(char *path) { - kore_parse_config_file(path); + FILE *fp; + + if ((fp = fopen(path, "r")) == NULL) + fatal("failed to open include '%s'", path); + + kore_parse_config_file(fp); + (void)fclose(fp); + return (KORE_RESULT_OK); } @@ -303,7 +412,18 @@ return (kore_server_bind(argv[0], argv[1], argv[2])); } -#if !defined(KORE_SINGLE_BINARY) +static int +configure_bind_unix(char *options) +{ + char *argv[3]; + + kore_split_string(options, " ", argv, 3); + if (argv[0] == NULL) + return (KORE_RESULT_ERROR); + + return (kore_server_bind_unix(argv[0], argv[1])); +} + static int configure_load(char *options) { @@ -313,10 +433,11 @@ if (argv[0] == NULL) return (KORE_RESULT_ERROR); - kore_module_load(argv[0], argv[1]); + kore_module_load(argv[0], argv[1], KORE_MODULE_NATIVE); return (KORE_RESULT_OK); } -#else + +#if defined(KORE_SINGLE_BINARY) static FILE * config_file_write(void) { @@ -361,10 +482,10 @@ static int configure_tls_version(char *version) { - if (!strcmp(version, "1.2")) { + if (!strcmp(version, "1.3")) { + tls_version = KORE_TLS_VERSION_1_3; + } else if (!strcmp(version, "1.2")) { tls_version = KORE_TLS_VERSION_1_2; - } else if (!strcmp(version, "1.0")) { - tls_version = KORE_TLS_VERSION_1_0; } else if (!strcmp(version, "both")) { tls_version = KORE_TLS_VERSION_BOTH; } else { @@ -414,23 +535,44 @@ } static int -configure_client_certificates(char *options) +configure_client_verify_depth(char *value) +{ + int err, depth; + + if (current_domain == NULL) { + printf("client_verify_depth not specified in domain context\n"); + return (KORE_RESULT_ERROR); + } + + depth = kore_strtonum(value, 10, 0, INT_MAX, &err); + if (err != KORE_RESULT_OK) { + printf("bad client_verify_depth value: %s\n", value); + return (KORE_RESULT_ERROR); + } + + current_domain->x509_verify_depth = depth; + + return (KORE_RESULT_OK); +} + +static int +configure_client_verify(char *options) { char *argv[3]; if (current_domain == NULL) { - printf("client_certificates not specified in domain context\n"); + printf("client_verify not specified in domain context\n"); return (KORE_RESULT_ERROR); } kore_split_string(options, " ", argv, 3); if (argv[0] == NULL) { - printf("client_certificate is missing a parameter\n"); + printf("client_verify is missing a parameter\n"); return (KORE_RESULT_ERROR); } if (current_domain->cafile != NULL) { - printf("client_certificate already set for %s\n", + printf("client_verify already set for %s\n", current_domain->domain); return (KORE_RESULT_ERROR); } @@ -443,6 +585,17 @@ } static int +configure_rand_file(char *path) +{ + if (rand_file != NULL) + kore_free(rand_file); + + rand_file = kore_strdup(path); + + return (KORE_RESULT_OK); +} + +static int configure_certfile(char *path) { if (current_domain == NULL) { @@ -478,6 +631,26 @@ return (KORE_RESULT_OK); } +static int +configure_keymgr_runas(char *user) +{ + if (keymgr_runas_user != NULL) + kore_free(keymgr_runas_user); + keymgr_runas_user = kore_strdup(user); + + return (KORE_RESULT_OK); +} + +static int +configure_keymgr_root(char *root) +{ + if (keymgr_root_path != NULL) + kore_free(keymgr_root_path); + keymgr_root_path = kore_strdup(root); + + return (KORE_RESULT_OK); +} + #endif /* !KORE_NO_TLS */ static int @@ -551,6 +724,31 @@ } static int +configure_filemap(char *options) +{ + char *argv[3]; + + if (current_domain == NULL) { + printf("filemap outside of domain context\n"); + return (KORE_RESULT_ERROR); + } + + kore_split_string(options, " ", argv, 3); + + if (argv[0] == NULL || argv[1] == NULL) { + printf("missing parameters for filemap\n"); + return (KORE_RESULT_ERROR); + } + + if (!kore_filemap_create(current_domain, argv[1], argv[0])) { + printf("cannot create filemap for %s\n", argv[1]); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int configure_accesslog(char *path) { if (current_domain == NULL) { @@ -576,6 +774,108 @@ } static int +configure_restrict(char *options) +{ + struct kore_module_handle *hdlr; + int i, cnt; + char *argv[10]; + + if (current_domain == NULL) { + printf("restrict not used in domain context\n"); + return (KORE_RESULT_ERROR); + } + + cnt = kore_split_string(options, " ", argv, 10); + if (cnt < 2) { + printf("bad restrict option '%s', missing methods\n", options); + return (KORE_RESULT_ERROR); + } + + hdlr = NULL; + TAILQ_FOREACH(hdlr, &(current_domain->handlers), list) { + if (!strcmp(hdlr->path, argv[0])) + break; + } + + if (hdlr == NULL) { + printf("bad restrict option handler '%s' not found\n", argv[0]); + return (KORE_RESULT_ERROR); + } + + hdlr->methods = 0; + + for (i = 1; i < cnt; i++) { + if (!strcasecmp(argv[i], "post")) { + hdlr->methods |= HTTP_METHOD_POST; + } else if (!strcasecmp(argv[i], "get")) { + hdlr->methods |= HTTP_METHOD_GET; + } else if (!strcasecmp(argv[i], "put")) { + hdlr->methods |= HTTP_METHOD_PUT; + } else if (!strcasecmp(argv[i], "delete")) { + hdlr->methods |= HTTP_METHOD_DELETE; + } else if (!strcasecmp(argv[i], "head")) { + hdlr->methods |= HTTP_METHOD_HEAD; + } else if (!strcasecmp(argv[i], "patch")) { + hdlr->methods |= HTTP_METHOD_PATCH; + } else { + printf("unknown method: %s in restrict for %s\n", + argv[i], argv[0]); + return (KORE_RESULT_ERROR); + } + } + + return (KORE_RESULT_OK); +} + +static int +configure_filemap_ext(char *ext) +{ + kore_free(kore_filemap_ext); + kore_filemap_ext = kore_strdup(ext); + + return (KORE_RESULT_OK); +} + +static int +configure_filemap_index(char *index) +{ + kore_free(kore_filemap_index); + kore_filemap_index = kore_strdup(index); + + return (KORE_RESULT_OK); +} + +static int +configure_http_media_type(char *type) +{ + int i; + char *extensions, *ext[10]; + + extensions = strchr(type, ' '); + if (extensions == NULL) { + printf("bad http_media_type value: %s\n", type); + return (KORE_RESULT_ERROR); + } + + *(extensions)++ = '\0'; + + kore_split_string(extensions, " \t", ext, 10); + for (i = 0; ext[i] != NULL; i++) { + if (!http_media_register(ext[i], type)) { + printf("duplicate extension found: %s\n", ext[i]); + return (KORE_RESULT_ERROR); + } + } + + if (i == 0) { + printf("missing extensions in: %s\n", type); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int configure_http_header_max(char *option) { int err; @@ -590,6 +890,20 @@ } static int +configure_http_header_timeout(char *option) +{ + int err; + + http_header_timeout = kore_strtonum(option, 10, 1, 65535, &err); + if (err != KORE_RESULT_OK) { + printf("bad http_header_timeout value: %s\n", option); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int configure_http_body_max(char *option) { int err; @@ -604,6 +918,20 @@ } static int +configure_http_body_timeout(char *option) +{ + int err; + + http_body_timeout = kore_strtonum(option, 10, 1, 65535, &err); + if (err != KORE_RESULT_OK) { + printf("bad http_body_timeout value: %s\n", option); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int configure_http_body_disk_offload(char *option) { int err; @@ -656,6 +984,20 @@ } static int +configure_http_request_ms(char *option) +{ + int err; + + http_request_ms = kore_strtonum(option, 10, 0, UINT_MAX, &err); + if (err != KORE_RESULT_OK) { + printf("bad http_request_ms value: %s\n", option); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int configure_http_request_limit(char *option) { int err; @@ -711,7 +1053,7 @@ configure_params(char *options) { struct kore_module_handle *hdlr; - char *argv[3]; + char *method, *argv[3]; if (current_domain == NULL) { printf("params not used in domain context\n"); @@ -727,19 +1069,36 @@ if (argv[1] == NULL) return (KORE_RESULT_ERROR); - if (!strcasecmp(argv[0], "post")) { + if ((method = strchr(argv[0], ':')) != NULL) { + *(method)++ = '\0'; + if (!strcasecmp(argv[0], "qs")) { + current_flags = KORE_PARAMS_QUERY_STRING; + } else { + printf("unknown prefix '%s' for '%s'\n", + argv[0], argv[1]); + return (KORE_RESULT_ERROR); + } + } else { + method = argv[0]; + } + + if (!strcasecmp(method, "post")) { current_method = HTTP_METHOD_POST; - } else if (!strcasecmp(argv[0], "get")) { + } else if (!strcasecmp(method, "get")) { current_method = HTTP_METHOD_GET; - } else if (!strcasecmp(argv[0], "put")) { + /* Let params get /foo {} imply qs:get automatically. */ + current_flags |= KORE_PARAMS_QUERY_STRING; + } else if (!strcasecmp(method, "put")) { current_method = HTTP_METHOD_PUT; - } else if (!strcasecmp(argv[0], "delete")) { + } else if (!strcasecmp(method, "delete")) { current_method = HTTP_METHOD_DELETE; - } else if (!strcasecmp(argv[0], "head")) { + } else if (!strcasecmp(method, "head")) { current_method = HTTP_METHOD_HEAD; + } else if (!strcasecmp(method, "patch")) { + current_method = HTTP_METHOD_PATCH; } else { printf("unknown method: %s in params block for %s\n", - argv[0], argv[1]); + method, argv[1]); return (KORE_RESULT_ERROR); } @@ -781,6 +1140,7 @@ p = kore_malloc(sizeof(*p)); p->validator = val; + p->flags = current_flags; p->method = current_method; p->name = kore_strdup(argv[0]); @@ -922,11 +1282,11 @@ #endif /* !KORE_NO_HTTP */ static int -configure_chroot(char *path) +configure_root(char *path) { - if (chroot_path != NULL) - kore_free(chroot_path); - chroot_path = kore_strdup(path); + if (kore_root_path != NULL) + kore_free(kore_root_path); + kore_root_path = kore_strdup(path); return (KORE_RESULT_OK); } @@ -934,9 +1294,9 @@ static int configure_runas(char *user) { - if (runas_user != NULL) - kore_free(runas_user); - runas_user = kore_strdup(user); + if (kore_runas_user != NULL) + kore_free(kore_runas_user); + kore_runas_user = kore_strdup(user); return (KORE_RESULT_OK); } @@ -1008,6 +1368,21 @@ } static int +configure_death_policy(char *option) +{ + if (!strcmp(option, "restart")) { + worker_policy = KORE_WORKER_POLICY_RESTART; + } else if (!strcmp(option, "terminate")) { + worker_policy = KORE_WORKER_POLICY_TERMINATE; + } else { + printf("bad value for worker_death_policy: %s\n", option); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int configure_set_affinity(char *option) { int err; @@ -1035,13 +1410,6 @@ return (KORE_RESULT_OK); } -static void -domain_sslstart(void) -{ - kore_domain_sslstart(current_domain); - current_domain = NULL; -} - #if defined(KORE_USE_PGSQL) static int configure_pgsql_conn_max(char *option) @@ -1056,6 +1424,20 @@ return (KORE_RESULT_OK); } + +static int +configure_pgsql_queue_limit(char *option) +{ + int err; + + pgsql_queue_limit = kore_strtonum(option, 10, 0, UINT_MAX, &err); + if (err != KORE_RESULT_OK) { + printf("bad value for pgsql_queue_limit: %s\n", option); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} #endif #if defined(KORE_USE_TASKS) @@ -1070,6 +1452,69 @@ return (KORE_RESULT_ERROR); } + return (KORE_RESULT_OK); +} +#endif + +#if defined(KORE_USE_PYTHON) +static int +configure_python_path(char *path) +{ + kore_python_path(path); + + return (KORE_RESULT_OK); +} + +static int +configure_python_import(char *module) +{ + char *argv[3]; + + kore_split_string(module, " ", argv, 3); + if (argv[0] == NULL) + return (KORE_RESULT_ERROR); + + kore_module_load(argv[0], argv[1], KORE_MODULE_PYTHON); + return (KORE_RESULT_OK); +} +#endif + +#if defined(KORE_USE_PLATFORM_PLEDGE) +static int +configure_add_pledge(char *pledge) +{ + kore_platform_add_pledge(pledge); + + return (KORE_RESULT_OK); +} +#endif + +#if defined(KORE_USE_CURL) +static int +configure_curl_recv_max(char *option) +{ + int err; + + kore_curl_recv_max = kore_strtonum64(option, 1, &err); + if (err != KORE_RESULT_OK) { + printf("bad curl_recv_max value: %s\n", option); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int +configure_curl_timeout(char *option) +{ + int err; + + kore_curl_timeout = kore_strtonum(option, 10, 0, USHRT_MAX, &err); + if (err != KORE_RESULT_OK) { + printf("bad kore_curl_timeout value: %s\n", option); + return (KORE_RESULT_ERROR); + } + return (KORE_RESULT_OK); } #endif diff -Nru kore-2.0.0/src/connection.c kore-3.3.1/src/connection.c --- kore-2.0.0/src/connection.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/connection.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -19,6 +19,7 @@ #include +#include #include #include "kore.h" @@ -31,11 +32,16 @@ void kore_connection_init(void) { + u_int32_t elm; + TAILQ_INIT(&connections); TAILQ_INIT(&disconnected); + /* Add some overhead so we don't rollover for internal items. */ + elm = worker_max_connections + 10; + kore_pool_init(&connection_pool, "connection_pool", - sizeof(struct connection), worker_max_connections); + sizeof(struct connection), elm); } void @@ -68,12 +74,18 @@ c->disconnect = NULL; c->hdlr_extra = NULL; c->proto = CONN_PROTO_UNKNOWN; - c->type = KORE_TYPE_CONNECTION; c->idle_timer.start = 0; c->idle_timer.length = KORE_IDLE_TIMER_MAX; + c->evt.type = KORE_TYPE_CONNECTION; + c->evt.handle = kore_connection_event; + #if !defined(KORE_NO_HTTP) - c->wscbs = NULL; + c->ws_connect = NULL; + c->ws_message = NULL; + c->ws_disconnect = NULL; + c->http_start = kore_time_ms(); + c->http_timeout = http_header_timeout * 1000; TAILQ_INIT(&(c->http_requests)); #endif @@ -86,7 +98,7 @@ kore_connection_accept(struct listener *listener, struct connection **out) { struct connection *c; - struct sockaddr *sin; + struct sockaddr *s; socklen_t len; kore_debug("kore_connection_accept(%p)", listener); @@ -94,22 +106,38 @@ *out = NULL; c = kore_connection_new(listener); - c->addrtype = listener->addrtype; - if (c->addrtype == AF_INET) { + c->family = listener->family; + + switch (c->family) { + case AF_INET: len = sizeof(struct sockaddr_in); - sin = (struct sockaddr *)&(c->addr.ipv4); - } else { + s = (struct sockaddr *)&(c->addr.ipv4); + break; + case AF_INET6: len = sizeof(struct sockaddr_in6); - sin = (struct sockaddr *)&(c->addr.ipv6); + s = (struct sockaddr *)&(c->addr.ipv6); + break; + case AF_UNIX: + len = sizeof(struct sockaddr_un); + s = (struct sockaddr *)&(c->addr.sun); + break; + default: + fatal("unknown family type %d", c->family); } - if ((c->fd = accept(listener->fd, sin, &len)) == -1) { + if ((c->fd = accept(listener->fd, s, &len)) == -1) { kore_pool_put(&connection_pool, c); kore_debug("accept(): %s", errno_s); return (KORE_RESULT_ERROR); } - if (!kore_connection_nonblock(c->fd, 1)) { + if (!kore_connection_nonblock(c->fd, listener->family != AF_UNIX)) { + close(c->fd); + kore_pool_put(&connection_pool, c); + return (KORE_RESULT_ERROR); + } + + if (fcntl(c->fd, F_SETFD, FD_CLOEXEC) == -1) { close(c->fd); kore_pool_put(&connection_pool, c); return (KORE_RESULT_ERROR); @@ -119,16 +147,16 @@ TAILQ_INSERT_TAIL(&connections, c, list); #if !defined(KORE_NO_TLS) - c->state = CONN_STATE_SSL_SHAKE; - c->write = net_write_ssl; - c->read = net_read_ssl; + c->state = CONN_STATE_TLS_SHAKE; + c->write = net_write_tls; + c->read = net_read_tls; #else c->state = CONN_STATE_ESTABLISHED; c->write = net_write; c->read = net_read; if (listener->connect != NULL) { - listener->connect(c); + kore_runtime_connect(listener->connect, c); } else { #if !defined(KORE_NO_HTTP) c->proto = CONN_PROTO_HTTP; @@ -141,24 +169,29 @@ #endif kore_connection_start_idletimer(c); + worker_active_connections++; *out = c; return (KORE_RESULT_OK); } void -kore_connection_check_timeout(void) +kore_connection_check_timeout(u_int64_t now) { - struct connection *c; - u_int64_t now; + struct connection *c, *next; - now = kore_time_ms(); - TAILQ_FOREACH(c, &connections, list) { + for (c = TAILQ_FIRST(&connections); c != NULL; c = next) { + next = TAILQ_NEXT(c, list); if (c->proto == CONN_PROTO_MSG) continue; - if (!(c->flags & CONN_IDLE_TIMER_ACT)) +#if !defined(KORE_NO_HTTP) + if (!http_check_timeout(c, now)) + continue; + if (!TAILQ_EMPTY(&c->http_requests)) continue; - kore_connection_check_idletimer(now, c); +#endif + if (c->flags & CONN_IDLE_TIMER_ACT) + kore_connection_check_idletimer(now, c); } } @@ -196,6 +229,20 @@ } } +void +kore_connection_event(void *arg, int error) +{ + struct connection *c = arg; + + if (error) { + kore_connection_disconnect(c); + return; + } + + if (!c->handle(c)) + kore_connection_disconnect(c); +} + int kore_connection_handle(struct connection *c) { @@ -210,7 +257,14 @@ switch (c->state) { #if !defined(KORE_NO_TLS) - case CONN_STATE_SSL_SHAKE: + case CONN_STATE_TLS_SHAKE: + if (primary_dom->ssl_ctx == NULL) { + kore_log(LOG_NOTICE, + "TLS configuration for %s not yet complete", + primary_dom->domain); + return (KORE_RESULT_ERROR); + } + if (c->ssl == NULL) { c->ssl = SSL_new(primary_dom->ssl_ctx); if (c->ssl == NULL) { @@ -223,12 +277,14 @@ SSL_set_app_data(c->ssl, c); } + ERR_clear_error(); r = SSL_accept(c->ssl); if (r <= 0) { r = SSL_get_error(c->ssl, r); switch (r) { case SSL_ERROR_WANT_READ: case SSL_ERROR_WANT_WRITE: + kore_connection_start_idletimer(c); return (KORE_RESULT_OK); default: kore_debug("SSL_accept(): %s", ssl_errno_s); @@ -253,17 +309,11 @@ c->cert = NULL; } - r = SSL_get_verify_result(c->ssl); - if (r != X509_V_OK) { - kore_debug("SSL_get_verify_result(): %d, %s", - r, ssl_errno_s); - return (KORE_RESULT_ERROR); - } - if (c->owner != NULL) { listener = (struct listener *)c->owner; if (listener->connect != NULL) { - listener->connect(c); + kore_runtime_connect(listener->connect, c); + kore_connection_start_idletimer(c); return (KORE_RESULT_OK); } } @@ -283,12 +333,12 @@ /* FALLTHROUGH */ #endif /* !KORE_NO_TLS */ case CONN_STATE_ESTABLISHED: - if (c->flags & CONN_READ_POSSIBLE) { + if (c->evt.flags & KORE_EVENT_READ) { if (!net_recv_flush(c)) return (KORE_RESULT_ERROR); } - if (c->flags & CONN_WRITE_POSSIBLE) { + if (c->evt.flags & KORE_EVENT_WRITE) { if (!net_send_flush(c)) return (KORE_RESULT_ERROR); } @@ -331,23 +381,24 @@ kore_free(c->hdlr_extra); #if !defined(KORE_NO_HTTP) - for (req = TAILQ_FIRST(&(c->http_requests)); req != NULL; req = rnext) { + for (req = TAILQ_FIRST(&(c->http_requests)); + req != NULL; req = rnext) { rnext = TAILQ_NEXT(req, olist); TAILQ_REMOVE(&(c->http_requests), req, olist); + req->owner = NULL; req->flags |= HTTP_REQUEST_DELETE; http_request_wakeup(req); } + + kore_free(c->ws_connect); + kore_free(c->ws_message); + kore_free(c->ws_disconnect); #endif for (nb = TAILQ_FIRST(&(c->send_queue)); nb != NULL; nb = next) { next = TAILQ_NEXT(nb, list); - TAILQ_REMOVE(&(c->send_queue), nb, list); - if (!(nb->flags & NETBUF_IS_STREAM)) { - kore_free(nb->buf); - } else if (nb->cb != NULL) { - (void)nb->cb(nb); - } - kore_pool_put(&nb_pool, nb); + nb->flags &= ~NETBUF_MUST_RESEND; + net_remove_netbuf(c, nb); } if (c->rnb != NULL) { @@ -356,6 +407,7 @@ } kore_pool_put(&connection_pool, c); + worker_active_connections--; } void @@ -363,9 +415,13 @@ { u_int64_t d; - d = now - c->idle_timer.start; + if (now > c->idle_timer.start) + d = now - c->idle_timer.start; + else + d = 0; + if (d >= c->idle_timer.length) { - kore_debug("%p idle for %d ms, expiring", c, d); + kore_debug("%p idle for %" PRIu64 " ms, expiring", c, d); kore_connection_disconnect(c); } } @@ -407,9 +463,7 @@ } if (nodelay) { - flags = 1; - if (setsockopt(fd, IPPROTO_TCP, - TCP_NODELAY, (char *)&flags, sizeof(flags)) == -1) { + if (!kore_sockopt(fd, IPPROTO_TCP, TCP_NODELAY)) { kore_log(LOG_NOTICE, "failed to set TCP_NODELAY on %d", fd); } diff -Nru kore-2.0.0/src/curl.c kore-3.3.1/src/curl.c --- kore-2.0.0/src/curl.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/src/curl.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,655 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include + +#include + +#include "kore.h" +#include "http.h" +#include "curl.h" + +#define FD_CACHE_BUCKETS 2048 + +struct fd_cache { + struct kore_event evt; + int fd; + int scheduled; + LIST_ENTRY(fd_cache) list; +}; + +static void curl_process(void); +static void curl_event_handle(void *, int); +static int curl_timer(CURLM *, long, void *); +static int curl_socket(CURL *, curl_socket_t, int, void *, void *); + +static struct fd_cache *fd_cache_get(int); + +static int running = 0; +static CURLM *multi = NULL; +static struct kore_timer *timer = NULL; +static struct kore_pool fd_cache_pool; +static char user_agent[64]; +static LIST_HEAD(, fd_cache) cache[FD_CACHE_BUCKETS]; + +u_int16_t kore_curl_timeout = KORE_CURL_TIMEOUT; +u_int64_t kore_curl_recv_max = KORE_CURL_RECV_MAX; + +void +kore_curl_sysinit(void) +{ + CURLMcode res; + int i, len; + + if (curl_global_init(CURL_GLOBAL_ALL)) + fatal("failed to initialize libcurl"); + + if ((multi = curl_multi_init()) == NULL) + fatal("curl_multi_init(): failed"); + + /* XXX - make configurable? */ + curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, 500); + + if ((res = curl_multi_setopt(multi, + CURLMOPT_SOCKETFUNCTION, curl_socket)) != CURLM_OK) + fatal("curl_multi_setopt: %s", curl_multi_strerror(res)); + + if ((res = curl_multi_setopt(multi, + CURLMOPT_TIMERFUNCTION, curl_timer)) != CURLM_OK) + fatal("curl_multi_setopt: %s", curl_multi_strerror(res)); + + for (i = 0; i < FD_CACHE_BUCKETS; i++) + LIST_INIT(&cache[i]); + + kore_pool_init(&fd_cache_pool, "fd_cache_pool", 100, + sizeof(struct fd_cache)); + + len = snprintf(user_agent, sizeof(user_agent), "kore/%s", kore_version); + if (len == -1 || (size_t)len >= sizeof(user_agent)) + fatal("user-agent string too long"); +} + +int +kore_curl_init(struct kore_curl *client, const char *url) +{ + CURL *handle; + + memset(client, 0, sizeof(*client)); + + TAILQ_INIT(&client->http.resp_hdrs); + + if ((handle = curl_easy_init()) == NULL) + return (KORE_RESULT_ERROR); + + curl_easy_setopt(handle, CURLOPT_WRITEDATA, &client->response); + curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, kore_curl_tobuf); + + curl_easy_setopt(handle, CURLOPT_URL, url); + curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(handle, CURLOPT_PRIVATE, client); + curl_easy_setopt(handle, CURLOPT_USERAGENT, user_agent); + curl_easy_setopt(handle, CURLOPT_TIMEOUT, kore_curl_timeout); + curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, client->errbuf); + + client->handle = handle; + client->url = kore_strdup(url); + client->type = KORE_CURL_TYPE_CUSTOM; + + return (KORE_RESULT_OK); +} + +void +kore_curl_cleanup(struct kore_curl *client) +{ + struct http_header *hdr, *next; + + kore_free(client->url); + + if (client->flags & KORE_CURL_FLAG_BOUND) + LIST_REMOVE(client, list); + + if (client->handle != NULL) { + curl_multi_remove_handle(multi, client->handle); + curl_easy_cleanup(client->handle); + } + + if (client->http.hdrlist != NULL) + curl_slist_free_all(client->http.hdrlist); + + if (client->response != NULL) + kore_buf_free(client->response); + + if (client->http.headers != NULL) + kore_buf_free(client->http.headers); + + if (client->http.tosend != NULL) + kore_buf_free(client->http.tosend); + + for (hdr = TAILQ_FIRST(&client->http.resp_hdrs); + hdr != NULL; hdr = next) { + next = TAILQ_NEXT(hdr, list); + TAILQ_REMOVE(&client->http.resp_hdrs, hdr, list); + kore_pool_put(&http_header_pool, hdr); + } +} + +size_t +kore_curl_tobuf(char *ptr, size_t size, size_t nmemb, void *udata) +{ + size_t len; + struct kore_buf **buf, *b; + + if (SIZE_MAX / nmemb < size) + fatal("%s: %zu * %zu overflow", __func__, nmemb, size); + + buf = udata; + len = size * nmemb; + + if (*buf == NULL) + *buf = kore_buf_alloc(len); + + b = *buf; + + if (b->offset + len < b->offset) + fatal("%s: %zu+%zu overflows", __func__, b->offset, len); + + if ((b->offset + len) > kore_curl_recv_max) { + kore_log(LOG_ERR, + "received too large transfer (%zu > %" PRIu64 ")", + b->offset + len, kore_curl_recv_max); + return (0); + } + + kore_buf_append(b, ptr, len); + + return (len); +} + +size_t +kore_curl_frombuf(char *ptr, size_t size, size_t nmemb, void *udata) +{ + size_t len; + struct kore_buf *buf; + + if (SIZE_MAX / nmemb < size) + fatal("%s: %zu * %zu overflow", __func__, nmemb, size); + + buf = udata; + len = size * nmemb; + + if (buf->offset == buf->length) + return (0); + + if (buf->offset + len < buf->offset) + fatal("%s: %zu+%zu overflows", __func__, buf->offset, len); + + if ((buf->offset + len) < buf->length) { + memcpy(ptr, buf->data + buf->offset, len); + } else { + len = buf->length - buf->offset; + memcpy(ptr, buf->data + buf->offset, len); + } + + buf->offset += len; + + return (len); +} + +void +kore_curl_bind_request(struct kore_curl *client, struct http_request *req) +{ + if (client->cb != NULL) + fatal("%s: already bound to callback", __func__); + + client->req = req; + http_request_sleep(req); + + client->flags |= KORE_CURL_FLAG_BOUND; + LIST_INSERT_HEAD(&req->chandles, client, list); +} + +void +kore_curl_bind_callback(struct kore_curl *client, + void (*cb)(struct kore_curl *, void *), void *arg) +{ + if (client->req != NULL) + fatal("%s: already bound to request", __func__); + + client->cb = cb; + client->arg = arg; +} + +void +kore_curl_run(struct kore_curl *client) +{ + curl_multi_add_handle(multi, client->handle); +} + +int +kore_curl_success(struct kore_curl *client) +{ + return (client->result == CURLE_OK); +} + +const char * +kore_curl_strerror(struct kore_curl *client) +{ + const char *err; + + if (client->errbuf[0] != '\0') + err = &client->errbuf[0]; + else + err = curl_easy_strerror(client->result); + + return (err); +} + +void +kore_curl_logerror(struct kore_curl *client) +{ + kore_log(LOG_NOTICE, "curl error: %s -> %s", client->url, + kore_curl_strerror(client)); +} + +void +kore_curl_response_as_bytes(struct kore_curl *client, const u_int8_t **body, + size_t *len) +{ + if (client->response == NULL) { + *len = 0; + *body = NULL; + } else { + *len = client->response->offset; + *body = client->response->data; + } +} + +char * +kore_curl_response_as_string(struct kore_curl *client) +{ + kore_buf_stringify(client->response, NULL); + + return ((char *)client->response->data); +} + +void +kore_curl_http_setup(struct kore_curl *client, int method, const void *data, + size_t len) +{ + const char *mname; + int has_body; + + if (client->handle == NULL) + fatal("%s: called without setup", __func__); + + mname = NULL; + has_body = 1; + + client->type = KORE_CURL_TYPE_HTTP_CLIENT; + + curl_easy_setopt(client->handle, CURLOPT_HEADERDATA, + &client->http.headers); + curl_easy_setopt(client->handle, CURLOPT_HEADERFUNCTION, + kore_curl_tobuf); + + kore_curl_http_set_header(client, "expect", ""); + + switch (method) { + case HTTP_METHOD_GET: + break; + case HTTP_METHOD_HEAD: + curl_easy_setopt(client->handle, CURLOPT_NOBODY, 1); + break; + case HTTP_METHOD_DELETE: + case HTTP_METHOD_OPTIONS: + break; + case HTTP_METHOD_PUT: + has_body = 1; + curl_easy_setopt(client->handle, CURLOPT_UPLOAD, 1); + break; + case HTTP_METHOD_PATCH: + mname = http_method_text(method); + /* fallthrough */ + case HTTP_METHOD_POST: + has_body = 1; + curl_easy_setopt(client->handle, CURLOPT_POST, 1); + break; + default: + fatal("%s: unknown method %d", __func__, method); + } + + if (has_body && data != NULL && len > 0) { + client->http.tosend = kore_buf_alloc(len); + kore_buf_append(client->http.tosend, data, len); + kore_buf_reset(client->http.tosend); + + curl_easy_setopt(client->handle, CURLOPT_READDATA, + client->http.tosend); + curl_easy_setopt(client->handle, CURLOPT_READFUNCTION, + kore_curl_frombuf); + } + + if (has_body) { + if (method == HTTP_METHOD_PUT) { + curl_easy_setopt(client->handle, + CURLOPT_INFILESIZE_LARGE, (curl_off_t)len); + } else { + curl_easy_setopt(client->handle, + CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)len); + } + } else { + if (data != NULL || len != 0) { + fatal("%s: %d should not have a body", + __func__, method); + } + } + + if (mname != NULL) + curl_easy_setopt(client->handle, CURLOPT_CUSTOMREQUEST, mname); +} + +void +kore_curl_http_set_header(struct kore_curl *client, const char *header, + const char *value) +{ + struct kore_buf buf; + const char *hdr; + + kore_buf_init(&buf, 512); + + if (value != NULL || *value != '\0') { + kore_buf_appendf(&buf, "%s: %s", header, value); + } else { + kore_buf_appendf(&buf, "%s:", header); + } + + hdr = kore_buf_stringify(&buf, NULL); + + client->http.hdrlist = curl_slist_append(client->http.hdrlist, hdr); + kore_buf_cleanup(&buf); + + curl_easy_setopt(client->handle, + CURLOPT_HTTPHEADER, client->http.hdrlist); +} + +int +kore_curl_http_get_header(struct kore_curl *client, const char *header, + const char **out) +{ + struct http_header *hdr; + + if (!(client->flags & KORE_CURL_FLAG_HTTP_PARSED_HEADERS)) + kore_curl_http_parse_headers(client); + + TAILQ_FOREACH(hdr, &(client->http.resp_hdrs), list) { + if (!strcasecmp(hdr->header, header)) { + *out = hdr->value; + return (KORE_RESULT_OK); + } + } + + return (KORE_RESULT_ERROR); +} + +void +kore_curl_http_parse_headers(struct kore_curl *client) +{ + struct http_header *hdr; + int i, cnt; + char *value, *hbuf, *headers[HTTP_REQ_HEADER_MAX]; + + if (client->flags & KORE_CURL_FLAG_HTTP_PARSED_HEADERS) + fatal("%s: headers already parsed", __func__); + + client->flags |= KORE_CURL_FLAG_HTTP_PARSED_HEADERS; + + if (client->http.headers == NULL) + return; + + hbuf = kore_buf_stringify(client->http.headers, NULL); + cnt = kore_split_string(hbuf, "\r\n", headers, HTTP_REQ_HEADER_MAX); + + for (i = 0; i < cnt; i++) { + if ((value = http_validate_header(headers[i])) == NULL) + continue; + + if (*value == '\0') + continue; + + hdr = kore_pool_get(&http_header_pool); + hdr->header = headers[i]; + hdr->value = value; + TAILQ_INSERT_TAIL(&(client->http.resp_hdrs), hdr, list); + } +} + +static int +curl_socket(CURL *easy, curl_socket_t fd, int action, void *arg, void *sock) +{ + CURLcode res; + struct fd_cache *fdc; + struct kore_curl *client; + + client = NULL; + + res = curl_easy_getinfo(easy, CURLINFO_PRIVATE, (char **)&client); + if (res != CURLE_OK) + fatal("curl_easy_getinfo: %s", curl_easy_strerror(res)); + + if (client == NULL) + fatal("%s: failed to get client context", __func__); + + fdc = fd_cache_get(fd); + + switch (action) { + case CURL_POLL_NONE: + break; + case CURL_POLL_IN: + case CURL_POLL_OUT: + case CURL_POLL_INOUT: + if (fdc->scheduled == 0) { + kore_platform_event_all(fd, fdc); + fdc->scheduled = 1; + } + break; + case CURL_POLL_REMOVE: + if (fdc->scheduled) { + fdc->evt.flags = 0; + fdc->scheduled = 0; + kore_platform_disable_read(fd); +#if !defined(__linux__) + kore_platform_disable_write(fd); +#endif + } + break; + default: + fatal("unknown action value: %d", action); + } + + /* + * XXX - libcurl hates edge triggered io. + */ + if (action == CURL_POLL_OUT || action == CURL_POLL_INOUT) { + if (fdc->evt.flags & KORE_EVENT_WRITE) { + if (fdc->scheduled) { + kore_platform_disable_read(fdc->fd); +#if !defined(__linux__) + kore_platform_disable_write(fdc->fd); +#endif + } + + fdc->evt.flags = 0; + kore_platform_event_all(fdc->fd, fdc); + } + } + + return (CURLM_OK); +} + +static void +curl_process(void) +{ + CURLcode res; + CURLMsg *msg; + CURL *handle; + struct kore_curl *client; + int pending; + + pending = 0; + + while ((msg = curl_multi_info_read(multi, &pending)) != NULL) { + if (msg->msg != CURLMSG_DONE) + continue; + + handle = msg->easy_handle; + + res = curl_easy_getinfo(handle, CURLINFO_PRIVATE, + (char **)&client); + if (res != CURLE_OK) + fatal("curl_easy_getinfo: %s", curl_easy_strerror(res)); + + if (client == NULL) + fatal("%s: failed to get client context", __func__); + + client->result = msg->data.result; + + if (client->type == KORE_CURL_TYPE_HTTP_CLIENT) { + curl_easy_getinfo(client->handle, + CURLINFO_RESPONSE_CODE, &client->http.status); + } + + curl_multi_remove_handle(multi, client->handle); + curl_easy_cleanup(client->handle); + + client->handle = NULL; + + if (client->req != NULL) + http_request_wakeup(client->req); + else if (client->cb != NULL) + client->cb(client, client->arg); + } +} + +static void +curl_timeout(void *uarg, u_int64_t now) +{ + CURLMcode res; + + timer = NULL; + + res = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &running); + if (res != CURLM_OK) + fatal("curl_multi_socket_action: %s", curl_multi_strerror(res)); + + curl_process(); +} + +static int +curl_timer(CURLM *mctx, long timeout, void *arg) +{ + if (timeout < 0) { + if (timer != NULL) { + kore_timer_remove(timer); + timer = NULL; + } + return (CURLM_OK); + } + + if (timer != NULL) { + kore_timer_remove(timer); + timer = NULL; + } + + if (timeout == 0) + timeout = 10; + + timer = kore_timer_add(curl_timeout, timeout, mctx, KORE_TIMER_ONESHOT); + + return (CURLM_OK); +} + +static void +curl_event_handle(void *arg, int eof) +{ + CURLMcode res; + int flags; + ssize_t bytes; + char buf[32]; + struct fd_cache *fdc = arg; + + flags = 0; + + if (fdc->evt.flags & KORE_EVENT_READ) + flags |= CURL_CSELECT_IN; + + if (fdc->evt.flags & KORE_EVENT_WRITE) + flags |= CURL_CSELECT_OUT; + + if (eof) + flags |= CURL_CSELECT_ERR; + + res = curl_multi_socket_action(multi, fdc->fd, flags, &running); + if (res != CURLM_OK) + fatal("curl_multi_socket_action: %s", curl_multi_strerror(res)); + + /* + * XXX - libcurl doesn't work with edge triggered i/o so check + * if we need to reprime the event. Not optimal. + */ + if (fdc->evt.flags & KORE_EVENT_READ) { + bytes = recv(fdc->fd, buf, sizeof(buf), MSG_PEEK); + if (bytes > 0) { + if (fdc->scheduled) { + kore_platform_disable_read(fdc->fd); +#if !defined(__linux__) + kore_platform_disable_write(fdc->fd); +#endif + } + + fdc->evt.flags = 0; + kore_platform_event_all(fdc->fd, fdc); + } + } + + curl_process(); +} + +static struct fd_cache * +fd_cache_get(int fd) +{ + struct fd_cache *fdc; + int bucket; + + bucket = fd % FD_CACHE_BUCKETS; + + LIST_FOREACH(fdc, &cache[bucket], list) { + if (fdc->fd == fd) + return (fdc); + } + + fdc = kore_pool_get(&fd_cache_pool); + + fdc->fd = fd; + fdc->scheduled = 0; + + fdc->evt.flags = 0; + fdc->evt.handle = curl_event_handle; + fdc->evt.type = KORE_TYPE_CURL_HANDLE; + + LIST_INSERT_HEAD(&cache[bucket], fdc, list); + + return (fdc); +} diff -Nru kore-2.0.0/src/domain.c kore-3.3.1/src/domain.c --- kore-2.0.0/src/domain.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/domain.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,7 +14,14 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +/* + * XXX - Lots of OPENSSL ifdefs here for 1.0.2 and 1.1.0 release lines. + * The idea is to only support 1.1.1 down the line and remove the rest. + * (although we have to remain compat with 1.0.2 due to LibreSSL). + */ + #include +#include #if !defined(KORE_NO_TLS) #include @@ -29,6 +36,11 @@ #include "kore.h" +#if !defined(KORE_NO_HTTP) +#include "http.h" +#endif + +#define KORE_DOMAIN_CACHE 16 #define SSL_SESSION_ID "kore_ssl_sessionid" struct kore_domain_h domains; @@ -39,13 +51,13 @@ static size_t keymgr_buflen = 0; static int keymgr_response = 0; DH *tls_dhparam = NULL; -int tls_version = KORE_TLS_VERSION_1_2; +int tls_version = KORE_TLS_VERSION_BOTH; #endif -static void domain_load_crl(struct kore_domain *); - #if !defined(KORE_NO_TLS) +static BIO *domain_bio_mem(const void *, size_t); static int domain_x509_verify(int, X509_STORE_CTX *); +static X509 *domain_load_certificate_chain(SSL_CTX *, const void *, size_t); static void keymgr_init(void); static void keymgr_await_data(void); @@ -59,12 +71,14 @@ static ECDSA_SIG *keymgr_ecdsa_sign(const unsigned char *, int, const BIGNUM *, const BIGNUM *, EC_KEY *); +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L +static RSA_METHOD *keymgr_rsa_meth = NULL; +static EC_KEY_METHOD *keymgr_ec_meth = NULL; +#else #if !defined(LIBRESSL_VERSION_TEXT) /* * Run own ecdsa_method data structure as OpenSSL has this in ecs_locl.h * and does not export this on systems. - * - * XXX - OpenSSL is merging ECDSA functionality into EC in 1.1.0. */ struct ecdsa_method { const char *name; @@ -105,12 +119,51 @@ NULL }; -#endif +#endif /* OPENSSL_VERSION_NUMBER */ +#endif /* KORE_NO_TLS */ + +static u_int16_t domain_id = 0; +static struct kore_domain *cached[KORE_DOMAIN_CACHE]; void kore_domain_init(void) { + int i; + TAILQ_INIT(&domains); + + for (i = 0; i < KORE_DOMAIN_CACHE; i++) + cached[i] = NULL; + +#if !defined(KORE_NO_TLS) +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + if (keymgr_rsa_meth == NULL) { + if ((keymgr_rsa_meth = RSA_meth_new("kore RSA keymgr method", + RSA_METHOD_FLAG_NO_CHECK)) == NULL) + fatal("failed to allocate RSA method"); + } + + RSA_meth_set_init(keymgr_rsa_meth, keymgr_rsa_init); + RSA_meth_set_finish(keymgr_rsa_meth, keymgr_rsa_finish); + RSA_meth_set_priv_enc(keymgr_rsa_meth, keymgr_rsa_privenc); + + if (keymgr_ec_meth == NULL) { + if ((keymgr_ec_meth = EC_KEY_METHOD_new(NULL)) == NULL) + fatal("failed to allocate EC KEY method"); + } + + EC_KEY_METHOD_set_sign(keymgr_ec_meth, NULL, NULL, keymgr_ecdsa_sign); +#endif + +#if !defined(TLS1_3_VERSION) + if (!kore_quiet) { + kore_log(LOG_NOTICE, + "%s has no TLS 1.3 - will only use TLS 1.2", + OPENSSL_VERSION_TEXT); + } +#endif + +#endif } void @@ -122,6 +175,20 @@ TAILQ_REMOVE(&domains, dom, list); kore_domain_free(dom); } + +#if !defined(KORE_NO_TLS) +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + if (keymgr_rsa_meth != NULL) { + RSA_meth_free(keymgr_rsa_meth); + keymgr_rsa_meth = NULL; + } + + if (keymgr_ec_meth != NULL) { + EC_KEY_METHOD_free(keymgr_ec_meth); + keymgr_ec_meth = NULL; + } +#endif +#endif } int @@ -135,16 +202,31 @@ kore_debug("kore_domain_new(%s)", domain); dom = kore_malloc(sizeof(*dom)); + dom->id = domain_id++; + dom->logbuf = NULL; dom->accesslog = -1; + dom->logwarn = 0; + dom->logerr = 0; #if !defined(KORE_NO_TLS) dom->cafile = NULL; dom->certkey = NULL; dom->ssl_ctx = NULL; dom->certfile = NULL; dom->crlfile = NULL; + dom->x509_verify_depth = 1; #endif dom->domain = kore_strdup(domain); + +#if !defined(KORE_NO_HTTP) TAILQ_INIT(&(dom->handlers)); +#endif + + if (dom->id < KORE_DOMAIN_CACHE) { + if (cached[dom->id] != NULL) + fatal("non free domain cache slot"); + cached[dom->id] = dom; + } + TAILQ_INSERT_TAIL(&domains, dom, list); if (primary_dom == NULL) @@ -193,134 +275,158 @@ kore_free(dom); } +#if !defined(KORE_NO_TLS) void -kore_domain_sslstart(struct kore_domain *dom) +kore_domain_tlsinit(struct kore_domain *dom, const void *pem, size_t pemlen) { -#if !defined(KORE_NO_TLS) - BIO *in; RSA *rsa; X509 *x509; EVP_PKEY *pkey; STACK_OF(X509_NAME) *certs; EC_KEY *eckey; - X509_STORE *store; const SSL_METHOD *method; -#if !defined(OPENSSL_NO_EC) - EC_KEY *ecdh; +#if defined(LIBRESSL_VERSION_TEXT) || OPENSSL_VERSION_NUMBER < 0x10100000L + EC_KEY *ecdh; #endif - kore_debug("kore_domain_sslstart(%s)", dom->domain); + kore_debug("kore_domain_tlsinit(%s)", dom->domain); + if (dom->ssl_ctx != NULL) + SSL_CTX_free(dom->ssl_ctx); + +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + if ((method = TLS_method()) == NULL) + fatalx("TLS_method(): %s", ssl_errno_s); +#else switch (tls_version) { + case KORE_TLS_VERSION_1_3: case KORE_TLS_VERSION_1_2: - method = TLSv1_2_server_method(); - break; - case KORE_TLS_VERSION_1_0: - method = TLSv1_server_method(); - break; case KORE_TLS_VERSION_BOTH: - method = SSLv23_server_method(); + method = TLSv1_2_server_method(); break; default: - fatal("unknown tls_version: %d", tls_version); + fatalx("unknown tls_version: %d", tls_version); return; } +#endif - dom->ssl_ctx = SSL_CTX_new(method); - if (dom->ssl_ctx == NULL) - fatal("kore_domain_sslstart(): SSL_ctx_new(): %s", ssl_errno_s); - if (!SSL_CTX_use_certificate_chain_file(dom->ssl_ctx, dom->certfile)) { - fatal("SSL_CTX_use_certificate_chain_file(%s): %s", - dom->certfile, ssl_errno_s); - } + if ((dom->ssl_ctx = SSL_CTX_new(method)) == NULL) + fatalx("SSL_ctx_new(): %s", ssl_errno_s); - if ((in = BIO_new(BIO_s_file_internal())) == NULL) - fatal("BIO_new: %s", ssl_errno_s); - if (BIO_read_filename(in, dom->certfile) <= 0) - fatal("BIO_read_filename: %s", ssl_errno_s); - if ((x509 = PEM_read_bio_X509(in, NULL, NULL, NULL)) == NULL) - fatal("PEM_read_bio_X509: %s", ssl_errno_s); +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + if (!SSL_CTX_set_min_proto_version(dom->ssl_ctx, TLS1_2_VERSION)) + fatalx("SSL_CTX_set_min_proto_version: %s", ssl_errno_s); + +#if defined(TLS1_3_VERSION) + if (!SSL_CTX_set_max_proto_version(dom->ssl_ctx, TLS1_3_VERSION)) + fatalx("SSL_CTX_set_max_proto_version: %s", ssl_errno_s); +#else + if (!SSL_CTX_set_max_proto_version(dom->ssl_ctx, TLS1_2_VERSION)) + fatalx("SSL_CTX_set_min_proto_version: %s", ssl_errno_s); +#endif - BIO_free(in); + switch (tls_version) { + case KORE_TLS_VERSION_1_3: +#if defined(TLS1_3_VERSION) + if (!SSL_CTX_set_min_proto_version(dom->ssl_ctx, + TLS1_3_VERSION)) { + fatalx("SSL_CTX_set_min_proto_version: %s", + ssl_errno_s); + } + break; +#endif + case KORE_TLS_VERSION_1_2: + if (!SSL_CTX_set_max_proto_version(dom->ssl_ctx, + TLS1_2_VERSION)) { + fatalx("SSL_CTX_set_min_proto_version: %s", + ssl_errno_s); + } + break; + case KORE_TLS_VERSION_BOTH: + break; + default: + fatalx("unknown tls_version: %d", tls_version); + return; + } +#endif + x509 = domain_load_certificate_chain(dom->ssl_ctx, pem, pemlen); if ((pkey = X509_get_pubkey(x509)) == NULL) - fatal("certificate has no public key"); + fatalx("certificate has no public key"); switch (EVP_PKEY_id(pkey)) { case EVP_PKEY_RSA: if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) - fatal("no RSA public key present"); + fatalx("no RSA public key present"); RSA_set_app_data(rsa, dom); +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + RSA_set_method(rsa, keymgr_rsa_meth); +#else RSA_set_method(rsa, &keymgr_rsa); +#endif break; case EVP_PKEY_EC: if ((eckey = EVP_PKEY_get1_EC_KEY(pkey)) == NULL) - fatal("no EC public key present"); + fatalx("no EC public key present"); +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + EC_KEY_set_ex_data(eckey, 0, dom); + EC_KEY_set_method(eckey, keymgr_ec_meth); +#else ECDSA_set_ex_data(eckey, 0, dom); ECDSA_set_method(eckey, &keymgr_ecdsa); +#endif break; default: - fatal("unknown public key in certificate"); + fatalx("unknown public key in certificate"); } if (!SSL_CTX_use_PrivateKey(dom->ssl_ctx, pkey)) - fatal("SSL_CTX_use_PrivateKey(): %s", ssl_errno_s); + fatalx("SSL_CTX_use_PrivateKey(): %s", ssl_errno_s); if (!SSL_CTX_check_private_key(dom->ssl_ctx)) - fatal("Public/Private key for %s do not match", dom->domain); + fatalx("Public/Private key for %s do not match", dom->domain); if (tls_dhparam == NULL) - fatal("No DH parameters given"); + fatalx("No DH parameters given"); SSL_CTX_set_tmp_dh(dom->ssl_ctx, tls_dhparam); SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_SINGLE_DH_USE); +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + if (!SSL_CTX_set_ecdh_auto(dom->ssl_ctx, 1)) + fatalx("SSL_CTX_set_ecdh_auto: %s", ssl_errno_s); +#else if ((ecdh = EC_KEY_new_by_curve_name(NID_secp384r1)) == NULL) - fatal("EC_KEY_new_by_curve_name: %s", ssl_errno_s); + fatalx("EC_KEY_new_by_curve_name: %s", ssl_errno_s); SSL_CTX_set_tmp_ecdh(dom->ssl_ctx, ecdh); EC_KEY_free(ecdh); +#endif + SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_SINGLE_ECDH_USE); SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_NO_COMPRESSION); if (dom->cafile != NULL) { if ((certs = SSL_load_client_CA_file(dom->cafile)) == NULL) { - fatal("SSL_load_client_CA_file(%s): %s", + fatalx("SSL_load_client_CA_file(%s): %s", dom->cafile, ssl_errno_s); } SSL_CTX_load_verify_locations(dom->ssl_ctx, dom->cafile, NULL); - SSL_CTX_set_verify_depth(dom->ssl_ctx, 1); + SSL_CTX_set_verify_depth(dom->ssl_ctx, dom->x509_verify_depth); SSL_CTX_set_client_CA_list(dom->ssl_ctx, certs); SSL_CTX_set_verify(dom->ssl_ctx, SSL_VERIFY_PEER | - SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL); - - if ((store = SSL_CTX_get_cert_store(dom->ssl_ctx)) == NULL) - fatal("SSL_CTX_get_cert_store(): %s", ssl_errno_s); - - X509_STORE_set_verify_cb(store, domain_x509_verify); + SSL_VERIFY_FAIL_IF_NO_PEER_CERT, domain_x509_verify); } SSL_CTX_set_session_id_context(dom->ssl_ctx, (unsigned char *)SSL_SESSION_ID, strlen(SSL_SESSION_ID)); - - /* - * Force OpenSSL to not use its freelists. Even without using - * SSL_MODE_RELEASE_BUFFERS there are times it will use the - * freelists. So forcefully putting its max length to 0 is the - * only we choice we seem to have. - * - * Note that OpenBSD has since heartbleed removed freelists - * from its OpenSSL in base so we don't need to care about it. - */ -#if !defined(LIBRESSL_VERSION_TEXT) - dom->ssl_ctx->freelist_max_len = 0; -#endif SSL_CTX_set_mode(dom->ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE); if (tls_version == KORE_TLS_VERSION_BOTH) { SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_NO_SSLv2); SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_NO_SSLv3); + SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_NO_TLSv1); SSL_CTX_set_options(dom->ssl_ctx, SSL_OP_NO_TLSv1_1); } @@ -330,10 +436,60 @@ SSL_CTX_set_info_callback(dom->ssl_ctx, kore_tls_info_callback); SSL_CTX_set_tlsext_servername_callback(dom->ssl_ctx, kore_tls_sni_cb); - kore_free(dom->certfile); - dom->certfile = NULL; -#endif + X509_free(x509); +} + +void +kore_domain_crl_add(struct kore_domain *dom, const void *pem, size_t pemlen) +{ + int err; + BIO *in; + X509_CRL *crl; + X509_STORE *store; + + ERR_clear_error(); + in = domain_bio_mem(pem, pemlen); + + if ((store = SSL_CTX_get_cert_store(dom->ssl_ctx)) == NULL) { + BIO_free(in); + kore_log(LOG_ERR, "SSL_CTX_get_cert_store(): %s", ssl_errno_s); + return; + } + + for (;;) { + crl = PEM_read_bio_X509_CRL(in, NULL, NULL, NULL); + if (crl == NULL) { + err = ERR_GET_REASON(ERR_peek_last_error()); + if (err == PEM_R_NO_START_LINE) { + ERR_clear_error(); + break; + } + + kore_log(LOG_WARNING, "failed to read CRL %s: %s", + dom->crlfile, ssl_errno_s); + continue; + } + + if (!X509_STORE_add_crl(store, crl)) { + err = ERR_GET_REASON(ERR_peek_last_error()); + if (err == X509_R_CERT_ALREADY_IN_HASH_TABLE) { + X509_CRL_free(crl); + continue; + } + + kore_log(LOG_WARNING, "failed to add CRL %s: %s", + dom->crlfile, ssl_errno_s); + X509_CRL_free(crl); + continue; + } + } + + BIO_free(in); + + X509_STORE_set_flags(store, + X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL); } +#endif void kore_domain_callback(void (*cb)(struct kore_domain *)) @@ -351,6 +507,8 @@ struct kore_domain *dom; TAILQ_FOREACH(dom, &domains, list) { + if (!strcmp(dom->domain, domain)) + return (dom); if (!fnmatch(dom->domain, domain, FNM_CASEFOLD)) return (dom); } @@ -358,25 +516,40 @@ return (NULL); } -void -kore_domain_closelogs(void) +struct kore_domain * +kore_domain_byid(u_int16_t id) { struct kore_domain *dom; + if (id < KORE_DOMAIN_CACHE) + return (cached[id]); + TAILQ_FOREACH(dom, &domains, list) { - if (dom->accesslog != -1) { - (void)close(dom->accesslog); - } + if (dom->id == id) + return (dom); } + + return (NULL); } +/* + * Called by the worker processes to close the file descriptor towards + * the accesslog as they do not need it locally. + */ void -kore_domain_load_crl(void) +kore_domain_closelogs(void) { struct kore_domain *dom; - TAILQ_FOREACH(dom, &domains, list) - domain_load_crl(dom); + TAILQ_FOREACH(dom, &domains, list) { + if (dom->accesslog != -1) { + (void)close(dom->accesslog); + /* turn into flag to indicate accesslogs are active. */ + dom->accesslog = 1; + } else { + dom->accesslog = 0; + } + } } void @@ -388,38 +561,6 @@ #endif } -static void -domain_load_crl(struct kore_domain *dom) -{ -#if !defined(KORE_NO_TLS) - X509_STORE *store; - - ERR_clear_error(); - - if (dom->cafile == NULL) - return; - - if (dom->crlfile == NULL) { - kore_log(LOG_WARNING, "WARNING: Running without CRL"); - return; - } - - if ((store = SSL_CTX_get_cert_store(dom->ssl_ctx)) == NULL) { - kore_log(LOG_ERR, "SSL_CTX_get_cert_store(): %s", ssl_errno_s); - return; - } - - if (!X509_STORE_load_locations(store, dom->crlfile, NULL)) { - kore_log(LOG_ERR, "X509_STORE_load_locations(): %s", - ssl_errno_s); - return; - } - - X509_STORE_set_flags(store, - X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL); -#endif -} - #if !defined(KORE_NO_TLS) static void keymgr_init(void) @@ -429,16 +570,27 @@ if ((meth = RSA_get_default_method()) == NULL) fatal("failed to obtain RSA method"); +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + RSA_meth_set_pub_enc(keymgr_rsa_meth, RSA_meth_get_pub_enc(meth)); + RSA_meth_set_pub_dec(keymgr_rsa_meth, RSA_meth_get_pub_dec(meth)); + RSA_meth_set_bn_mod_exp(keymgr_rsa_meth, RSA_meth_get_bn_mod_exp(meth)); +#else keymgr_rsa.rsa_pub_enc = meth->rsa_pub_enc; keymgr_rsa.rsa_pub_dec = meth->rsa_pub_dec; keymgr_rsa.bn_mod_exp = meth->bn_mod_exp; +#endif } static int keymgr_rsa_init(RSA *rsa) { if (rsa != NULL) { +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + RSA_set_flags(rsa, RSA_flags(rsa) | + RSA_FLAG_EXT_PKEY | RSA_METHOD_FLAG_NO_CHECK); +#else rsa->flags |= RSA_FLAG_EXT_PKEY | RSA_METHOD_FLAG_NO_CHECK; +#endif return (1); } @@ -515,8 +667,13 @@ if (len > sizeof(keymgr_buf)) fatal("keymgr_buf too small"); +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + if ((dom = EC_KEY_get_ex_data(eckey, 0)) == NULL) + fatal("EC_KEY has no domain"); +#else if ((dom = ECDSA_get_ex_data(eckey, 0)) == NULL) fatal("EC_KEY has no domain"); +#endif memset(keymgr_buf, 0, sizeof(keymgr_buf)); @@ -550,6 +707,9 @@ int ret; struct pollfd pfd[1]; u_int64_t start, cur; +#if !defined(KORE_NO_HTTP) + int process_requests; +#endif /* * We need to wait until the keymgr responds to us, so keep doing @@ -559,10 +719,15 @@ * This means other internal messages can still be delivered by * this worker process to the appropriate callbacks but we do not * drop out until we've either received an answer from the keymgr - * or until the timeout has been reached. + * or until the timeout has been reached (1 second currently). + * + * If we end up waiting for the keymgr process we will call + * http_process (if not built with NOHTTP=1) to further existing + * requests so those do not block too much. * - * It will however block any other I/O and request handling on - * this worker until either of the above criteria is met. + * This means that all incoming data will stop being processed + * while existing requests will get processed until we return + * from this call. */ start = kore_time_ms(); kore_platform_disable_read(worker->msg[1]->fd); @@ -570,7 +735,17 @@ keymgr_response = 0; memset(keymgr_buf, 0, sizeof(keymgr_buf)); +#if !defined(KORE_NO_HTTP) + process_requests = 0; +#endif + for (;;) { +#if !defined(KORE_NO_HTTP) + if (process_requests) { + http_process(); + process_requests = 0; + } +#endif pfd[0].fd = worker->msg[1]->fd; pfd[0].events = POLLIN; pfd[0].revents = 0; @@ -586,20 +761,32 @@ if ((cur - start) > 1000) break; - if (ret == 0) + if (ret == 0) { +#if !defined(KORE_NO_HTTP) + /* No activity on channel, process HTTP requests. */ + process_requests = 1; +#endif continue; + } if (pfd[0].revents & (POLLERR | POLLHUP)) break; if (!(pfd[0].revents & POLLIN)) break; - worker->msg[1]->flags |= CONN_READ_POSSIBLE; + worker->msg[1]->evt.flags |= KORE_EVENT_READ; if (!net_recv_flush(worker->msg[1])) break; if (keymgr_response) break; + +#if !defined(KORE_NO_HTTP) + /* If we've spent 100ms already, process HTTP requests. */ + if ((cur - start) > 100) { + process_requests = 1; + } +#endif } } @@ -644,4 +831,86 @@ return (ok); } + +/* + * What follows is basically a reimplementation of + * SSL_CTX_use_certificate_chain_file() from OpenSSL but with our + * BIO set to the pem data that we received. + */ +static X509 * +domain_load_certificate_chain(SSL_CTX *ctx, const void *data, size_t len) +{ + unsigned long err; + BIO *in; + X509 *x, *ca; + + ERR_clear_error(); + in = domain_bio_mem(data, len); + + if ((x = PEM_read_bio_X509_AUX(in, NULL, NULL, NULL)) == NULL) + fatal("PEM_read_bio_X509_AUX: %s", ssl_errno_s); + + /* refcount for x509 will go up one. */ + if (SSL_CTX_use_certificate(ctx, x) == 0) + fatal("SSL_CTX_use_certificate: %s", ssl_errno_s); + +#if defined(LIBRESSL_VERSION_TEXT) + sk_X509_pop_free(ctx->extra_certs, X509_free); + ctx->extra_certs = NULL; +#else + SSL_CTX_clear_chain_certs(ctx); +#endif + + ERR_clear_error(); + while ((ca = PEM_read_bio_X509(in, NULL, NULL, NULL)) != NULL) { + /* ca its reference count won't be increased. */ +#if defined(LIBRESSL_VERSION_TEXT) + if (SSL_CTX_add_extra_chain_cert(ctx, ca) == 0) + fatal("SSL_CTX_add_extra_chain_cert: %s", ssl_errno_s); +#else + if (SSL_CTX_add0_chain_cert(ctx, ca) == 0) + fatal("SSL_CTX_add0_chain_cert: %s", ssl_errno_s); +#endif + } + + err = ERR_peek_last_error(); + + if (ERR_GET_LIB(err) != ERR_LIB_PEM || + ERR_GET_REASON(err) != PEM_R_NO_START_LINE) + fatal("PEM_read_bio_X509: %s", ssl_errno_s); + + BIO_free(in); + + return (x); +} + +/* + * XXX - Hack around the fact that LibreSSL its BIO_new_mem_buf() does not + * take a const pointer for their first argument. + * + * Since we build with -Wcast-qual and -Werror I rather do this than having + * a bunch of pragma preprocessor magic to remove the warnings for that code + * if we're dealing with LibreSSL. + * + * They fixed this in their upcoming 2.8.0 release but that is not out yet + * and I'd like this to run on older OpenBSD platforms as well. + */ +static BIO * +domain_bio_mem(const void *data, size_t len) +{ + BIO *in; + union { void *p; const void *cp; } deconst; + + /* because OpenSSL likes taking ints as memory buffer lengths. */ + if (len > INT_MAX) + fatal("domain_bio_mem: len(%zu) > INT_MAX", len); + + deconst.cp = data; + + if ((in = BIO_new_mem_buf(deconst.p, len)) == NULL) + fatal("BIO_new_mem_buf: %s", ssl_errno_s); + + return (in); +} + #endif diff -Nru kore-2.0.0/src/filemap.c kore-3.3.1/src/filemap.c --- kore-2.0.0/src/filemap.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/src/filemap.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,318 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include + +#include +#include +#include + +#include "kore.h" + +#if !defined(KORE_NO_HTTP) + +#include "http.h" + +struct filemap_entry { + char *root; + size_t root_len; + struct kore_domain *domain; + char *ondisk; + size_t ondisk_len; + TAILQ_ENTRY(filemap_entry) list; +}; + +int filemap_resolve(struct http_request *); + +static void filemap_serve(struct http_request *, struct filemap_entry *); + +static TAILQ_HEAD(, filemap_entry) maps; + +char *kore_filemap_ext = NULL; +char *kore_filemap_index = NULL; + +void +kore_filemap_init(void) +{ + TAILQ_INIT(&maps); +} + +int +kore_filemap_create(struct kore_domain *dom, const char *path, const char *root) +{ + size_t sz; + struct stat st; + int len; + struct kore_module_handle *hdlr; + struct filemap_entry *entry; + char regex[1024], fpath[PATH_MAX]; + + sz = strlen(root); + if (sz == 0) + return (KORE_RESULT_ERROR); + + if (root[0] != '/' || root[sz - 1] != '/') + return (KORE_RESULT_ERROR); + + if (kore_root_path != NULL) { + len = snprintf(fpath, sizeof(fpath), "%s/%s", + kore_root_path, path); + if (len == -1 || (size_t)len >= sizeof(fpath)) + fatal("kore_filemap_create: failed to concat paths"); + } else { + if (kore_strlcpy(fpath, path, sizeof(fpath)) >= sizeof(fpath)) + fatal("kore_filemap_create: failed to copy path"); + } + + if (stat(fpath, &st) == -1) + return (KORE_RESULT_ERROR); + + len = snprintf(regex, sizeof(regex), "^%s.*$", root); + if (len == -1 || (size_t)len >= sizeof(regex)) + fatal("kore_filemap_create: buffer too small"); + + if (!kore_module_handler_new(regex, dom->domain, + "filemap_resolve", NULL, HANDLER_TYPE_DYNAMIC)) + return (KORE_RESULT_ERROR); + + hdlr = NULL; + TAILQ_FOREACH(hdlr, &dom->handlers, list) { + if (!strcmp(hdlr->path, regex)) + break; + } + + if (hdlr == NULL) + fatal("couldn't find newly created handler for filemap"); + + hdlr->methods = HTTP_METHOD_GET | HTTP_METHOD_HEAD; + + entry = kore_calloc(1, sizeof(*entry)); + entry->domain = dom; + entry->root_len = sz; + entry->root = kore_strdup(root); + + /* + * Resolve the ondisk component inside the workers to make sure + * realpath() resolves the correct path (they maybe chrooted). + */ + entry->ondisk_len = strlen(path); + entry->ondisk = kore_strdup(path); + + TAILQ_INSERT_TAIL(&maps, entry, list); + + return (KORE_RESULT_OK); +} + +void +kore_filemap_resolve_paths(void) +{ + struct filemap_entry *entry; + char rpath[PATH_MAX]; + + TAILQ_FOREACH(entry, &maps, list) { + if (realpath(entry->ondisk, rpath) == NULL) + fatal("realpath(%s): %s", entry->ondisk, errno_s); + + kore_free(entry->ondisk); + entry->ondisk_len = strlen(rpath); + entry->ondisk = kore_strdup(rpath); + } +} + +int +filemap_resolve(struct http_request *req) +{ + size_t best_len; + struct filemap_entry *entry, *best; + + if (req->method != HTTP_METHOD_GET && + req->method != HTTP_METHOD_HEAD) { + http_response_header(req, "allow", "get, head"); + http_response(req, HTTP_STATUS_BAD_REQUEST, NULL, 0); + return (KORE_RESULT_OK); + } + + best = NULL; + best_len = 0; + + TAILQ_FOREACH(entry, &maps, list) { + if (entry->domain != req->hdlr->dom) + continue; + + if (!strncmp(entry->root, req->path, entry->root_len)) { + if (best == NULL || entry->root_len > best_len) { + best = entry; + best_len = entry->root_len; + continue; + } + } + } + + if (best == NULL) { + http_response(req, HTTP_STATUS_NOT_FOUND, NULL, 0); + return (KORE_RESULT_OK); + } + + filemap_serve(req, best); + + return (KORE_RESULT_OK); +} + +static void +filemap_serve(struct http_request *req, struct filemap_entry *map) +{ + struct stat st; + struct kore_fileref *ref; + const char *path; + int len, fd, index; + char fpath[PATH_MAX], rpath[PATH_MAX]; + + path = req->path + map->root_len; + + len = snprintf(fpath, sizeof(fpath), "%s/%s", map->ondisk, path); + if (len == -1 || (size_t)len >= sizeof(fpath)) { + http_response(req, HTTP_STATUS_INTERNAL_ERROR, NULL, 0); + return; + } + + if (!http_argument_urldecode(fpath)) { + http_response(req, HTTP_STATUS_BAD_REQUEST, NULL, 0); + return; + } + + index = 0; + +lookup: + if (realpath(fpath, rpath) == NULL) { + if (errno == ENOENT) { + if (index == 0 && kore_filemap_ext != NULL) { + len = snprintf(fpath, sizeof(fpath), + "%s/%s%s", map->ondisk, path, + kore_filemap_ext); + if (len == -1 || + (size_t)len >= sizeof(fpath)) { + http_response(req, + HTTP_STATUS_INTERNAL_ERROR, + NULL, 0); + return; + } + index++; + goto lookup; + } + } + http_response(req, HTTP_STATUS_NOT_FOUND, NULL, 0); + return; + } + + if (strncmp(rpath, fpath, map->ondisk_len)) { + http_response(req, HTTP_STATUS_NOT_FOUND, NULL, 0); + return; + } + + if ((ref = kore_fileref_get(rpath)) == NULL) { + if ((fd = open(fpath, O_RDONLY | O_NOFOLLOW)) == -1) { + switch (errno) { + case ENOENT: + if (index || kore_filemap_ext == NULL) { + req->status = HTTP_STATUS_NOT_FOUND; + } else { + len = snprintf(fpath, sizeof(fpath), + "%s/%s%s", map->ondisk, path, + kore_filemap_ext); + if (len == -1 || + (size_t)len >= sizeof(fpath)) { + http_response(req, + HTTP_STATUS_INTERNAL_ERROR, + NULL, 0); + return; + } + index++; + goto lookup; + } + break; + case EPERM: + case EACCES: + req->status = HTTP_STATUS_FORBIDDEN; + break; + default: + req->status = HTTP_STATUS_INTERNAL_ERROR; + break; + } + + http_response(req, req->status, NULL, 0); + return; + } + + if (fstat(fd, &st) == -1) { + http_response(req, HTTP_STATUS_INTERNAL_ERROR, NULL, 0); + goto cleanup; + } + + if (S_ISREG(st.st_mode)) { + if (st.st_size <= 0) { + http_response(req, + HTTP_STATUS_NOT_FOUND, NULL, 0); + goto cleanup; + } + + /* kore_fileref_create() takes ownership of the fd. */ + ref = kore_fileref_create(fpath, fd, + st.st_size, &st.st_mtim); + if (ref == NULL) { + http_response(req, + HTTP_STATUS_INTERNAL_ERROR, NULL, 0); + } else { + fd = -1; + } + } else if (S_ISDIR(st.st_mode) && index == 0) { + close(fd); + if (req->path[strlen(req->path) - 1] != '/') { + (void)snprintf(fpath, + sizeof(fpath), "%s/", req->path); + http_response_header(req, "location", fpath); + http_response(req, HTTP_STATUS_FOUND, NULL, 0); + return; + } + + len = snprintf(fpath, sizeof(fpath), + "%s/%s%s", map->ondisk, path, + kore_filemap_index != NULL ? + kore_filemap_index : "index.html"); + if (len == -1 || (size_t)len >= sizeof(fpath)) { + http_response(req, + HTTP_STATUS_INTERNAL_ERROR, NULL, 0); + return; + } + index++; + goto lookup; + } else { + http_response(req, HTTP_STATUS_NOT_FOUND, NULL, 0); + } + } + + if (ref != NULL) { + http_response_fileref(req, HTTP_STATUS_OK, ref); + fd = -1; + } + +cleanup: + if (fd != -1) + close(fd); +} + +#endif diff -Nru kore-2.0.0/src/fileref.c kore-3.3.1/src/fileref.c --- kore-2.0.0/src/fileref.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/src/fileref.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include + +#include + +#include "kore.h" + +/* cached filerefs expire after 30 seconds of inactivity. */ +#define FILEREF_EXPIRATION (1000 * 30) + +static void fileref_timer_prime(void); +static void fileref_drop(struct kore_fileref *); +static void fileref_soft_remove(struct kore_fileref *); +static void fileref_expiration_check(void *, u_int64_t); + +static TAILQ_HEAD(, kore_fileref) refs; +static struct kore_pool ref_pool; +static struct kore_timer *ref_timer = NULL; + +void +kore_fileref_init(void) +{ + TAILQ_INIT(&refs); + kore_pool_init(&ref_pool, "ref_pool", sizeof(struct kore_fileref), 100); +} + +struct kore_fileref * +kore_fileref_create(const char *path, int fd, off_t size, struct timespec *ts) +{ + struct kore_fileref *ref; + + fileref_timer_prime(); + + if ((ref = kore_fileref_get(path)) != NULL) + return (ref); + + ref = kore_pool_get(&ref_pool); + + ref->cnt = 1; + ref->flags = 0; + ref->size = size; + ref->path = kore_strdup(path); + ref->mtime_sec = ts->tv_sec; + ref->mtime = ((u_int64_t)(ts->tv_sec * 1000 + (ts->tv_nsec / 1000000))); + +#if !defined(KORE_USE_PLATFORM_SENDFILE) + if ((uintmax_t)size> SIZE_MAX) { + kore_pool_put(&ref_pool, ref); + return (NULL); + } + + ref->base = mmap(NULL, (size_t)size, PROT_READ, MAP_PRIVATE, fd, 0); + if (ref->base == MAP_FAILED) + fatal("net_send_file: mmap failed: %s", errno_s); + if (madvise(ref->base, (size_t)size, MADV_SEQUENTIAL) == -1) + fatal("net_send_file: madvise: %s", errno_s); + close(fd); +#else + ref->fd = fd; +#endif + +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p created", (void *)ref); +#endif + + TAILQ_INSERT_TAIL(&refs, ref, list); + + return (ref); +} + +/* + * Caller must call kore_fileref_release() after kore_fileref_get() even + * if they don't end up using the ref. + */ +struct kore_fileref * +kore_fileref_get(const char *path) +{ + struct stat st; + struct kore_fileref *ref; + u_int64_t mtime; + + TAILQ_FOREACH(ref, &refs, list) { + if (!strcmp(ref->path, path)) { + if (stat(ref->path, &st) == -1) { + if (errno != ENOENT) { + kore_log(LOG_ERR, "stat(%s): %s", + ref->path, errno_s); + } + fileref_soft_remove(ref); + return (NULL); + } + + mtime = ((u_int64_t)(st.st_mtim.tv_sec * 1000 + + (st.st_mtim.tv_nsec / 1000000))); + + if (ref->mtime != mtime) { + fileref_soft_remove(ref); + return (NULL); + } + + ref->cnt++; +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p cnt:%d", + (void *)ref, ref->cnt); +#endif + TAILQ_REMOVE(&refs, ref, list); + TAILQ_INSERT_HEAD(&refs, ref, list); + return (ref); + } + } + + return (NULL); +} + +void +kore_fileref_release(struct kore_fileref *ref) +{ + ref->cnt--; + +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p released cnt:%d", (void *)ref, ref->cnt); +#endif + + if (ref->cnt < 0) { + fatal("kore_fileref_release: cnt < 0 (%p:%d)", + (void *)ref, ref->cnt); + } + + if (ref->cnt == 0) { + if (ref->flags & KORE_FILEREF_SOFT_REMOVED) + fileref_drop(ref); + else + ref->expiration = kore_time_ms() + FILEREF_EXPIRATION; + } +} + +static void +fileref_timer_prime(void) +{ + if (ref_timer != NULL) + return; + + ref_timer = kore_timer_add(fileref_expiration_check, 10000, NULL, 0); +} + +static void +fileref_soft_remove(struct kore_fileref *ref) +{ + if (ref->flags & KORE_FILEREF_SOFT_REMOVED) + fatal("fileref_soft_remove: %p already removed", (void *)ref); + +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p softremoved", (void *)ref); +#endif + + TAILQ_REMOVE(&refs, ref, list); + ref->flags |= KORE_FILEREF_SOFT_REMOVED; + + if (ref->cnt == 0) + fileref_drop(ref); +} + +static void +fileref_expiration_check(void *arg, u_int64_t now) +{ + struct kore_fileref *ref, *next; + + for (ref = TAILQ_FIRST(&refs); ref != NULL; ref = next) { + next = TAILQ_NEXT(ref, list); + + if (ref->cnt != 0) + continue; + + if (ref->expiration > now) + continue; + +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p expired, removing", (void *)ref); +#endif + + fileref_drop(ref); + } + + if (TAILQ_EMPTY(&refs)) { + /* remove the timer. */ + ref_timer->flags |= KORE_TIMER_ONESHOT; + ref_timer = NULL; + } +} + +static void +fileref_drop(struct kore_fileref *ref) +{ +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p dropped", (void *)ref); +#endif + + if (!(ref->flags & KORE_FILEREF_SOFT_REMOVED)) + TAILQ_REMOVE(&refs, ref, list); + + kore_free(ref->path); + +#if !defined(KORE_USE_PLATFORM_SENDFILE) + (void)munmap(ref->base, ref->size); +#else + close(ref->fd); +#endif + kore_pool_put(&ref_pool, ref); +} diff -Nru kore-2.0.0/src/http.c kore-3.3.1/src/http.c --- kore-2.0.0/src/http.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/http.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,18 +15,26 @@ */ #include +#include + +#include +#include #include +#include #include +#include +#include #include #include -#include -#include - #include "kore.h" #include "http.h" +#if defined(KORE_USE_PYTHON) +#include "python_api.h" +#endif + #if defined(KORE_USE_PGSQL) #include "pgsql.h" #endif @@ -35,14 +43,90 @@ #include "tasks.h" #endif +#if defined(KORE_USE_CURL) +#include "curl.h" +#endif + +static struct { + const char *ext; + const char *type; +} builtin_media[] = { + { "gif", "image/gif" }, + { "png", "image/png" }, + { "jpeg", "image/jpeg" }, + { "jpg", "image/jpeg" }, + { "zip", "application/zip" }, + { "pdf", "application/pdf" }, + { "json", "application/json" }, + { "js", "application/javascript" }, + { "htm", "text/html" }, + { "txt", "text/plain" }, + { "css", "text/css" }, + { "html", "text/html" }, + { NULL, NULL }, +}; + +#define HTTP_MAP_LIMIT 127 + +/* + * token = 1* + * separators = "(" | ")" | "<" | ">" | "@" + * | "," | ";" | ":" | "\" | <"> + * | "/" | "[" | "]" | "?" | "=" + * | "{" | "}" | SP | HT + */ +static const char http_token[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, '!' , 0x00, '#' , '$' , '%' , '&' , '\'', + 0x00, 0x00, '*' , '+' , 0x00, '-' , '.' , 0x00, + '0' , '1' , '2' , '3' , '4' , '5' , '6' , '7' , + '8' , '9' , 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 'A' , 'B' , 'C' , 'D' , 'E' , 'F' , 'G' , + 'H' , 'I' , 'J' , 'K' , 'L' , 'M' , 'N' , 'O' , + 'P' , 'Q' , 'R' , 'S' , 'T' , 'U' , 'V' , 'W' , + 'X' , 'Y' , 'Z' , 0x00, 0x00, 0x00, '^' , '_' , + '`' , 'a' , 'b' , 'c' , 'd' , 'e' , 'f' , 'g' , + 'h' , 'i' , 'j' , 'k' , 'l' , 'm' , 'n' , 'o' , + 'p' , 'q' , 'r' , 's' , 't' , 'u' , 'v' , 'w' , + 'x' , 'y' , 'z' , 0x00, '|' , 0x00, '~' , +}; + +/* + * field-content = + */ +static const char http_field_content[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ' ' , '!' , '"' , '#' , '$' , '%' , '&' , '\'', + '(' , ')' , '*' , '+' , ',' , '-' , '.' , '/' , + '0' , '1' , '2' , '3' , '4' , '5' , '6' , '7' , + '8' , '9' , ':' , ';' , '<' , '=' , '>' , '?' , + '@' , 'A' , 'B' , 'C' , 'D' , 'E' , 'F' , 'G' , + 'H' , 'I' , 'J' , 'K' , 'L' , 'M' , 'N' , 'O' , + 'P' , 'Q' , 'R' , 'S' , 'T' , 'U' , 'V' , 'W' , + 'X' , 'Y' , 'Z' , '[' , '\\', ']' , '^' , '_' , + '`' , 'a' , 'b' , 'c' , 'd' , 'e' , 'f' , 'g' , + 'h' , 'i' , 'j' , 'k' , 'l' , 'm' , 'n' , 'o' , + 'p' , 'q' , 'r' , 's' , 't' , 'u' , 'v' , 'w' , + 'x' , 'y' , 'z' , '{' , '|' , '}' , '~' , +}; + static int http_body_recv(struct netbuf *); -static int http_body_rewind(struct http_request *); static void http_error_response(struct connection *, int); -static void http_argument_add(struct http_request *, const char *, char *); +static void http_write_response_cookie(struct http_cookie *); +static void http_argument_add(struct http_request *, char *, char *, + int, int); static void http_response_normal(struct http_request *, struct connection *, int, const void *, size_t); static void multipart_add_field(struct http_request *, struct kore_buf *, - const char *, const char *, const int); + char *, const char *, const int); static void multipart_file_add(struct http_request *, struct kore_buf *, const char *, const char *, const char *, const int); static int multipart_find_data(struct kore_buf *, struct kore_buf *, @@ -51,39 +135,56 @@ struct kore_buf *, struct kore_buf *, const char *, const int); +static struct http_request *http_request_new(struct connection *, + const char *, const char *, char *, + const char *); + static struct kore_buf *header_buf; -static char http_version[32]; +static struct kore_buf *ckhdr_buf; +static char http_version[64]; static u_int16_t http_version_len; static TAILQ_HEAD(, http_request) http_requests; static TAILQ_HEAD(, http_request) http_requests_sleeping; +static LIST_HEAD(, http_media_type) http_media_types; static struct kore_pool http_request_pool; -static struct kore_pool http_header_pool; -static struct kore_pool http_host_pool; -static struct kore_pool http_path_pool; +static struct kore_pool http_cookie_pool; static struct kore_pool http_body_path; +static struct kore_pool http_rlq_pool; + +struct kore_pool http_header_pool; -int http_request_count = 0; +u_int32_t http_request_count = 0; +u_int32_t http_request_ms = HTTP_REQUEST_MS; +u_int16_t http_body_timeout = HTTP_BODY_TIMEOUT; u_int32_t http_request_limit = HTTP_REQUEST_LIMIT; u_int64_t http_hsts_enable = HTTP_HSTS_ENABLE; u_int16_t http_header_max = HTTP_HEADER_MAX_LEN; u_int16_t http_keepalive_time = HTTP_KEEPALIVE_TIME; -u_int64_t http_body_max = HTTP_BODY_MAX_LEN; -u_int64_t http_body_disk_offload = HTTP_BODY_DISK_OFFLOAD; +u_int16_t http_header_timeout = HTTP_HEADER_TIMEOUT; + +size_t http_body_max = HTTP_BODY_MAX_LEN; char *http_body_disk_path = HTTP_BODY_DISK_PATH; +u_int64_t http_body_disk_offload = HTTP_BODY_DISK_OFFLOAD; + +void +http_parent_init(void) +{ + LIST_INIT(&http_media_types); +} void http_init(void) { - int prealloc, l; + int prealloc, l, i; TAILQ_INIT(&http_requests); TAILQ_INIT(&http_requests_sleeping); - header_buf = kore_buf_alloc(1024); + header_buf = kore_buf_alloc(HTTP_HEADER_BUFSIZE); + ckhdr_buf = kore_buf_alloc(HTTP_COOKIE_BUFSIZE); l = snprintf(http_version, sizeof(http_version), - "server: kore (%d.%d.%d-%s)\r\n", KORE_VERSION_MAJOR, - KORE_VERSION_MINOR, KORE_VERSION_PATCH, KORE_VERSION_STATE); + "server: kore (%s)\r\n", kore_version); if (l == -1 || (size_t)l >= sizeof(http_version)) fatal("http_init(): http_version buffer too small"); @@ -91,16 +192,24 @@ prealloc = MIN((worker_max_connections / 10), 1000); kore_pool_init(&http_request_pool, "http_request_pool", - sizeof(struct http_request), prealloc); + sizeof(struct http_request), http_request_limit); kore_pool_init(&http_header_pool, "http_header_pool", sizeof(struct http_header), prealloc * HTTP_REQ_HEADER_MAX); + kore_pool_init(&http_cookie_pool, "http_cookie_pool", + sizeof(struct http_cookie), prealloc * HTTP_MAX_COOKIES); + kore_pool_init(&http_rlq_pool, "http_rlq_pool", + sizeof(struct http_runlock_queue), http_request_limit); - kore_pool_init(&http_host_pool, - "http_host_pool", KORE_DOMAINNAME_LEN, prealloc); - kore_pool_init(&http_path_pool, - "http_path_pool", HTTP_URI_LEN, prealloc); kore_pool_init(&http_body_path, "http_body_path", HTTP_BODY_PATH_MAX, prealloc); + + for (i = 0; builtin_media[i].ext != NULL; i++) { + if (!http_media_register(builtin_media[i].ext, + builtin_media[i].type)) { + fatal("duplicate media type for %s", + builtin_media[i].ext); + } + } } void @@ -111,134 +220,48 @@ header_buf = NULL; } + if (ckhdr_buf != NULL) { + kore_buf_free(ckhdr_buf); + ckhdr_buf = NULL; + } + kore_pool_cleanup(&http_request_pool); kore_pool_cleanup(&http_header_pool); - kore_pool_cleanup(&http_host_pool); - kore_pool_cleanup(&http_path_pool); kore_pool_cleanup(&http_body_path); } -int -http_request_new(struct connection *c, const char *host, - const char *method, const char *path, const char *version, - struct http_request **out) +void +http_server_version(const char *version) { - char *p; - struct http_request *req; - struct kore_module_handle *hdlr; - int m, flags; - size_t hostlen, pathlen, qsoff; - - kore_debug("http_request_new(%p, %s, %s, %s, %s)", c, host, - method, path, version); + int l; - if ((hostlen = strlen(host)) >= KORE_DOMAINNAME_LEN - 1) { - http_error_response(c, 500); - return (KORE_RESULT_ERROR); - } - - if ((pathlen = strlen(path)) >= HTTP_URI_LEN - 1) { - http_error_response(c, 414); - return (KORE_RESULT_ERROR); - } + l = snprintf(http_version, sizeof(http_version), + "server: %s\r\n", version); + if (l == -1 || (size_t)l >= sizeof(http_version)) + fatal("http_server_version(): http_version buffer too small"); - if (strcasecmp(version, "http/1.1")) { - http_error_response(c, 505); - return (KORE_RESULT_ERROR); - } + http_version_len = l; +} - if ((p = strchr(path, '?')) != NULL) { - *p = '\0'; - qsoff = p - path; - } else { - qsoff = 0; - } +int +http_check_timeout(struct connection *c, u_int64_t now) +{ + u_int64_t d; - if ((hdlr = kore_module_handler_find(host, path)) == NULL) { - http_error_response(c, 404); - return (KORE_RESULT_ERROR); - } + if (c->http_timeout == 0) + return (KORE_RESULT_OK); - if (p != NULL) - *p = '?'; + if (now > c->http_start) + d = now - c->http_start; + else + d = 0; - if (!strcasecmp(method, "get")) { - m = HTTP_METHOD_GET; - flags = HTTP_REQUEST_COMPLETE; - } else if (!strcasecmp(method, "delete")) { - m = HTTP_METHOD_DELETE; - flags = HTTP_REQUEST_COMPLETE; - } else if (!strcasecmp(method, "post")) { - m = HTTP_METHOD_POST; - flags = HTTP_REQUEST_EXPECT_BODY; - } else if (!strcasecmp(method, "put")) { - m = HTTP_METHOD_PUT; - flags = HTTP_REQUEST_EXPECT_BODY; - } else if (!strcasecmp(method, "head")) { - m = HTTP_METHOD_HEAD; - flags = HTTP_REQUEST_COMPLETE; - } else { - http_error_response(c, 400); + if (d >= c->http_timeout) { + http_error_response(c, 408); + kore_connection_disconnect(c); return (KORE_RESULT_ERROR); } - req = kore_pool_get(&http_request_pool); - req->end = 0; - req->total = 0; - req->start = 0; - req->owner = c; - req->status = 0; - req->method = m; - req->hdlr = hdlr; - req->agent = NULL; - req->flags = flags; - req->fsm_state = 0; - req->http_body = NULL; - req->http_body_fd = -1; - req->hdlr_extra = NULL; - req->query_string = NULL; - req->http_body_length = 0; - req->http_body_offset = 0; - req->http_body_path = NULL; - - if ((p = strrchr(host, ':')) != NULL) - *p = '\0'; - - req->host = kore_pool_get(&http_host_pool); - memcpy(req->host, host, hostlen); - req->host[hostlen] = '\0'; - - req->path = kore_pool_get(&http_path_pool); - memcpy(req->path, path, pathlen); - req->path[pathlen] = '\0'; - - if (qsoff > 0) { - req->query_string = req->path + qsoff; - *(req->query_string)++ = '\0'; - } else { - req->query_string = NULL; - } - - TAILQ_INIT(&(req->resp_headers)); - TAILQ_INIT(&(req->req_headers)); - TAILQ_INIT(&(req->arguments)); - TAILQ_INIT(&(req->files)); - -#if defined(KORE_USE_TASKS) - LIST_INIT(&(req->tasks)); -#endif - -#if defined(KORE_USE_PGSQL) - LIST_INIT(&(req->pgsqls)); -#endif - - http_request_count++; - TAILQ_INSERT_HEAD(&http_requests, req, list); - TAILQ_INSERT_TAIL(&(c->http_requests), req, olist); - - if (out != NULL) - *out = req; - return (KORE_RESULT_OK); } @@ -269,12 +292,13 @@ void http_process(void) { - u_int32_t count; + u_int64_t total; struct http_request *req, *next; - count = 0; + total = 0; + for (req = TAILQ_FIRST(&http_requests); req != NULL; req = next) { - if (count >= http_request_limit) + if (total >= http_request_ms) break; next = TAILQ_NEXT(req, list); @@ -290,15 +314,18 @@ if (!(req->flags & HTTP_REQUEST_COMPLETE)) continue; - count++; http_process_request(req); + total += req->ms; + + if (req->flags & HTTP_REQUEST_DELETE) + http_request_free(req); } } void http_process_request(struct http_request *req) { - int r, (*cb)(struct http_request *); + int r; kore_debug("http_process_request: %p->%p (%s)", req->owner, req, req->path); @@ -314,10 +341,7 @@ switch (r) { case KORE_RESULT_OK: - *(void **)&(cb) = req->hdlr->addr; - worker->active_hdlr = req->hdlr; - r = cb(req); - worker->active_hdlr = NULL; + r = kore_runtime_http_request(req->hdlr->rcall, req); break; case KORE_RESULT_RETRY: break; @@ -332,7 +356,8 @@ fatal("kore_auth() returned unknown %d", r); } req->end = kore_time_ms(); - req->total += req->end - req->start; + req->ms = req->end - req->start; + req->total += req->ms; switch (r) { case KORE_RESULT_OK: @@ -349,7 +374,7 @@ fatal("A page handler returned an unknown result: %d", r); } - if (req->hdlr->dom->accesslog != -1) + if (req->hdlr->dom->accesslog) kore_accesslog(req); req->flags |= HTTP_REQUEST_DELETE; @@ -379,9 +404,21 @@ #if defined(KORE_USE_PGSQL) struct kore_pgsql *pgsql; #endif +#if defined(KORE_USE_CURL) + struct kore_curl *client; +#endif struct http_file *f, *fnext; struct http_arg *q, *qnext; struct http_header *hdr, *next; + struct http_cookie *ck, *cknext; + + if (req->onfree != NULL) + req->onfree(req); + + if (req->runlock != NULL) { + LIST_REMOVE(req->runlock, list); + req->runlock = NULL; + } #if defined(KORE_USE_TASKS) pending_tasks = 0; @@ -400,30 +437,37 @@ } #endif +#if defined(KORE_USE_PYTHON) + if (req->py_coro != NULL) { + kore_python_coro_delete(req->py_coro); + req->py_coro = NULL; + } +#endif #if defined(KORE_USE_PGSQL) while (!LIST_EMPTY(&(req->pgsqls))) { pgsql = LIST_FIRST(&(req->pgsqls)); kore_pgsql_cleanup(pgsql); } - - if (req->flags & HTTP_REQUEST_PGSQL_QUEUE) - kore_pgsql_queue_remove(req); #endif - +#if defined(KORE_USE_CURL) + while (!LIST_EMPTY(&req->chandles)) { + client = LIST_FIRST(&req->chandles); + kore_curl_cleanup(client); + } +#endif kore_debug("http_request_free: %p->%p", req->owner, req); - - kore_pool_put(&http_host_pool, req->host); - kore_pool_put(&http_path_pool, req->path); + kore_free(req->headers); req->host = NULL; req->path = NULL; + req->headers = NULL; TAILQ_REMOVE(&http_requests, req, list); - TAILQ_REMOVE(&(req->owner->http_requests), req, olist); + if (req->owner != NULL) + TAILQ_REMOVE(&(req->owner->http_requests), req, olist); for (hdr = TAILQ_FIRST(&(req->resp_headers)); hdr != NULL; hdr = next) { next = TAILQ_NEXT(hdr, list); - TAILQ_REMOVE(&(req->resp_headers), hdr, list); kore_free(hdr->header); kore_free(hdr->value); @@ -432,27 +476,39 @@ for (hdr = TAILQ_FIRST(&(req->req_headers)); hdr != NULL; hdr = next) { next = TAILQ_NEXT(hdr, list); - TAILQ_REMOVE(&(req->req_headers), hdr, list); - kore_free(hdr->header); - kore_free(hdr->value); kore_pool_put(&http_header_pool, hdr); } + for (ck = TAILQ_FIRST(&(req->resp_cookies)); ck != NULL; ck = cknext) { + cknext = TAILQ_NEXT(ck, list); + TAILQ_REMOVE(&(req->resp_cookies), ck, list); + kore_free(ck->name); + kore_free(ck->value); + kore_free(ck->path); + kore_free(ck->domain); + kore_pool_put(&http_cookie_pool, ck); + } + + for (ck = TAILQ_FIRST(&(req->req_cookies)); ck != NULL; ck = cknext) { + cknext = TAILQ_NEXT(ck, list); + TAILQ_REMOVE(&(req->req_cookies), ck, list); + kore_free(ck->name); + kore_free(ck->value); + kore_pool_put(&http_cookie_pool, ck); + } + for (q = TAILQ_FIRST(&(req->arguments)); q != NULL; q = qnext) { qnext = TAILQ_NEXT(q, list); - TAILQ_REMOVE(&(req->arguments), q, list); kore_free(q->name); - if (q->s_value != NULL) - kore_free(q->s_value); + kore_free(q->s_value); kore_free(q); } for (f = TAILQ_FIRST(&(req->files)); f != NULL; f = fnext) { fnext = TAILQ_NEXT(f, list); TAILQ_REMOVE(&(req->files), f, list); - kore_free(f->filename); kore_free(f->name); kore_free(f); @@ -465,7 +521,7 @@ (void)close(req->http_body_fd); if (req->http_body_path != NULL) { - if (unlink(req->http_body_path) == -1) { + if (unlink(req->http_body_path) == -1 && errno != ENOENT) { kore_log(LOG_NOTICE, "failed to unlink %s: %s", req->http_body_path, errno_s); } @@ -481,12 +537,38 @@ } void +http_serveable(struct http_request *req, const void *data, size_t len, + const char *etag, const char *type) +{ + const char *match; + + if (req->method != HTTP_METHOD_GET) { + http_response_header(req, "allow", "get"); + http_response(req, HTTP_STATUS_BAD_REQUEST, NULL, 0); + return; + } + + if (http_request_header(req, "if-none-match", &match)) { + if (!strcmp(match, etag)) { + http_response(req, HTTP_STATUS_NOT_MODIFIED, NULL, 0); + return; + } + } + + http_response_header(req, "etag", etag); + http_response_header(req, "content-type", type); + http_response(req, HTTP_STATUS_OK, data, len); +} + +void http_response(struct http_request *req, int status, const void *d, size_t l) { + if (req->owner == NULL) + return; + kore_debug("http_response(%p, %d, %p, %zu)", req, status, d, l); req->status = status; - switch (req->owner->proto) { case CONN_PROTO_HTTP: case CONN_PROTO_WEBSOCKET: @@ -504,6 +586,9 @@ { struct netbuf *nb; + if (req->owner == NULL) + return; + req->status = status; switch (req->owner->proto) { @@ -521,8 +606,57 @@ } } +void +http_response_fileref(struct http_request *req, int status, + struct kore_fileref *ref) +{ + struct tm *tm; + time_t mtime; + char tbuf[128]; + const char *media_type, *modified; + + if (req->owner == NULL) + return; + + media_type = http_media_type(ref->path); + if (media_type != NULL) + http_response_header(req, "content-type", media_type); + + if (http_request_header(req, "if-modified-since", &modified)) { + mtime = kore_date_to_time(modified); + if (mtime == ref->mtime_sec) { + kore_fileref_release(ref); + http_response(req, HTTP_STATUS_NOT_MODIFIED, NULL, 0); + return; + } + } + + if ((tm = gmtime(&ref->mtime_sec)) != NULL) { + if (strftime(tbuf, sizeof(tbuf), + "%a, %d %b %Y %H:%M:%S GMT", tm) > 0) { + http_response_header(req, "last-modified", tbuf); + } + } + + req->status = status; + switch (req->owner->proto) { + case CONN_PROTO_HTTP: + http_response_normal(req, req->owner, status, NULL, ref->size); + break; + default: + fatal("http_response_fd() bad proto %d", req->owner->proto); + /* NOTREACHED. */ + } + + if (req->method != HTTP_METHOD_HEAD) + net_send_fileref(req->owner, ref); + else + kore_fileref_release(ref); +} + int -http_request_header(struct http_request *req, const char *header, char **out) +http_request_header(struct http_request *req, const char *header, + const char **out) { struct http_header *hdr; @@ -533,23 +667,45 @@ } } + if (!strcasecmp(header, "host")) { + *out = req->host; + return (KORE_RESULT_OK); + } + + return (KORE_RESULT_ERROR); +} + +int +http_request_cookie(struct http_request *req, const char *cookie, char **out) +{ + struct http_cookie *ck; + + TAILQ_FOREACH(ck, &(req->req_cookies), list) { + if (!strcasecmp(ck->name, cookie)) { + *out = ck->value; + return (KORE_RESULT_OK); + } + } + return (KORE_RESULT_ERROR); } int http_header_recv(struct netbuf *nb) { + struct connection *c; size_t len; ssize_t ret; struct http_header *hdr; struct http_request *req; + const char *clp; u_int64_t bytes_left; u_int8_t *end_headers; int h, i, v, skip, l; - char *request[4], *host[3], *hbuf; - char *p, *headers[HTTP_REQ_HEADER_MAX]; - struct connection *c = (struct connection *)nb->owner; + char *headers[HTTP_REQ_HEADER_MAX]; + char *value, *host, *request[4], *hbuf; + c = nb->owner; kore_debug("http_header_recv(%p)", nb); if (nb->b_len < 4) @@ -582,58 +738,67 @@ } skip = 0; - host[0] = NULL; + host = NULL; for (i = 0; i < h; i++) { if (strncasecmp(headers[i], "host", 4)) continue; - v = kore_split_string(headers[i], ":", host, 3); - if (v != 2) { + if ((host = http_validate_header(headers[i])) == NULL) { http_error_response(c, 400); return (KORE_RESULT_OK); } - if ((host[1] - host[0]) != 5 || - strncasecmp(host[0], "host", 4) || host[1] == '\0') { + if (*host == '\0') { http_error_response(c, 400); return (KORE_RESULT_OK); } - host[1]++; skip = i; break; } - if (host[0] == NULL) { + if (host == NULL) { http_error_response(c, 400); return (KORE_RESULT_OK); } - if (!http_request_new(c, host[1], - request[0], request[1], request[2], &req)) + req = http_request_new(c, host, request[0], request[1], request[2]); + if (req == NULL) return (KORE_RESULT_OK); + /* take full ownership of the buffer. */ + req->headers = nb->buf; + nb->buf = NULL; + nb->m_len = 0; + for (i = 1; i < h; i++) { if (i == skip) continue; - p = strchr(headers[i], ':'); - if (p == NULL) { - kore_debug("malformed header: '%s'", headers[i]); - continue; + if ((value = http_validate_header(headers[i])) == NULL) { + req->flags |= HTTP_REQUEST_DELETE; + http_error_response(c, 400); + return (KORE_RESULT_OK); + } + + if (*value == '\0') { + req->flags |= HTTP_REQUEST_DELETE; + http_error_response(c, 400); + return (KORE_RESULT_OK); } - *(p++) = '\0'; - if (*p == ' ') - p++; hdr = kore_pool_get(&http_header_pool); - hdr->header = kore_strdup(headers[i]); - hdr->value = kore_strdup(p); + hdr->header = headers[i]; + hdr->value = value; TAILQ_INSERT_TAIL(&(req->req_headers), hdr, list); if (req->agent == NULL && !strcasecmp(hdr->header, "user-agent")) req->agent = hdr->value; + + if (req->referer == NULL && + !strcasecmp(hdr->header, "referer")) + req->referer = hdr->value; } if (req->flags & HTTP_REQUEST_EXPECT_BODY) { @@ -643,16 +808,16 @@ return (KORE_RESULT_OK); } - if (!http_request_header(req, "content-length", &p)) { + if (!http_request_header(req, "content-length", &clp)) { kore_debug("expected body but no content-length"); req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, 411); return (KORE_RESULT_OK); } - req->content_length = kore_strtonum(p, 10, 0, LONG_MAX, &v); + req->content_length = kore_strtonum(clp, 10, 0, LONG_MAX, &v); if (v == KORE_RESULT_ERROR) { - kore_debug("content-length invalid: %s", p); + kore_debug("content-length invalid: %s", clp); req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, 411); return (KORE_RESULT_OK); @@ -665,7 +830,7 @@ } if (req->content_length > http_body_max) { - kore_log(LOG_NOTICE, "body too large (%ld > %ld)", + kore_log(LOG_NOTICE, "body too large (%zu > %zu)", req->content_length, http_body_max); req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, 413); @@ -707,6 +872,9 @@ (nb->s_off - len)); } + SHA256_Init(&req->hashctx); + SHA256_Update(&req->hashctx, end_headers, (nb->s_off - len)); + bytes_left = req->content_length - (nb->s_off - len); if (bytes_left > 0) { kore_debug("%ld/%ld (%ld - %ld) more bytes for body", @@ -717,17 +885,20 @@ c->rnb->extra = req; http_request_sleep(req); req->content_length = bytes_left; - } else if (bytes_left == 0) { + c->http_timeout = http_body_timeout * 1000; + } else { + c->http_timeout = 0; req->flags |= HTTP_REQUEST_COMPLETE; req->flags &= ~HTTP_REQUEST_EXPECT_BODY; + SHA256_Final(req->http_body_digest, &req->hashctx); if (!http_body_rewind(req)) { req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, 500); return (KORE_RESULT_OK); } - } else { - http_error_response(req->owner, 500); } + } else { + c->http_timeout = 0; } return (KORE_RESULT_OK); @@ -768,6 +939,12 @@ case HTTP_ARG_TYPE_UINT64: COPY_AS_INTTYPE_64(u_int64_t, 0); return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_FLOAT: + COPY_ARG_DOUBLE(-FLT_MAX, FLT_MAX, float); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_DOUBLE: + COPY_ARG_DOUBLE(-DBL_MAX, DBL_MAX, double); + return (KORE_RESULT_OK); case HTTP_ARG_TYPE_STRING: *out = q->s_value; return (KORE_RESULT_OK); @@ -817,10 +994,13 @@ h[3] = *(p + 2); h[4] = '\0'; - v = kore_strtonum(h, 16, 0, 255, &err); + v = kore_strtonum(h, 16, 0x0, 0xff, &err); if (err != KORE_RESULT_OK) return (err); + if (v <= 0x1f || v == 0x7f || (v >= 0x80 && v <= 0x9f)) + return (KORE_RESULT_ERROR); + *in++ = (char)v; p += 3; } @@ -857,7 +1037,7 @@ off = file->position + file->offset; toread = MIN(len, (file->length - file->offset)); - if (toread <= 0) + if (toread == 0) return (0); if (file->req->http_body_fd != -1) { @@ -901,6 +1081,71 @@ } void +http_response_cookie(struct http_request *req, const char *name, + const char *val, const char *path, time_t expires, u_int32_t maxage, + struct http_cookie **out) +{ + char *p; + struct http_cookie *ck; + + if (name == NULL || val == NULL) + fatal("http_response_cookie: invalid parameters"); + + ck = kore_pool_get(&http_cookie_pool); + + ck->maxage = maxage; + ck->expires = expires; + ck->name = kore_strdup(name); + ck->value = kore_strdup(val); + ck->domain = kore_strdup(req->host); + ck->flags = HTTP_COOKIE_HTTPONLY | HTTP_COOKIE_SECURE; + + if ((p = strrchr(ck->domain, ':')) != NULL) + *p = '\0'; + + if (path != NULL) + ck->path = kore_strdup(path); + else + ck->path = NULL; + + TAILQ_INSERT_TAIL(&(req->resp_cookies), ck, list); + + if (out != NULL) + *out = ck; +} + +void +http_populate_cookies(struct http_request *req) +{ + struct http_cookie *ck; + const char *hdr; + int i, v, n; + char *c, *header, *pair[3]; + char *cookies[HTTP_MAX_COOKIES]; + + if (!http_request_header(req, "cookie", &hdr)) + return; + + header = kore_strdup(hdr); + v = kore_split_string(header, ";", cookies, HTTP_MAX_COOKIES); + for (i = 0; i < v; i++) { + for (c = cookies[i]; isspace(*(unsigned char *)c); c++) + ; + + n = kore_split_string(c, "=", pair, 3); + if (n != 2) + continue; + + ck = kore_pool_get(&http_cookie_pool); + ck->name = kore_strdup(pair[0]); + ck->value = kore_strdup(pair[1]); + TAILQ_INSERT_TAIL(&(req->req_cookies), ck, list); + } + + kore_free(header); +} + +void http_populate_post(struct http_request *req) { ssize_t ret; @@ -916,6 +1161,8 @@ body = NULL; req->http_body->offset = req->content_length; string = kore_buf_stringify(req->http_body, NULL); + req->http_body_length = 0; + req->http_body_offset = 0; } else { body = kore_buf_alloc(128); for (;;) { @@ -933,7 +1180,7 @@ for (i = 0; i < v; i++) { kore_split_string(args[i], "=", val, 3); if (val[0] != NULL && val[1] != NULL) - http_argument_add(req, val[0], val[1]); + http_argument_add(req, val[0], val[1], 0, 1); } out: @@ -942,12 +1189,12 @@ } void -http_populate_get(struct http_request *req) +http_populate_qs(struct http_request *req) { int i, v; char *query, *args[HTTP_MAX_QUERY_ARGS], *val[3]; - if (req->method != HTTP_METHOD_GET || req->query_string == NULL) + if (req->query_string == NULL) return; query = kore_strdup(req->query_string); @@ -955,7 +1202,7 @@ for (i = 0; i < v; i++) { kore_split_string(args[i], "=", val, 3); if (val[0] != NULL && val[1] != NULL) - http_argument_add(req, val[0], val[1]); + http_argument_add(req, val[0], val[1], 1, 1); } kore_free(query); @@ -964,54 +1211,100 @@ void http_populate_multipart_form(struct http_request *req) { + const char *hdr; int h, blen; - struct kore_buf *in, *out; + struct kore_buf in, out; char *type, *val, *args[3]; char boundary[HTTP_BOUNDARY_MAX]; if (req->method != HTTP_METHOD_POST) return; - if (!http_request_header(req, "content-type", &type)) + if (!http_request_header(req, "content-type", &hdr)) return; + kore_buf_init(&in, 128); + kore_buf_init(&out, 128); + + type = kore_strdup(hdr); h = kore_split_string(type, ";", args, 3); if (h != 2) - return; + goto cleanup; if (strcasecmp(args[0], "multipart/form-data")) - return; + goto cleanup; if ((val = strchr(args[1], '=')) == NULL) - return; + goto cleanup; val++; blen = snprintf(boundary, sizeof(boundary), "--%s", val); if (blen == -1 || (size_t)blen >= sizeof(boundary)) - return; - - in = kore_buf_alloc(128); - out = kore_buf_alloc(128); + goto cleanup; - if (!multipart_find_data(in, NULL, NULL, req, boundary, blen)) + if (!multipart_find_data(&in, NULL, NULL, req, boundary, blen)) goto cleanup; for (;;) { - if (!multipart_find_data(in, NULL, NULL, req, "\r\n", 2)) + if (!multipart_find_data(&in, NULL, NULL, req, "\r\n", 2)) break; - if (in->offset < 4 && req->http_body_length == 0) + if (in.offset < 4 && req->http_body_length == 0) break; - if (!multipart_find_data(in, out, NULL, req, "\r\n\r\n", 4)) + if (!multipart_find_data(&in, &out, NULL, req, "\r\n\r\n", 4)) break; - if (!multipart_parse_headers(req, in, out, boundary, blen)) + if (!multipart_parse_headers(req, &in, &out, boundary, blen)) break; - kore_buf_reset(out); + kore_buf_reset(&out); } cleanup: - kore_buf_free(in); - kore_buf_free(out); + kore_free(type); + kore_buf_cleanup(&in); + kore_buf_cleanup(&out); +} + +int +http_body_rewind(struct http_request *req) +{ + if (req->http_body_fd != -1) { + if (lseek(req->http_body_fd, 0, SEEK_SET) == -1) { + kore_log(LOG_ERR, "lseek(%s) failed: %s", + req->http_body_path, errno_s); + return (KORE_RESULT_ERROR); + } + } else if (req->http_body != NULL) { + kore_buf_reset(req->http_body); + } + + req->http_body_offset = 0; + req->http_body_length = req->content_length; + + return (KORE_RESULT_OK); +} + +int +http_body_digest(struct http_request *req, char *out, size_t len) +{ + size_t idx; + int slen; + + if (len != HTTP_BODY_DIGEST_STRLEN) { + fatal("http_body_digest: bad len:%zu wanted:%zu", + len, HTTP_BODY_DIGEST_STRLEN); + } + + if (!(req->flags & HTTP_REQUEST_COMPLETE)) + return (KORE_RESULT_ERROR); + + for (idx = 0; idx < sizeof(req->http_body_digest); idx++) { + slen = snprintf(out + (idx * 2), len - (idx * 2), "%02x", + req->http_body_digest[idx]); + if (slen == -1 || (size_t)slen >= len) + fatal("failed to create hex string"); + } + + return (KORE_RESULT_OK); } ssize_t @@ -1021,7 +1314,7 @@ size_t toread; toread = MIN(req->http_body_length, len); - if (toread <= 0) + if (toread == 0) return (0); if (req->http_body_fd != -1) { @@ -1093,6 +1386,273 @@ return (KORE_RESULT_OK); } +int +http_state_exists(struct http_request *req) +{ + return (req->hdlr_extra != NULL); +} + +void * +http_state_create(struct http_request *req, size_t len, + void (*onfree)(struct http_request *)) +{ + if (req->hdlr_extra != NULL) + fatal("http_state_create: state already exists"); + + req->state_len = len; + req->onfree = onfree; + req->hdlr_extra = kore_calloc(1, len); + + return (req->hdlr_extra); +} + +void * +http_state_get(struct http_request *req) +{ + return (req->hdlr_extra); +} + +void +http_state_cleanup(struct http_request *req) +{ + kore_free(req->hdlr_extra); + req->hdlr_extra = NULL; +} + +void +http_start_recv(struct connection *c) +{ + c->http_start = kore_time_ms(); + c->http_timeout = http_header_timeout * 1000; + net_recv_reset(c, http_header_max, http_header_recv); + (void)net_recv_flush(c); +} + +void +http_runlock_init(struct http_runlock *lock) +{ + lock->owner = NULL; + LIST_INIT(&lock->queue); +} + +int +http_runlock_acquire(struct http_runlock *lock, struct http_request *req) +{ + if (lock->owner != NULL) { + if (req->runlock != NULL) + fatal("%s: request already waiting on lock", __func__); + + req->runlock = kore_pool_get(&http_rlq_pool); + req->runlock->req = req; + LIST_INSERT_HEAD(&lock->queue, req->runlock, list); + + http_request_sleep(req); + return (KORE_RESULT_ERROR); + } + + lock->owner = req; + + return (KORE_RESULT_OK); +} + +void +http_runlock_release(struct http_runlock *lock, struct http_request *req) +{ + struct http_runlock_queue *next; + struct http_request *nextreq; + + if (lock->owner != req) + fatal("%s: calling request != owner of runlock", __func__); + + lock->owner = NULL; + + if ((next = LIST_FIRST(&lock->queue)) != NULL) { + LIST_REMOVE(next, list); + + nextreq = next->req; + nextreq->runlock = NULL; + + http_request_wakeup(nextreq); + kore_pool_put(&http_rlq_pool, next); + } +} + +static struct http_request * +http_request_new(struct connection *c, const char *host, + const char *method, char *path, const char *version) +{ + struct http_request *req; + struct kore_module_handle *hdlr; + char *p, *hp; + int m, flags; + size_t hostlen, pathlen, qsoff; + + if (http_request_count >= http_request_limit) { + http_error_response(c, 503); + return (NULL); + } + + kore_debug("http_request_new(%p, %s, %s, %s, %s)", c, host, + method, path, version); + + if ((hostlen = strlen(host)) >= KORE_DOMAINNAME_LEN - 1) { + http_error_response(c, 400); + return (NULL); + } + + if ((pathlen = strlen(path)) >= HTTP_URI_LEN - 1) { + http_error_response(c, 414); + return (NULL); + } + + if (strcasecmp(version, "http/1.1")) { + if (strcasecmp(version, "http/1.0")) { + http_error_response(c, 505); + return (NULL); + } + + flags = HTTP_VERSION_1_0; + } else { + flags = HTTP_VERSION_1_1; + } + + if ((p = strchr(path, '?')) != NULL) { + *p = '\0'; + qsoff = p - path; + } else { + qsoff = 0; + } + + hp = NULL; + + switch (c->family) { + case AF_INET6: + if (*host == '[') { + if ((hp = strrchr(host, ']')) == NULL) { + http_error_response(c, 400); + return (NULL); + } + hp++; + if (*hp == ':') + *hp = '\0'; + else + hp = NULL; + } + break; + default: + if ((hp = strrchr(host, ':')) != NULL) + *hp = '\0'; + break; + } + + if ((hdlr = kore_module_handler_find(host, path)) == NULL) { + http_error_response(c, 404); + return (NULL); + } + + if (hp != NULL) + *hp = ':'; + + if (p != NULL) + *p = '?'; + + if (!strcasecmp(method, "get")) { + m = HTTP_METHOD_GET; + flags |= HTTP_REQUEST_COMPLETE; + } else if (!strcasecmp(method, "delete")) { + m = HTTP_METHOD_DELETE; + flags |= HTTP_REQUEST_COMPLETE; + } else if (!strcasecmp(method, "post")) { + m = HTTP_METHOD_POST; + flags |= HTTP_REQUEST_EXPECT_BODY; + } else if (!strcasecmp(method, "put")) { + m = HTTP_METHOD_PUT; + flags |= HTTP_REQUEST_EXPECT_BODY; + } else if (!strcasecmp(method, "head")) { + m = HTTP_METHOD_HEAD; + flags |= HTTP_REQUEST_COMPLETE; + } else if (!strcasecmp(method, "options")) { + m = HTTP_METHOD_OPTIONS; + flags |= HTTP_REQUEST_COMPLETE; + } else if (!strcasecmp(method, "patch")) { + m = HTTP_METHOD_PATCH; + flags |= HTTP_REQUEST_EXPECT_BODY; + } else { + http_error_response(c, 400); + return (NULL); + } + + if (flags & HTTP_VERSION_1_0) { + if (m != HTTP_METHOD_GET && m != HTTP_METHOD_POST && + m != HTTP_METHOD_HEAD) { + http_error_response(c, HTTP_STATUS_METHOD_NOT_ALLOWED); + return (NULL); + } + } + + if (!(hdlr->methods & m)) { + http_error_response(c, HTTP_STATUS_METHOD_NOT_ALLOWED); + return (NULL); + } + + req = kore_pool_get(&http_request_pool); + req->end = 0; + req->total = 0; + req->start = 0; + req->owner = c; + req->status = 0; + req->method = m; + req->hdlr = hdlr; + req->agent = NULL; + req->onfree = NULL; + req->referer = NULL; + req->runlock = NULL; + req->flags = flags; + req->fsm_state = 0; + req->http_body = NULL; + req->http_body_fd = -1; + req->hdlr_extra = NULL; + req->query_string = NULL; + req->http_body_length = 0; + req->http_body_offset = 0; + req->http_body_path = NULL; + + req->host = host; + req->path = path; + +#if defined(KORE_USE_PYTHON) + req->py_coro = NULL; +#endif + + if (qsoff > 0) { + req->query_string = path + qsoff; + *(req->query_string)++ = '\0'; + } else { + req->query_string = NULL; + } + + TAILQ_INIT(&(req->resp_headers)); + TAILQ_INIT(&(req->req_headers)); + TAILQ_INIT(&(req->resp_cookies)); + TAILQ_INIT(&(req->req_cookies)); + TAILQ_INIT(&(req->arguments)); + TAILQ_INIT(&(req->files)); + +#if defined(KORE_USE_TASKS) + LIST_INIT(&(req->tasks)); +#endif + +#if defined(KORE_USE_PGSQL) + LIST_INIT(&(req->pgsqls)); +#endif + + http_request_count++; + TAILQ_INSERT_HEAD(&http_requests, req, list); + TAILQ_INSERT_TAIL(&(c->http_requests), req, olist); + + return (req); +} + static int multipart_find_data(struct kore_buf *in, struct kore_buf *out, size_t *olen, struct http_request *req, const void *needle, size_t len) @@ -1182,7 +1742,7 @@ if (strcasecmp(args[0], "content-disposition")) continue; - for (d = args[1]; isspace(*d); d++) + for (d = args[1]; isspace(*(unsigned char *)d); d++) ; c = kore_split_string(d, ";", opt, 5); @@ -1206,7 +1766,7 @@ continue; } - for (d = opt[2]; isspace(*d); d++) + for (d = opt[2]; isspace(*(unsigned char *)d); d++) ; if (!strncasecmp(d, "filename=", 9)) { @@ -1234,7 +1794,7 @@ static void multipart_add_field(struct http_request *req, struct kore_buf *in, - const char *name, const char *boundary, const int blen) + char *name, const char *boundary, const int blen) { struct kore_buf *data; char *string; @@ -1253,7 +1813,7 @@ data->offset -= 2; string = kore_buf_stringify(data, NULL); - http_argument_add(req, name, string); + http_argument_add(req, name, string, 0, 0); kore_buf_free(data); } @@ -1264,7 +1824,7 @@ struct http_file *f; size_t position, len; - position= req->http_body_offset - in->offset; + position = req->http_body_offset - in->offset; if (!multipart_find_data(in, NULL, &len, req, boundary, blen)) return; @@ -1284,18 +1844,30 @@ } static void -http_argument_add(struct http_request *req, const char *name, char *value) +http_argument_add(struct http_request *req, char *name, char *value, int qs, + int decode) { struct http_arg *q; struct kore_handler_params *p; + if (decode) + http_argument_urldecode(name); + TAILQ_FOREACH(p, &(req->hdlr->params), list) { + if (qs == 1 && !(p->flags & KORE_PARAMS_QUERY_STRING)) + continue; + if (qs == 0 && (p->flags & KORE_PARAMS_QUERY_STRING)) + continue; + if (p->method != req->method) continue; + if (strcmp(p->name, name)) continue; - http_argument_urldecode(value); + if (decode) + http_argument_urldecode(value); + if (!kore_validator_check(req, p->validator, value)) break; @@ -1314,6 +1886,8 @@ u_int64_t bytes_left; struct http_request *req = (struct http_request *)nb->extra; + SHA256_Update(&req->hashctx, nb->buf, nb->s_off); + if (req->http_body_fd != -1) { ret = write(req->http_body_fd, nb->buf, nb->s_off); if (ret == -1 || (size_t)ret != nb->s_off) { @@ -1342,6 +1916,7 @@ http_error_response(req->owner, 500); return (KORE_RESULT_ERROR); } + SHA256_Final(req->http_body_digest, &req->hashctx); net_recv_reset(nb->owner, http_header_max, http_header_recv); } else { bytes_left = req->content_length; @@ -1353,26 +1928,11 @@ return (KORE_RESULT_OK); } -static int -http_body_rewind(struct http_request *req) -{ - if (req->http_body_fd != -1) { - if (lseek(req->http_body_fd, 0, SEEK_SET) == -1) { - kore_log(LOG_ERR, "lseek(%s) failed: %s", - req->http_body_path, errno_s); - return (KORE_RESULT_ERROR); - } - } else { - kore_buf_reset(req->http_body); - } - - return (KORE_RESULT_OK); -} - static void http_error_response(struct connection *c, int status) { kore_debug("http_error_response(%p, %d)", c, status); + c->flags |= CONN_CLOSE_EMPTY; switch (c->proto) { case CONN_PROTO_HTTP: @@ -1382,37 +1942,54 @@ fatal("http_error_response() bad proto %d", c->proto); /* NOTREACHED. */ } + + if (!net_send_flush(c)) + kore_connection_disconnect(c); } static void http_response_normal(struct http_request *req, struct connection *c, int status, const void *d, size_t len) { + struct http_cookie *ck; struct http_header *hdr; - char *conn; + const char *conn; + char version; int connection_close; - header_buf->offset = 0; + kore_buf_reset(header_buf); + + if (req != NULL) { + if (req->flags & HTTP_VERSION_1_0) + version = '0'; + else + version = '1'; + } else { + version = '1'; + } - kore_buf_appendf(header_buf, "HTTP/1.1 %d %s\r\n", - status, http_status_text(status)); + kore_buf_appendf(header_buf, "HTTP/1.%c %d %s\r\n", + version, status, http_status_text(status)); kore_buf_append(header_buf, http_version, http_version_len); - if (c->flags & CONN_CLOSE_EMPTY) + if ((c->flags & CONN_CLOSE_EMPTY) || + (req->flags & HTTP_VERSION_1_0)) { connection_close = 1; - else + } else { connection_close = 0; + } if (connection_close == 0 && req != NULL) { if (http_request_header(req, "connection", &conn)) { if ((*conn == 'c' || *conn == 'C') && - !strcasecmp(conn, "close")) + !strcasecmp(conn, "close")) { connection_close = 1; + } } } /* Note that req CAN be NULL. */ - if (req != NULL && req->owner->proto != CONN_PROTO_WEBSOCKET) { + if (req == NULL || req->owner->proto != CONN_PROTO_WEBSOCKET) { if (http_keepalive_time && connection_close == 0) { kore_buf_appendf(header_buf, "connection: keep-alive\r\n"); @@ -1432,6 +2009,9 @@ } if (req != NULL) { + TAILQ_FOREACH(ck, &(req->resp_cookies), list) + http_write_response_cookie(ck); + TAILQ_FOREACH(hdr, &(req->resp_headers), list) { kore_buf_appendf(header_buf, "%s: %s\r\n", hdr->header, hdr->value); @@ -1455,14 +2035,58 @@ if (d != NULL && req != NULL && req->method != HTTP_METHOD_HEAD) net_send_queue(c, d, len); - if (!(c->flags & CONN_CLOSE_EMPTY)) - net_recv_reset(c, http_header_max, http_header_recv); + if (!(c->flags & CONN_CLOSE_EMPTY) && !(c->flags & CONN_IS_BUSY)) + http_start_recv(c); + + if (req != NULL) + req->content_length = len; +} + +static void +http_write_response_cookie(struct http_cookie *ck) +{ + struct tm tm; + char expires[HTTP_DATE_MAXSIZE]; + + kore_buf_reset(ckhdr_buf); + kore_buf_appendf(ckhdr_buf, "%s=%s", ck->name, ck->value); + + if (ck->path != NULL) + kore_buf_appendf(ckhdr_buf, "; Path=%s", ck->path); + if (ck->domain != NULL) + kore_buf_appendf(ckhdr_buf, "; Domain=%s", ck->domain); + + if (ck->expires > 0) { + if (gmtime_r(&ck->expires, &tm) == NULL) { + kore_log(LOG_ERR, "gmtime_r(): %s", errno_s); + return; + } + + if (strftime(expires, sizeof(expires), + "%a, %d %b %y %H:%M:%S GMT", &tm) == 0) { + kore_log(LOG_ERR, "strftime(): %s", errno_s); + return; + } + + kore_buf_appendf(ckhdr_buf, "; Expires=%s", expires); + } + + if (ck->maxage > 0) + kore_buf_appendf(ckhdr_buf, "; Max-Age=%u", ck->maxage); + + if (ck->flags & HTTP_COOKIE_HTTPONLY) + kore_buf_appendf(ckhdr_buf, "; HttpOnly"); + if (ck->flags & HTTP_COOKIE_SECURE) + kore_buf_appendf(ckhdr_buf, "; Secure"); + + kore_buf_appendf(header_buf, "set-cookie: %s\r\n", + kore_buf_stringify(ckhdr_buf, NULL)); } const char * http_status_text(int status) { - char *r; + const char *r; switch (status) { case HTTP_STATUS_CONTINUE: @@ -1614,6 +2238,12 @@ case HTTP_METHOD_HEAD: r = "HEAD"; break; + case HTTP_METHOD_OPTIONS: + r = "OPTIONS"; + break; + case HTTP_METHOD_PATCH: + r = "PATCH"; + break; default: r = ""; break; @@ -1621,3 +2251,82 @@ return (r); } + +int +http_media_register(const char *ext, const char *type) +{ + struct http_media_type *media; + + LIST_FOREACH(media, &http_media_types, list) { + if (!strcasecmp(media->ext, ext)) + return (KORE_RESULT_ERROR); + } + + media = kore_calloc(1, sizeof(*media)); + media->ext = kore_strdup(ext); + media->type = kore_strdup(type); + + LIST_INSERT_HEAD(&http_media_types, media, list); + + return (KORE_RESULT_OK); +} + +const char * +http_media_type(const char *path) +{ + const char *p; + struct http_media_type *media; + + if ((p = strrchr(path, '.')) == NULL) + return (NULL); + + p++; + if (*p == '\0') + return (NULL); + + LIST_FOREACH(media, &http_media_types, list) { + if (!strcasecmp(media->ext, p)) + return (media->type); + } + + return (NULL); +} + +char * +http_validate_header(char *header) +{ + u_int8_t idx; + char *p, *value; + + for (p = header; *p != '\0'; p++) { + idx = *p; + if (idx > HTTP_MAP_LIMIT) + return (NULL); + + if (*p == ':') { + *(p)++ = '\0'; + break; + } + + if (http_token[idx] == 0x00) + return (NULL); + } + + while (isspace(*(unsigned char *)p)) + p++; + + if (*p == '\0') + return (NULL); + + value = p; + while (*p != '\0') { + idx = *p; + if (idx > HTTP_MAP_LIMIT) + return (NULL); + if (http_field_content[idx] == 0x00) + return (NULL); + p++; + } + + return (value); +} diff -Nru kore-2.0.0/src/keymgr.c kore-3.3.1/src/keymgr.c --- kore-2.0.0/src/keymgr.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/keymgr.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Joris Vink + * Copyright (c) 2017-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,61 +14,124 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include +/* + * The kore keymgr process is responsible for managing certificates + * and their matching private keys. + * + * It is the only process in Kore that holds the private keys (the workers + * do not have a copy of them in memory). + * + * When a worker requires the private key for signing it will send a message + * to the keymgr with the to-be-signed data (KORE_MSG_KEYMGR_REQ). The keymgr + * will perform the signing and respond with a KORE_MSG_KEYMGR_RESP message. + * + * The keymgr can transparently reload the private keys and certificates + * for a configured domain when it receives a SIGUSR1. It it reloads them + * it will send the newly loaded certificate chains to the worker processes + * which will update their TLS contexts accordingly. + */ + +#include +#include #include +#include +#include #include #include +#include #include #include #include "kore.h" #if !defined(KORE_NO_TLS) + +#define RAND_TMP_FILE "rnd.tmp" +#define RAND_POLL_INTERVAL (1800 * 1000) +#define RAND_FILE_SIZE 1024 + struct key { EVP_PKEY *pkey; struct kore_domain *dom; TAILQ_ENTRY(key) list; }; +char *rand_file = NULL; + static TAILQ_HEAD(, key) keys; -extern volatile sig_atomic_t sig_recv; static int initialized = 0; +static void keymgr_reload(void); +static void keymgr_load_randfile(void); +static void keymgr_save_randfile(void); + static void keymgr_load_privatekey(struct kore_domain *); static void keymgr_msg_recv(struct kore_msg *, const void *); +static void keymgr_entropy_request(struct kore_msg *, const void *); +static void keymgr_certificate_request(struct kore_msg *, const void *); +static void keymgr_submit_certificates(struct kore_domain *, u_int16_t); +static void keymgr_submit_file(u_int8_t, struct kore_domain *, + const char *, u_int16_t, int); static void keymgr_rsa_encrypt(struct kore_msg *, const void *, struct key *); static void keymgr_ecdsa_sign(struct kore_msg *, const void *, struct key *); +char *keymgr_root_path = NULL; +char *keymgr_runas_user = NULL; + void kore_keymgr_run(void) { int quit; + u_int64_t now, last_seed; quit = 0; - initialized = 1; - TAILQ_INIT(&keys); kore_listener_cleanup(); kore_module_cleanup(); - kore_domain_callback(keymgr_load_privatekey); - kore_worker_privdrop(); - net_init(); kore_connection_init(); kore_platform_event_init(); - kore_msg_worker_init(); kore_msg_register(KORE_MSG_KEYMGR_REQ, keymgr_msg_recv); + kore_msg_register(KORE_MSG_ENTROPY_REQ, keymgr_entropy_request); + kore_msg_register(KORE_MSG_CERTIFICATE_REQ, keymgr_certificate_request); + + kore_worker_privdrop(keymgr_runas_user, keymgr_root_path); + + if (rand_file != NULL) { + keymgr_load_randfile(); + keymgr_save_randfile(); + } else if (!kore_quiet) { + kore_log(LOG_WARNING, "no rand_file location specified"); + } + + initialized = 1; - kore_log(LOG_NOTICE, "key manager started"); + keymgr_reload(); + RAND_poll(); + last_seed = 0; + +#if defined(__OpenBSD__) + if (pledge("stdio rpath", NULL) == -1) + fatal("failed to pledge keymgr process"); +#endif + + if (!kore_quiet) + kore_log(LOG_NOTICE, "key manager started"); while (quit != 1) { + now = kore_time_ms(); + if ((now - last_seed) > RAND_POLL_INTERVAL) { + RAND_poll(); + last_seed = now; + } + if (sig_recv != 0) { switch (sig_recv) { case SIGQUIT: @@ -76,6 +139,9 @@ case SIGTERM: quit = 1; break; + case SIGUSR1: + keymgr_reload(); + break; default: break; } @@ -86,18 +152,19 @@ kore_connection_prune(KORE_CONNECTION_PRUNE_DISCONNECT); } - kore_keymgr_cleanup(); + kore_keymgr_cleanup(1); kore_platform_event_cleanup(); kore_connection_cleanup(); net_cleanup(); } void -kore_keymgr_cleanup(void) +kore_keymgr_cleanup(int final) { struct key *key, *next; - kore_log(LOG_NOTICE, "cleaning up keys"); + if (final && !kore_quiet) + kore_log(LOG_NOTICE, "cleaning up keys"); if (initialized == 0) return; @@ -112,31 +179,230 @@ } static void +keymgr_reload(void) +{ + struct kore_domain *dom; + + if (!kore_quiet) + kore_log(LOG_INFO, "(re)loading certificates, keys and CRLs"); + + kore_keymgr_cleanup(0); + TAILQ_INIT(&keys); + + kore_domain_callback(keymgr_load_privatekey); + + /* can't use kore_domain_callback() due to dst parameter. */ + TAILQ_FOREACH(dom, &domains, list) + keymgr_submit_certificates(dom, KORE_MSG_WORKER_ALL); +} + +static void +keymgr_submit_certificates(struct kore_domain *dom, u_int16_t dst) +{ + keymgr_submit_file(KORE_MSG_CERTIFICATE, dom, dom->certfile, dst, 0); + + if (dom->crlfile != NULL) + keymgr_submit_file(KORE_MSG_CRL, dom, dom->crlfile, dst, 1); +} + +static void +keymgr_submit_file(u_int8_t id, struct kore_domain *dom, + const char *file, u_int16_t dst, int can_fail) +{ + int fd; + struct stat st; + ssize_t ret; + size_t len; + struct kore_x509_msg *msg; + u_int8_t *payload; + + if ((fd = open(file, O_RDONLY)) == -1) { + if (errno == ENOENT && can_fail) + return; + fatal("open(%s): %s", file, errno_s); + } + + if (fstat(fd, &st) == -1) + fatal("stat(%s): %s", file, errno_s); + + if (!S_ISREG(st.st_mode)) + fatal("%s is not a file", file); + + if (st.st_size <= 0 || st.st_size > (1024 * 1024 * 10)) { + fatal("%s length is not valid (%jd)", file, + (intmax_t)st.st_size); + } + + len = sizeof(*msg) + st.st_size; + payload = kore_calloc(1, len); + + msg = (struct kore_x509_msg *)payload; + msg->domain_len = strlen(dom->domain); + if (msg->domain_len > sizeof(msg->domain)) + fatal("domain name '%s' too long", dom->domain); + memcpy(msg->domain, dom->domain, msg->domain_len); + + msg->data_len = st.st_size; + ret = read(fd, &msg->data[0], msg->data_len); + if (ret == -1) + fatal("failed to read from %s: %s", file, errno_s); + if (ret == 0) + fatal("eof while reading %s", file); + + if ((size_t)ret != msg->data_len) { + fatal("bad read on %s: expected %zu, got %zd", + file, msg->data_len, ret); + } + + kore_msg_send(dst, id, payload, len); + kore_free(payload); + close(fd); +} + +static void +keymgr_load_randfile(void) +{ + int fd; + struct stat st; + ssize_t ret; + size_t total; + u_int8_t buf[RAND_FILE_SIZE]; + + if (rand_file == NULL) + return; + + if ((fd = open(rand_file, O_RDONLY)) == -1) + fatal("open(%s): %s", rand_file, errno_s); + + if (fstat(fd, &st) == -1) + fatal("stat(%s): %s", rand_file, errno_s); + if (!S_ISREG(st.st_mode)) + fatal("%s is not a file", rand_file); + if (st.st_size != RAND_FILE_SIZE) + fatal("%s has an invalid size", rand_file); + + total = 0; + + while (total != RAND_FILE_SIZE) { + ret = read(fd, buf, sizeof(buf)); + if (ret == 0) + fatal("EOF on %s", rand_file); + + if (ret == -1) { + if (errno == EINTR) + continue; + fatal("read(%s): %s", rand_file, errno_s); + } + + total += (size_t)ret; + RAND_seed(buf, (int)ret); + OPENSSL_cleanse(buf, sizeof(buf)); + } + + (void)close(fd); + if (unlink(rand_file) == -1) { + kore_log(LOG_WARNING, "failed to unlink %s: %s", + rand_file, errno_s); + } +} + +static void +keymgr_save_randfile(void) +{ + int fd; + struct stat st; + ssize_t ret; + u_int8_t buf[RAND_FILE_SIZE]; + + if (rand_file == NULL) + return; + + if (stat(RAND_TMP_FILE, &st) != -1) { + kore_log(LOG_WARNING, "removing stale %s file", RAND_TMP_FILE); + (void)unlink(RAND_TMP_FILE); + } + + if (RAND_bytes(buf, sizeof(buf)) != 1) { + kore_log(LOG_WARNING, "RAND_bytes: %s", ssl_errno_s); + goto cleanup; + } + + if ((fd = open(RAND_TMP_FILE, + O_CREAT | O_TRUNC | O_WRONLY, 0400)) == -1) { + kore_log(LOG_WARNING, + "failed to open %s: %s - random data not written", + RAND_TMP_FILE, errno_s); + goto cleanup; + } + + ret = write(fd, buf, sizeof(buf)); + if (ret == -1 || (size_t)ret != sizeof(buf)) { + kore_log(LOG_WARNING, "failed to write random data"); + (void)close(fd); + (void)unlink(RAND_TMP_FILE); + goto cleanup; + } + + if (close(fd) == -1) + kore_log(LOG_WARNING, "close(%s): %s", RAND_TMP_FILE, errno_s); + + if (rename(RAND_TMP_FILE, rand_file) == -1) { + kore_log(LOG_WARNING, "rename(%s, %s): %s", + RAND_TMP_FILE, rand_file, errno_s); + (void)unlink(rand_file); + (void)unlink(RAND_TMP_FILE); + } + +cleanup: + OPENSSL_cleanse(buf, sizeof(buf)); +} + +static void keymgr_load_privatekey(struct kore_domain *dom) { FILE *fp; struct key *key; - if (dom->certkey == NULL) - return; - if ((fp = fopen(dom->certkey, "r")) == NULL) fatal("failed to open private key: %s", dom->certkey); - key = kore_malloc(sizeof(*key)); + key = kore_calloc(1, sizeof(*key)); key->dom = dom; if ((key->pkey = PEM_read_PrivateKey(fp, NULL, NULL, NULL)) == NULL) fatal("PEM_read_PrivateKey: %s", ssl_errno_s); (void)fclose(fp); - kore_free(dom->certkey); - dom->certkey = NULL; TAILQ_INSERT_TAIL(&keys, key, list); } static void +keymgr_certificate_request(struct kore_msg *msg, const void *data) +{ + struct kore_domain *dom; + + TAILQ_FOREACH(dom, &domains, list) + keymgr_submit_certificates(dom, msg->src); +} + +static void +keymgr_entropy_request(struct kore_msg *msg, const void *data) +{ + u_int8_t buf[RAND_FILE_SIZE]; + + if (RAND_bytes(buf, sizeof(buf)) != 1) { + kore_log(LOG_WARNING, + "failed to generate entropy for worker %u: %s", + msg->src, ssl_errno_s); + return; + } + + /* No cleanse, this stuff is leaked in the kernel path anyway. */ + kore_msg_send(msg->src, KORE_MSG_ENTROPY_RESP, buf, sizeof(buf)); +} + +static void keymgr_msg_recv(struct kore_msg *msg, const void *data) { const struct kore_keyreq *req; @@ -146,8 +412,11 @@ return; req = (const struct kore_keyreq *)data; + if (msg->length != (sizeof(*req) + req->data_len)) return; + if (req->domain_len > KORE_DOMAINNAME_LEN) + return; key = NULL; TAILQ_FOREACH(key, &keys, list) { @@ -174,19 +443,25 @@ keymgr_rsa_encrypt(struct kore_msg *msg, const void *data, struct key *key) { int ret; + RSA *rsa; const struct kore_keyreq *req; size_t keylen; u_int8_t buf[1024]; req = (const struct kore_keyreq *)data; - keylen = RSA_size(key->pkey->pkey.rsa); +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + rsa = EVP_PKEY_get0_RSA(key->pkey); +#else + rsa = key->pkey->pkey.rsa; +#endif + keylen = RSA_size(rsa); if (req->data_len > keylen || keylen > sizeof(buf)) return; ret = RSA_private_encrypt(req->data_len, req->data, - buf, key->pkey->pkey.rsa, req->padding); - if (ret != RSA_size(key->pkey->pkey.rsa)) + buf, rsa, req->padding); + if (ret != RSA_size(rsa)) return; kore_msg_send(msg->src, KORE_MSG_KEYMGR_RESP, buf, ret); @@ -196,18 +471,23 @@ keymgr_ecdsa_sign(struct kore_msg *msg, const void *data, struct key *key) { size_t len; + EC_KEY *ec; const struct kore_keyreq *req; unsigned int siglen; u_int8_t sig[1024]; req = (const struct kore_keyreq *)data; - - len = ECDSA_size(key->pkey->pkey.ec); +#if !defined(LIBRESSL_VERSION_TEXT) && OPENSSL_VERSION_NUMBER >= 0x10100000L + ec = EVP_PKEY_get0_EC_KEY(key->pkey); +#else + ec = key->pkey->pkey.ec; +#endif + len = ECDSA_size(ec); if (req->data_len > len || len > sizeof(sig)) return; - if (ECDSA_sign(key->pkey->save_type, req->data, req->data_len, - sig, &siglen, key->pkey->pkey.ec) == 0) + if (ECDSA_sign(EVP_PKEY_NONE, req->data, req->data_len, + sig, &siglen, ec) == 0) return; if (siglen > sizeof(sig)) diff -Nru kore-2.0.0/src/kore.c kore-3.3.1/src/kore.c --- kore-2.0.0/src/kore.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/kore.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -16,10 +16,13 @@ #include #include +#include #include #include +#include +#include #include #include #include @@ -30,72 +33,87 @@ #include "http.h" #endif -volatile sig_atomic_t sig_recv; +#if defined(KORE_USE_CURL) +#include "curl.h" +#endif + +#if defined(KORE_USE_PYTHON) +#include "python_api.h" +#endif +volatile sig_atomic_t sig_recv; struct listener_head listeners; u_int8_t nlisteners; +int kore_argc = 0; pid_t kore_pid = -1; u_int16_t cpu_count = 1; int foreground = 0; int kore_debug = 0; -u_int8_t worker_count = 0; -int skip_chroot = 0; -char *chroot_path = NULL; +int kore_quiet = 0; int skip_runas = 0; -char *runas_user = NULL; +int skip_chroot = 0; +u_int8_t worker_count = 0; +char **kore_argv = NULL; +char *kore_progname = NULL; +char *kore_root_path = NULL; +char *kore_runas_user = NULL; u_int32_t kore_socket_backlog = 5000; char *kore_pidfile = KORE_PIDFILE_DEFAULT; char *kore_tls_cipher_list = KORE_DEFAULT_CIPHER_LIST; +extern char **environ; extern char *__progname; +static size_t proctitle_maxlen = 0; static void usage(void); static void version(void); -static void kore_server_start(void); static void kore_write_kore_pid(void); +static void kore_proctitle_setup(void); static void kore_server_sslstart(void); +static void kore_server_start(int, char *[]); static void usage(void) { -#if !defined(KORE_SINGLE_BINARY) - fprintf(stderr, "Usage: kore [options | command]\n"); -#else fprintf(stderr, "Usage: %s [options]\n", __progname); -#endif + fprintf(stderr, "\n"); fprintf(stderr, "Available options:\n"); #if !defined(KORE_SINGLE_BINARY) fprintf(stderr, "\t-c\tconfiguration to use\n"); #endif #if defined(KORE_DEBUG) - fprintf(stderr, "\t-d\trun with debug on)\n"); + fprintf(stderr, "\t-d\trun with debug on\n"); #endif fprintf(stderr, "\t-f\tstart in foreground\n"); fprintf(stderr, "\t-h\tthis help text\n"); fprintf(stderr, "\t-n\tdo not chroot\n"); + fprintf(stderr, "\t-q\tonly log errors\n"); fprintf(stderr, "\t-r\tdo not drop privileges\n"); - fprintf(stderr, "\t-v\tdisplay kore build information\n"); + fprintf(stderr, "\t-v\tdisplay %s build information\n", __progname); #if !defined(KORE_SINGLE_BINARY) - kore_cli_usage(0); + fprintf(stderr, "\nFind more information on https://kore.io\n"); #else - fprintf(stderr, "\nbuilt with https://kore.io\n"); - exit(1); + fprintf(stderr, "\nBuilt using https://kore.io\n"); #endif + + exit(1); } static void version(void) { - printf("%d.%d.%d-%s ", KORE_VERSION_MAJOR, KORE_VERSION_MINOR, - KORE_VERSION_PATCH, KORE_VERSION_STATE); + printf("%s ", kore_version); #if defined(KORE_NO_TLS) printf("no-tls "); #endif #if defined(KORE_NO_HTTP) printf("no-http "); #endif +#if defined(KORE_USE_CURL) + printf("curl "); +#endif #if defined(KORE_USE_PGSQL) printf("pgsql "); #endif @@ -108,22 +126,28 @@ #if defined(KORE_SINGLE_BINARY) printf("single "); #endif +#if defined(KORE_USE_PYTHON) + printf("python "); +#endif printf("\n"); - exit(0); } int main(int argc, char *argv[]) { - int ch, flags; + struct kore_runtime_call *rcall; + int ch, flags; flags = 0; + kore_argc = argc; + kore_argv = argv; + #if !defined(KORE_SINGLE_BINARY) - while ((ch = getopt(argc, argv, "c:dfhnrv")) != -1) { + while ((ch = getopt(argc, argv, "c:dfhnqrv")) != -1) { #else - while ((ch = getopt(argc, argv, "dfhnrv")) != -1) { + while ((ch = getopt(argc, argv, "dfhnqrv")) != -1) { #endif flags++; switch (ch) { @@ -146,6 +170,9 @@ case 'n': skip_chroot = 1; break; + case 'q': + kore_quiet = 1; + break; case 'r': skip_runas = 1; break; @@ -157,17 +184,16 @@ } } + kore_mem_init(); + kore_progname = kore_strdup(argv[0]); + kore_proctitle_setup(); + argc -= optind; argv += optind; - kore_mem_init(); - #if !defined(KORE_SINGLE_BINARY) - if (argc > 0) { - if (flags) - fatal("You cannot specify kore flags and a command"); - return (kore_cli_main(argc, argv)); - } + if (argc > 0) + fatal("did you mean to run `kodev' instead?"); #endif kore_pid = getpid(); @@ -176,8 +202,13 @@ kore_log_init(); #if !defined(KORE_NO_HTTP) + http_parent_init(); +#if defined(KORE_USE_CURL) + kore_curl_sysinit(); +#endif kore_auth_init(); kore_validator_init(); + kore_filemap_init(); #endif kore_domain_init(); kore_module_init(); @@ -186,15 +217,26 @@ #if !defined(KORE_SINGLE_BINARY) if (config_file == NULL) usage(); -#else - kore_module_load(NULL, NULL); +#endif + kore_module_load(NULL, NULL, KORE_MODULE_NATIVE); + +#if defined(KORE_USE_PYTHON) + kore_python_init(); #endif kore_parse_config(); + +#if defined(KORE_SINGLE_BINARY) + rcall = kore_runtime_getcall("kore_parent_configure"); + if (rcall != NULL) { + kore_runtime_configure(rcall, argc, argv); + kore_free(rcall); + } +#endif + kore_platform_init(); #if !defined(KORE_NO_HTTP) - kore_accesslog_init(); if (http_body_disk_offload > 0) { if (mkdir(http_body_disk_path, 0700) == -1 && errno != EEXIST) { printf("can't create http_body_disk_path '%s': %s\n", @@ -204,26 +246,33 @@ } #endif - sig_recv = 0; - signal(SIGHUP, kore_signal); - signal(SIGQUIT, kore_signal); - signal(SIGTERM, kore_signal); - - if (foreground) - signal(SIGINT, kore_signal); - else - signal(SIGINT, SIG_IGN); + kore_signal_setup(); + kore_server_start(argc, argv); - kore_server_start(); + if (!kore_quiet) + kore_log(LOG_NOTICE, "server shutting down"); - kore_log(LOG_NOTICE, "server shutting down"); kore_worker_shutdown(); - if (!foreground) - unlink(kore_pidfile); + rcall = kore_runtime_getcall("kore_parent_teardown"); + if (rcall != NULL) { + kore_runtime_execute(rcall); + kore_free(rcall); + } + + if (unlink(kore_pidfile) == -1 && errno != ENOENT) + kore_log(LOG_NOTICE, "failed to remove pidfile (%s)", errno_s); kore_listener_cleanup(); - kore_log(LOG_NOTICE, "goodbye"); + + if (!kore_quiet) + kore_log(LOG_NOTICE, "goodbye"); + +#if defined(KORE_USE_PYTHON) + kore_python_cleanup(); +#endif + + kore_mem_cleanup(); return (0); } @@ -239,6 +288,13 @@ kore_debug("kore_tls_sni_cb(): received host %s", sname); if (sname != NULL && (dom = kore_domain_lookup(sname)) != NULL) { + if (dom->ssl_ctx == NULL) { + kore_log(LOG_NOTICE, + "TLS configuration for %s not complete", + dom->domain); + return (SSL_TLSEXT_ERR_NOACK); + } + kore_debug("kore_ssl_sni_cb(): Using %s CTX", sname); SSL_set_SSL_CTX(ssl, dom->ssl_ctx); @@ -263,7 +319,11 @@ if (flags & SSL_CB_HANDSHAKE_START) { if ((c = SSL_get_app_data(ssl)) == NULL) fatal("no SSL_get_app_data"); - c->tls_reneg++; + +#if defined(TLS1_3_VERSION) + if (SSL_version(ssl) != TLS1_3_VERSION) +#endif + c->tls_reneg++; } } #endif @@ -271,8 +331,8 @@ int kore_server_bind(const char *ip, const char *port, const char *ccb) { + int r; struct listener *l; - int on, r; struct addrinfo hints, *results; kore_debug("kore_server_bind(%s, %s)", ip, port); @@ -287,94 +347,243 @@ if (r != 0) fatal("getaddrinfo(%s): %s", ip, gai_strerror(r)); - l = kore_malloc(sizeof(struct listener)); - l->type = KORE_TYPE_LISTENER; - l->addrtype = results->ai_family; - - if (l->addrtype != AF_INET && l->addrtype != AF_INET6) - fatal("getaddrinfo(): unknown address family %d", l->addrtype); - - if ((l->fd = socket(results->ai_family, SOCK_STREAM, 0)) == -1) { - kore_free(l); + if ((l = kore_listener_alloc(results->ai_family, ccb)) == NULL) { freeaddrinfo(results); - kore_debug("socket(): %s", errno_s); - printf("failed to create socket: %s\n", errno_s); return (KORE_RESULT_ERROR); } - if (!kore_connection_nonblock(l->fd, 1)) { - kore_free(l); + if (bind(l->fd, results->ai_addr, results->ai_addrlen) == -1) { + kore_listener_free(l); freeaddrinfo(results); - printf("failed to make socket non blocking: %s\n", errno_s); + kore_log(LOG_ERR, "bind(): %s", errno_s); return (KORE_RESULT_ERROR); } - on = 1; - if (setsockopt(l->fd, SOL_SOCKET, - SO_REUSEADDR, (const char *)&on, sizeof(on)) == -1) { - close(l->fd); - kore_free(l); - freeaddrinfo(results); - kore_debug("setsockopt(): %s", errno_s); - printf("failed to set SO_REUSEADDR: %s\n", errno_s); + freeaddrinfo(results); + + if (listen(l->fd, kore_socket_backlog) == -1) { + kore_listener_free(l); + kore_log(LOG_ERR, "listen(): %s", errno_s); return (KORE_RESULT_ERROR); } - if (bind(l->fd, results->ai_addr, results->ai_addrlen) == -1) { - close(l->fd); - kore_free(l); - freeaddrinfo(results); - kore_debug("bind(): %s", errno_s); - printf("failed to bind to %s port %s: %s\n", ip, port, errno_s); + if (foreground && !kore_quiet) { +#if !defined(KORE_NO_TLS) + kore_log(LOG_NOTICE, "running on https://%s:%s", ip, port); +#else + kore_log(LOG_NOTICE, "running on http://%s:%s", ip, port); +#endif + } + + return (KORE_RESULT_OK); +} + +int +kore_server_bind_unix(const char *path, const char *ccb) +{ + struct listener *l; + int len; + struct sockaddr_un sun; + socklen_t socklen; + + memset(&sun, 0, sizeof(sun)); + sun.sun_family = AF_UNIX; + + len = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s", path); + if (len == -1 || (size_t)len >= sizeof(sun.sun_path)) { + kore_log(LOG_ERR, "unix socket path '%s' too long", path); return (KORE_RESULT_ERROR); } - freeaddrinfo(results); +#if defined(__linux__) + if (sun.sun_path[0] == '@') + sun.sun_path[0] = '\0'; + socklen = sizeof(sun.sun_family) + len; +#else + socklen = sizeof(sun); +#endif + + if ((l = kore_listener_alloc(AF_UNIX, ccb)) == NULL) + return (KORE_RESULT_ERROR); + + if (bind(l->fd, (struct sockaddr *)&sun, socklen) == -1) { + kore_log(LOG_ERR, "bind: %s", errno_s); + kore_listener_free(l); + return (KORE_RESULT_ERROR); + } if (listen(l->fd, kore_socket_backlog) == -1) { - close(l->fd); - kore_free(l); - kore_debug("listen(): %s", errno_s); - printf("failed to listen on socket: %s\n", errno_s); + kore_log(LOG_ERR, "listen(): %s", errno_s); + kore_listener_free(l); return (KORE_RESULT_ERROR); } + if (foreground && !kore_quiet) + kore_log(LOG_NOTICE, "running on %s", path); + + return (KORE_RESULT_OK); +} + +struct listener * +kore_listener_alloc(int family, const char *ccb) +{ + struct listener *l; + + switch (family) { + case AF_INET: + case AF_INET6: + case AF_UNIX: + break; + default: + fatal("unknown address family %d", family); + } + + l = kore_calloc(1, sizeof(struct listener)); + + nlisteners++; + LIST_INSERT_HEAD(&listeners, l, list); + + l->fd = -1; + l->family = family; + + l->evt.type = KORE_TYPE_LISTENER; + l->evt.handle = kore_listener_accept; + + if ((l->fd = socket(family, SOCK_STREAM, 0)) == -1) { + kore_listener_free(l); + kore_log(LOG_ERR, "socket(): %s", errno_s); + return (NULL); + } + + if (fcntl(l->fd, F_SETFD, FD_CLOEXEC) == -1) { + kore_listener_free(l); + kore_log(LOG_ERR, "fcntl(): %s", errno_s); + return (NULL); + } + + if (!kore_connection_nonblock(l->fd, family != AF_UNIX)) { + kore_listener_free(l); + kore_log(LOG_ERR, "kore_connection_nonblock(): %s", errno_s); + return (NULL); + } + + if (!kore_sockopt(l->fd, SOL_SOCKET, SO_REUSEADDR)) { + kore_listener_free(l); + return (NULL); + } + if (ccb != NULL) { - *(void **)&(l->connect) = kore_module_getsym(ccb); - if (l->connect == NULL) { - printf("no such callback: '%s'\n", ccb); - close(l->fd); - kore_free(l); - return (KORE_RESULT_ERROR); + if ((l->connect = kore_runtime_getcall(ccb)) == NULL) { + kore_log(LOG_ERR, "no such callback: '%s'", ccb); + kore_listener_free(l); + return (NULL); } } else { l->connect = NULL; } - nlisteners++; - LIST_INSERT_HEAD(&listeners, l, list); + return (l); +} - if (foreground) { -#if !defined(KORE_NO_TLS) - kore_log(LOG_NOTICE, "running on https://%s:%s", ip, port); -#else - kore_log(LOG_NOTICE, "running on http://%s:%s", ip, port); -#endif +void +kore_listener_free(struct listener *l) +{ + LIST_REMOVE(l, list); + + if (l->fd != -1) + close(l->fd); + + kore_free(l); +} + +void +kore_listener_accept(void *arg, int error) +{ + struct connection *c; + struct listener *l = arg; + u_int32_t accepted; + + if (error) + fatal("error on listening socket"); + + if (!(l->evt.flags & KORE_EVENT_READ)) + return; + + accepted = 0; + + while (worker_active_connections < worker_max_connections) { + if (worker_accept_threshold != 0 && + accepted >= worker_accept_threshold) { + kore_worker_make_busy(); + break; + } + + if (!kore_connection_accept(l, &c)) + break; + + if (c == NULL) + break; + + accepted++; + kore_platform_event_all(c->fd, c); + } +} + +int +kore_sockopt(int fd, int what, int opt) +{ + int on; + + on = 1; + if (setsockopt(fd, what, opt, (const char *)&on, sizeof(on)) == -1) { + kore_log(LOG_ERR, "setsockopt(): %s", errno_s); + return (KORE_RESULT_ERROR); } return (KORE_RESULT_OK); } void +kore_signal_setup(void) +{ + struct sigaction sa; + + sig_recv = 0; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = kore_signal; + + if (sigfillset(&sa.sa_mask) == -1) + fatal("sigfillset: %s", errno_s); + + if (sigaction(SIGHUP, &sa, NULL) == -1) + fatal("sigaction: %s", errno_s); + if (sigaction(SIGQUIT, &sa, NULL) == -1) + fatal("sigaction: %s", errno_s); + if (sigaction(SIGTERM, &sa, NULL) == -1) + fatal("sigaction: %s", errno_s); + if (sigaction(SIGUSR1, &sa, NULL) == -1) + fatal("sigaction: %s", errno_s); + if (sigaction(SIGCHLD, &sa, NULL) == -1) + fatal("sigaction: %s", errno_s); + + if (foreground) { + if (sigaction(SIGINT, &sa, NULL) == -1) + fatal("sigaction: %s", errno_s); + } else { + (void)signal(SIGINT, SIG_IGN); + } + + (void)signal(SIGPIPE, SIG_IGN); +} + +void kore_listener_cleanup(void) { struct listener *l; while (!LIST_EMPTY(&listeners)) { l = LIST_FIRST(&listeners); - LIST_REMOVE(l, list); - close(l->fd); - kore_free(l); + kore_listener_free(l); } } @@ -384,6 +593,50 @@ sig_recv = sig; } +void +kore_shutdown(void) +{ + if (worker != NULL) { + kore_msg_send(KORE_MSG_PARENT, KORE_MSG_SHUTDOWN, NULL, 0); + return; + } + + fatal("kore_shutdown: called from parent"); +} + +void +kore_proctitle(const char *title) +{ + int len; + + kore_argv[1] = NULL; + + len = snprintf(kore_argv[0], proctitle_maxlen, "%s %s", + basename(kore_progname), title); + if (len == -1 || (size_t)len >= proctitle_maxlen) + fatal("proctitle '%s' too large", title); + + memset(kore_argv[0] + len, 0, proctitle_maxlen - len); +} + +static void +kore_proctitle_setup(void) +{ + int i; + char *p; + + proctitle_maxlen = 0; + + for (i = 0; environ[i] != NULL; i++) { + p = kore_strdup(environ[i]); + proctitle_maxlen += strlen(environ[i]) + 1; + environ[i] = p; + } + + for (i = 0; kore_argv[i] != NULL; i++) + proctitle_maxlen += strlen(kore_argv[i]) + 1; +} + static void kore_server_sslstart(void) { @@ -396,30 +649,53 @@ } static void -kore_server_start(void) +kore_server_start(int argc, char *argv[]) { - u_int32_t tmp; - int quit; - - if (foreground == 0 && daemon(1, 1) == -1) - fatal("cannot daemon(): %s", errno_s); + u_int32_t tmp; + struct kore_runtime_call *rcall; + u_int64_t netwait; + int quit, last_sig; + + if (foreground == 0) { + if (daemon(1, 0) == -1) + fatal("cannot daemon(): %s", errno_s); +#if defined(KORE_SINGLE_BINARY) + rcall = kore_runtime_getcall("kore_parent_daemonized"); + if (rcall != NULL) { + kore_runtime_execute(rcall); + kore_free(rcall); + } +#endif + } kore_pid = getpid(); - if (!foreground) - kore_write_kore_pid(); + kore_write_kore_pid(); - kore_log(LOG_NOTICE, "%s is starting up", __progname); + if (!kore_quiet) { + kore_log(LOG_NOTICE, "%s is starting up", __progname); #if defined(KORE_USE_PGSQL) - kore_log(LOG_NOTICE, "pgsql built-in enabled"); + kore_log(LOG_NOTICE, "pgsql built-in enabled"); #endif #if defined(KORE_USE_TASKS) - kore_log(LOG_NOTICE, "tasks built-in enabled"); + kore_log(LOG_NOTICE, "tasks built-in enabled"); #endif #if defined(KORE_USE_JSONRPC) - kore_log(LOG_NOTICE, "jsonrpc built-in enabled"); + kore_log(LOG_NOTICE, "jsonrpc built-in enabled"); +#endif +#if defined(KORE_USE_PYTHON) + kore_log(LOG_NOTICE, "python built-in enabled"); #endif + } - kore_platform_proctitle("kore [parent]"); +#if !defined(KORE_SINGLE_BINARY) + rcall = kore_runtime_getcall("kore_parent_configure"); + if (rcall != NULL) { + kore_runtime_configure(rcall, argc, argv); + kore_free(rcall); + } +#endif + + kore_platform_proctitle("[parent]"); kore_msg_init(); kore_worker_init(); @@ -434,14 +710,20 @@ quit = 0; worker_max_connections = tmp; + + kore_timer_init(); +#if !defined(KORE_NO_HTTP) + kore_timer_add(kore_accesslog_run, 100, NULL, 0); +#endif + while (quit != 1) { if (sig_recv != 0) { + last_sig = sig_recv; + switch (sig_recv) { case SIGHUP: -#if !defined(KORE_SINGLE_BINARY) kore_worker_dispatch_signal(sig_recv); kore_module_reload(0); -#endif break; case SIGINT: case SIGQUIT: @@ -449,20 +731,32 @@ quit = 1; kore_worker_dispatch_signal(sig_recv); continue; + case SIGUSR1: + kore_worker_dispatch_signal(sig_recv); + break; + case SIGCHLD: + kore_worker_reap(); + break; default: - kore_log(LOG_NOTICE, - "no action taken for signal %d", sig_recv); break; } - sig_recv = 0; + if (sig_recv == last_sig) + sig_recv = 0; + else + continue; } - kore_worker_wait(0); - kore_platform_event_wait(100); + netwait = kore_timer_next_run(kore_time_ms()); + kore_platform_event_wait(netwait); kore_connection_prune(KORE_CONNECTION_PRUNE_DISCONNECT); + kore_timer_run(kore_time_ms()); } +#if !defined(KORE_NO_HTTP) + kore_accesslog_gather(NULL, kore_time_ms(), 1); +#endif + kore_platform_event_cleanup(); kore_connection_cleanup(); kore_domain_cleanup(); diff -Nru kore-2.0.0/src/linux.c kore-3.3.1/src/linux.c --- kore-2.0.0/src/linux.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/linux.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,8 +14,10 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include +#include #include @@ -64,6 +66,11 @@ void kore_platform_event_init(void) { + if (efd != -1) + close(efd); + if (events != NULL) + kore_free(events); + if ((efd = epoll_create(10000)) == -1) fatal("epoll_create(): %s", errno_s); @@ -85,19 +92,22 @@ } } -int +void kore_platform_event_wait(u_int64_t timer) { u_int32_t r; - struct connection *c; - struct listener *l; - u_int8_t type; - int n, i; + struct kore_event *evt; + int n, i, timeo; - n = epoll_wait(efd, events, event_count, timer); + if (timer == KORE_WAIT_INFINITE) + timeo = -1; + else + timeo = timer; + + n = epoll_wait(efd, events, event_count, timeo); if (n == -1) { if (errno == EINTR) - return (0); + return; fatal("epoll_wait(): %s", errno_s); } @@ -110,83 +120,22 @@ if (events[i].data.ptr == NULL) fatal("events[%d].data.ptr == NULL", i); - type = *(u_int8_t *)events[i].data.ptr; + r = 0; + evt = (struct kore_event *)events[i].data.ptr; - if (events[i].events & EPOLLERR || - events[i].events & EPOLLHUP) { - switch (type) { - case KORE_TYPE_LISTENER: - fatal("failed on listener socket"); - /* NOTREACHED */ -#if defined(KORE_USE_PGSQL) - case KORE_TYPE_PGSQL_CONN: - kore_pgsql_handle(events[i].data.ptr, 1); - break; -#endif -#if defined(KORE_USE_TASKS) - case KORE_TYPE_TASK: - kore_task_handle(events[i].data.ptr, 1); - break; -#endif - default: - c = (struct connection *)events[i].data.ptr; - kore_connection_disconnect(c); - break; - } + if (events[i].events & EPOLLIN) + evt->flags |= KORE_EVENT_READ; - continue; - } + if (events[i].events & EPOLLOUT) + evt->flags |= KORE_EVENT_WRITE; - switch (type) { - case KORE_TYPE_LISTENER: - l = (struct listener *)events[i].data.ptr; - - while (worker_active_connections < - worker_max_connections) { - if (worker_accept_threshold != 0 && - r >= worker_accept_threshold) - break; - - if (!kore_connection_accept(l, &c)) { - r = 1; - break; - } - - if (c == NULL) - break; - - r++; - kore_platform_event_all(c->fd, c); - } - break; - case KORE_TYPE_CONNECTION: - c = (struct connection *)events[i].data.ptr; - if (events[i].events & EPOLLIN && - !(c->flags & CONN_READ_BLOCK)) - c->flags |= CONN_READ_POSSIBLE; - if (events[i].events & EPOLLOUT && - !(c->flags & CONN_WRITE_BLOCK)) - c->flags |= CONN_WRITE_POSSIBLE; - - if (c->handle != NULL && !c->handle(c)) - kore_connection_disconnect(c); - break; -#if defined(KORE_USE_PGSQL) - case KORE_TYPE_PGSQL_CONN: - kore_pgsql_handle(events[i].data.ptr, 0); - break; -#endif -#if defined(KORE_USE_TASKS) - case KORE_TYPE_TASK: - kore_task_handle(events[i].data.ptr, 0); - break; -#endif - default: - fatal("wrong type in event %d", type); - } - } + if (events[i].events & EPOLLERR || + events[i].events & EPOLLHUP || + events[i].events & EPOLLRDHUP) + r = 1; - return (r); + evt->handle(events[i].data.ptr, r); + } } void @@ -219,13 +168,13 @@ void kore_platform_schedule_read(int fd, void *data) { - kore_platform_event_schedule(fd, EPOLLIN, 0, data); + kore_platform_event_schedule(fd, EPOLLIN | EPOLLET, 0, data); } void kore_platform_schedule_write(int fd, void *data) { - kore_platform_event_schedule(fd, EPOLLOUT, 0, data); + kore_platform_event_schedule(fd, EPOLLOUT | EPOLLET, 0, data); } void @@ -262,7 +211,40 @@ void kore_platform_proctitle(char *title) { - if (prctl(PR_SET_NAME, title) == -1) { - kore_debug("prctl(): %s", errno_s); + kore_proctitle(title); +} + +#if defined(KORE_USE_PLATFORM_SENDFILE) +int +kore_platform_sendfile(struct connection *c, struct netbuf *nb) +{ + off_t smin; + ssize_t sent; + size_t len, prevoff; + + prevoff = nb->fd_off; + smin = nb->fd_len - nb->fd_off; + len = MIN(SENDFILE_PAYLOAD_MAX, smin); + +resend: + sent = sendfile(c->fd, nb->file_ref->fd, &nb->fd_off, len); + if (sent == -1) { + if (errno == EAGAIN) { + c->evt.flags &= ~KORE_EVENT_WRITE; + return (KORE_RESULT_OK); + } + + return (KORE_RESULT_ERROR); + } + + if (nb->fd_off - prevoff != (size_t)len) + goto resend; + + if (sent == 0 || nb->fd_off == nb->fd_len) { + net_remove_netbuf(c, nb); + c->snb = NULL; } + + return (KORE_RESULT_OK); } +#endif diff -Nru kore-2.0.0/src/mem.c kore-3.3.1/src/mem.c --- kore-2.0.0/src/mem.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/mem.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -32,7 +33,10 @@ #define KORE_MEMINFO(x) \ (struct meminfo *)((u_int8_t *)x + KORE_MEMSIZE(x)) +#define KORE_MEM_TAGGED 0x0001 + struct meminfo { + u_int16_t flags; u_int16_t magic; }; @@ -40,8 +44,16 @@ struct kore_pool pool; }; +struct tag { + void *ptr; + u_int32_t id; + TAILQ_ENTRY(tag) list; +}; + static size_t memblock_index(size_t); +static TAILQ_HEAD(, tag) tags; +static struct kore_pool tag_pool; static struct memblock blocks[KORE_MEM_BLOCKS]; void @@ -52,6 +64,8 @@ u_int32_t size, elm, mlen; size = 8; + TAILQ_INIT(&tags); + kore_pool_init(&tag_pool, "tag_pool", sizeof(struct tag), 100); for (i = 0; i < KORE_MEM_BLOCKS; i++) { len = snprintf(name, sizeof(name), "block-%u", size); @@ -69,6 +83,16 @@ } } +void +kore_mem_cleanup(void) +{ + int i; + + for (i = 0; i < KORE_MEM_BLOCKS; i++) { + kore_pool_cleanup(&blocks[i].pool); + } +} + void * kore_malloc(size_t len) { @@ -78,7 +102,7 @@ size_t mlen, idx, *plen; if (len == 0) - fatal("kore_malloc(): zero size"); + len = 8; if (len <= KORE_MEM_BLOCK_SIZE_MAX) { idx = memblock_index(len); @@ -86,7 +110,7 @@ } else { mlen = sizeof(size_t) + len + sizeof(struct meminfo); if ((ptr = calloc(1, mlen)) == NULL) - fatal("kore_malloc(%zd): %d", len, errno); + fatal("kore_malloc(%zu): %d", len, errno); } plen = (size_t *)ptr; @@ -94,6 +118,7 @@ addr = (u_int8_t *)ptr + sizeof(size_t); mem = KORE_MEMINFO(addr); + mem->flags = 0; mem->magic = KORE_MEM_MAGIC; return (addr); @@ -105,12 +130,11 @@ struct meminfo *mem; void *nptr; - if (len == 0) - fatal("kore_realloc(): zero size"); - if (ptr == NULL) { nptr = kore_malloc(len); } else { + if (len == KORE_MEMSIZE(ptr)) + return (ptr); mem = KORE_MEMINFO(ptr); if (mem->magic != KORE_MEM_MAGIC) fatal("kore_realloc(): magic boundary not found"); @@ -126,12 +150,17 @@ void * kore_calloc(size_t memb, size_t len) { - if (memb == 0 || len == 0) - fatal("kore_calloc(): zero size"); + void *ptr; + size_t total; + if (SIZE_MAX / memb < len) fatal("kore_calloc(): memb * len > SIZE_MAX"); - return (kore_malloc(memb * len)); + total = memb * len; + ptr = kore_malloc(total); + memset(ptr, 0, total); + + return (ptr); } void @@ -148,6 +177,11 @@ if (mem->magic != KORE_MEM_MAGIC) fatal("kore_free(): magic boundary not found"); + if (mem->flags & KORE_MEM_TAGGED) { + kore_mem_untag(ptr); + mem->flags &= ~KORE_MEM_TAGGED; + } + len = KORE_MEMSIZE(ptr); addr = (u_int8_t *)ptr - sizeof(size_t); @@ -172,6 +206,65 @@ return (nstr); } +void * +kore_malloc_tagged(size_t len, u_int32_t tag) +{ + void *ptr; + + ptr = kore_malloc(len); + kore_mem_tag(ptr, tag); + + return (ptr); +} + +void +kore_mem_tag(void *ptr, u_int32_t id) +{ + struct tag *tag; + struct meminfo *mem; + + if (kore_mem_lookup(id) != NULL) + fatal("kore_mem_tag: tag %u taken", id); + + mem = KORE_MEMINFO(ptr); + if (mem->magic != KORE_MEM_MAGIC) + fatal("kore_mem_tag: magic boundary not found"); + mem->flags |= KORE_MEM_TAGGED; + + tag = kore_pool_get(&tag_pool); + tag->id = id; + tag->ptr = ptr; + + TAILQ_INSERT_TAIL(&tags, tag, list); +} + +void +kore_mem_untag(void *ptr) +{ + struct tag *tag; + + TAILQ_FOREACH(tag, &tags, list) { + if (tag->ptr == ptr) { + TAILQ_REMOVE(&tags, tag, list); + kore_pool_put(&tag_pool, tag); + break; + } + } +} + +void * +kore_mem_lookup(u_int32_t id) +{ + struct tag *tag; + + TAILQ_FOREACH(tag, &tags, list) { + if (tag->id == id) + return (tag->ptr); + } + + return (NULL); +} + static size_t memblock_index(size_t len) { diff -Nru kore-2.0.0/src/module.c kore-3.3.1/src/module.c --- kore-2.0.0/src/module.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/module.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,14 +14,35 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include #include "kore.h" +#if !defined(KORE_NO_HTTP) +#include "http.h" +#endif + +#if defined(KORE_USE_PYTHON) +#include "python_api.h" +#endif + static TAILQ_HEAD(, kore_module) modules; +static void native_free(struct kore_module *); +static void native_load(struct kore_module *); +static void native_reload(struct kore_module *); +static void *native_getsym(struct kore_module *, const char *); + +struct kore_module_functions kore_native_module = { + .free = native_free, + .load = native_load, + .getsym = native_getsym, + .reload = native_reload, +}; + void kore_module_init(void) { @@ -36,77 +57,96 @@ for (module = TAILQ_FIRST(&modules); module != NULL; module = next) { next = TAILQ_NEXT(module, list); TAILQ_REMOVE(&modules, module, list); - - kore_free(module->path); - (void)dlclose(module->handle); - kore_free(module); + module->fun->free(module); } } -void -kore_module_load(const char *path, const char *onload) +struct kore_module * +kore_module_load(const char *path, const char *onload, int type) { -#if !defined(KORE_SINGLE_BINARY) struct stat st; -#endif struct kore_module *module; kore_debug("kore_module_load(%s, %s)", path, onload); module = kore_malloc(sizeof(struct kore_module)); - module->onload = NULL; module->ocb = NULL; + module->type = type; + module->onload = NULL; + module->handle = NULL; + + if (path != NULL) { + if (stat(path, &st) == -1) + fatal("stat(%s): %s", path, errno_s); + + module->path = kore_strdup(path); + module->mtime = st.st_mtime; + } else { + module->path = NULL; + module->mtime = 0; + } -#if !defined(KORE_SINGLE_BINARY) - if (stat(path, &st) == -1) - fatal("stat(%s): %s", path, errno_s); - - module->path = kore_strdup(path); - module->mtime = st.st_mtime; -#else - module->path = NULL; - module->mtime = 0; + switch (module->type) { + case KORE_MODULE_NATIVE: + module->fun = &kore_native_module; + module->runtime = &kore_native_runtime; + break; +#if defined(KORE_USE_PYTHON) + case KORE_MODULE_PYTHON: + module->fun = &kore_python_module; + module->runtime = &kore_python_runtime; + break; #endif + default: + fatal("kore_module_load: unknown type %d", type); + } - module->handle = dlopen(module->path, RTLD_NOW | RTLD_GLOBAL); - if (module->handle == NULL) - fatal("%s: %s", path, dlerror()); + module->fun->load(module); + TAILQ_INSERT_TAIL(&modules, module, list); if (onload != NULL) { module->onload = kore_strdup(onload); - *(void **)&(module->ocb) = dlsym(module->handle, onload); - if (module->ocb == NULL) - fatal("%s: onload '%s' not present", path, onload); + module->ocb = kore_malloc(sizeof(*module->ocb)); + module->ocb->runtime = module->runtime; + module->ocb->addr = module->fun->getsym(module, onload); + + if (module->ocb->addr == NULL) { + fatal("%s: onload '%s' not present", + module->path, onload); + } } - TAILQ_INSERT_TAIL(&modules, module, list); + return (module); } void kore_module_onload(void) { -#if !defined(KORE_SINGLE_BINARY) struct kore_module *module; TAILQ_FOREACH(module, &modules, list) { - if (module->ocb == NULL) + if (module->path == NULL || module->ocb == NULL) continue; - (void)module->ocb(KORE_MODULE_LOAD); + kore_runtime_onload(module->ocb, KORE_MODULE_LOAD); } -#endif } void kore_module_reload(int cbs) { -#if !defined(KORE_SINGLE_BINARY) struct stat st; + int ret; +#if !defined(KORE_NO_HTTP) struct kore_domain *dom; struct kore_module_handle *hdlr; +#endif struct kore_module *module; TAILQ_FOREACH(module, &modules, list) { + if (module->path == NULL) + continue; + if (stat(module->path, &st) == -1) { kore_log(LOG_NOTICE, "stat(%s): %s, skipping reload", module->path, errno_s); @@ -117,49 +157,51 @@ continue; if (module->ocb != NULL && cbs == 1) { - if (!module->ocb(KORE_MODULE_UNLOAD)) { + ret = kore_runtime_onload(module->ocb, + KORE_MODULE_UNLOAD); + if (ret == KORE_RESULT_ERROR) { kore_log(LOG_NOTICE, - "not reloading %s", module->path); + "%s forced no reloaded", module->path); continue; } } module->mtime = st.st_mtime; - if (dlclose(module->handle)) - fatal("cannot close existing module: %s", dlerror()); - - module->handle = dlopen(module->path, RTLD_NOW | RTLD_GLOBAL); - if (module->handle == NULL) - fatal("kore_module_reload(): %s", dlerror()); + module->fun->reload(module); if (module->onload != NULL) { - *(void **)&(module->ocb) = - dlsym(module->handle, module->onload); - if (module->ocb == NULL) { + kore_free(module->ocb); + module->ocb = kore_malloc(sizeof(*module->ocb)); + module->ocb->runtime = module->runtime; + module->ocb->addr = + module->fun->getsym(module, module->onload); + if (module->ocb->addr == NULL) { fatal("%s: onload '%s' not present", module->path, module->onload); } - - if (cbs) - (void)module->ocb(KORE_MODULE_LOAD); } + if (module->ocb != NULL && cbs == 1) + kore_runtime_onload(module->ocb, KORE_MODULE_LOAD); + kore_log(LOG_NOTICE, "reloaded '%s' module", module->path); } +#if !defined(KORE_NO_HTTP) TAILQ_FOREACH(dom, &domains, list) { TAILQ_FOREACH(hdlr, &(dom->handlers), list) { - hdlr->addr = kore_module_getsym(hdlr->func); - if (hdlr->func == NULL) + kore_free(hdlr->rcall); + hdlr->rcall = kore_runtime_getcall(hdlr->func); + if (hdlr->rcall == NULL) fatal("no function '%s' found", hdlr->func); hdlr->errors = 0; } } +#endif #if !defined(KORE_NO_HTTP) kore_validator_reload(); #endif -#endif } int @@ -177,19 +219,12 @@ const char *func, const char *auth, int type) { struct kore_auth *ap; - void *addr; struct kore_domain *dom; struct kore_module_handle *hdlr; kore_debug("kore_module_handler_new(%s, %s, %s, %s, %d)", path, domain, func, auth, type); - addr = kore_module_getsym(func); - if (addr == NULL) { - kore_debug("function '%s' not found", func); - return (KORE_RESULT_ERROR); - } - if ((dom = kore_domain_lookup(domain)) == NULL) return (KORE_RESULT_ERROR); @@ -204,11 +239,18 @@ hdlr->auth = ap; hdlr->dom = dom; hdlr->errors = 0; - hdlr->addr = addr; hdlr->type = type; - TAILQ_INIT(&(hdlr->params)); hdlr->path = kore_strdup(path); hdlr->func = kore_strdup(func); + hdlr->methods = HTTP_METHOD_ALL; + + TAILQ_INIT(&(hdlr->params)); + + if ((hdlr->rcall = kore_runtime_getcall(func)) == NULL) { + kore_module_handler_free(hdlr); + kore_log(LOG_ERR, "function '%s' not found", func); + return (KORE_RESULT_ERROR); + } if (hdlr->type == HANDLER_TYPE_DYNAMIC) { if (regcomp(&(hdlr->rctx), hdlr->path, @@ -270,20 +312,55 @@ return (NULL); } - #endif /* !KORE_NO_HTTP */ void * -kore_module_getsym(const char *symbol) +kore_module_getsym(const char *symbol, struct kore_runtime **runtime) { void *ptr; struct kore_module *module; + if (runtime != NULL) + *runtime = NULL; + TAILQ_FOREACH(module, &modules, list) { - ptr = dlsym(module->handle, symbol); - if (ptr != NULL) + ptr = module->fun->getsym(module, symbol); + if (ptr != NULL) { + if (runtime != NULL) + *runtime = module->runtime; return (ptr); + } } return (NULL); } + +static void * +native_getsym(struct kore_module *module, const char *symbol) +{ + return (dlsym(module->handle, symbol)); +} + +static void +native_free(struct kore_module *module) +{ + kore_free(module->path); + (void)dlclose(module->handle); + kore_free(module); +} + +static void +native_reload(struct kore_module *module) +{ + if (dlclose(module->handle)) + fatal("cannot close existing module: %s", dlerror()); + module->fun->load(module); +} + +static void +native_load(struct kore_module *module) +{ + module->handle = dlopen(module->path, RTLD_NOW | RTLD_GLOBAL); + if (module->handle == NULL) + fatal("%s: %s", module->path, dlerror()); +} diff -Nru kore-2.0.0/src/msg.c kore-3.3.1/src/msg.c --- kore-2.0.0/src/msg.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/msg.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Joris Vink + * Copyright (c) 2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include @@ -34,9 +35,9 @@ static int msg_recv_data(struct netbuf *); static void msg_disconnected_parent(struct connection *); static void msg_disconnected_worker(struct connection *); +static void msg_type_shutdown(struct kore_msg *, const void *); #if !defined(KORE_NO_HTTP) -static void msg_type_accesslog(struct kore_msg *, const void *); static void msg_type_websocket(struct kore_msg *, const void *); #endif @@ -57,9 +58,7 @@ kore_msg_parent_add(kw); } -#if !defined(KORE_NO_HTTP) - kore_msg_register(KORE_MSG_ACCESSLOG, msg_type_accesslog); -#endif + kore_msg_register(KORE_MSG_SHUTDOWN, msg_type_shutdown); } void @@ -104,6 +103,7 @@ worker->msg[1]->state = CONN_STATE_ESTABLISHED; worker->msg[1]->disconnect = msg_disconnected_parent; worker->msg[1]->handle = kore_connection_handle; + worker->msg[1]->evt.flags = KORE_EVENT_WRITE; TAILQ_INSERT_TAIL(&connections, worker->msg[1], list); kore_platform_event_all(worker->msg[1]->fd, worker->msg[1]); @@ -129,7 +129,7 @@ } void -kore_msg_send(u_int16_t dst, u_int8_t id, const void *data, u_int32_t len) +kore_msg_send(u_int16_t dst, u_int8_t id, const void *data, size_t len) { struct kore_msg m; @@ -139,7 +139,9 @@ m.src = worker->id; net_send_queue(worker->msg[1], &m, sizeof(m)); - net_send_queue(worker->msg[1], data, len); + if (data != NULL && len > 0) + net_send_queue(worker->msg[1], data, len); + net_send_flush(worker->msg[1]); } @@ -148,8 +150,12 @@ { struct kore_msg *msg = (struct kore_msg *)nb->buf; - net_recv_expand(nb->owner, msg->length, msg_recv_data); - return (KORE_RESULT_OK); + if (msg->length > 0) { + net_recv_expand(nb->owner, msg->length, msg_recv_data); + return (KORE_RESULT_OK); + } + + return (msg_recv_data(nb)); } static int @@ -166,14 +172,15 @@ if (worker != NULL && msg->dst != worker->id) fatal("received message for incorrect worker"); - type->cb(msg, nb->buf + sizeof(*msg)); + if (msg->length > 0) + type->cb(msg, nb->buf + sizeof(*msg)); + else + type->cb(msg, NULL); } if (worker == NULL && type == NULL) { destination = msg->dst; TAILQ_FOREACH(c, &connections, list) { - if (c == nb->owner) - continue; if (c->proto != CONN_PROTO_MSG || c->hdlr_extra == NULL) continue; @@ -196,7 +203,9 @@ static void msg_disconnected_parent(struct connection *c) { - kore_log(LOG_ERR, "parent gone, shutting down"); + if (!kore_quiet) + kore_log(LOG_ERR, "parent gone, shutting down"); + if (kill(worker->pid, SIGQUIT) == -1) kore_log(LOG_ERR, "failed to send SIGQUIT: %s", errno_s); } @@ -207,14 +216,16 @@ c->hdlr_extra = NULL; } -#if !defined(KORE_NO_HTTP) static void -msg_type_accesslog(struct kore_msg *msg, const void *data) +msg_type_shutdown(struct kore_msg *msg, const void *data) { - if (kore_accesslog_write(data, msg->length) == -1) - kore_log(LOG_WARNING, "failed to write to accesslog"); + kore_log(LOG_NOTICE, + "shutdown requested by worker %u, going down", msg->src); + + (void)raise(SIGQUIT); } +#if !defined(KORE_NO_HTTP) static void msg_type_websocket(struct kore_msg *msg, const void *data) { diff -Nru kore-2.0.0/src/net.c kore-3.3.1/src/net.c --- kore-2.0.0/src/net.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/net.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #if defined(__linux__) #include @@ -33,7 +34,11 @@ void net_init(void) { - kore_pool_init(&nb_pool, "nb_pool", sizeof(struct netbuf), 1000); + u_int32_t elm; + + /* Add some overhead so we don't roll over for internal items. */ + elm = worker_max_connections + 10; + kore_pool_init(&nb_pool, "nb_pool", sizeof(struct netbuf), elm); } void @@ -43,6 +48,33 @@ kore_pool_cleanup(&nb_pool); } +struct netbuf * +net_netbuf_get(void) +{ + struct netbuf *nb; + + nb = kore_pool_get(&nb_pool); + + nb->cb = NULL; + nb->buf = NULL; + nb->owner = NULL; + nb->extra = NULL; + nb->file_ref = NULL; + + nb->type = 0; + nb->s_off = 0; + nb->b_len = 0; + nb->m_len = 0; + nb->flags = 0; + +#if defined(KORE_USE_PLATFORM_SENDFILE) + nb->fd_off = -1; + nb->fd_len = -1; +#endif + + return (nb); +} + void net_send_queue(struct connection *c, const void *data, size_t len) { @@ -72,11 +104,9 @@ } } - nb = kore_pool_get(&nb_pool); - nb->flags = 0; - nb->cb = NULL; + nb = net_netbuf_get(); + nb->owner = c; - nb->s_off = 0; nb->b_len = len; nb->type = NETBUF_SEND; @@ -86,8 +116,7 @@ nb->m_len = nb->b_len; nb->buf = kore_malloc(nb->m_len); - if (len > 0) - memcpy(nb->buf, d, nb->b_len); + memcpy(nb->buf, d, nb->b_len); TAILQ_INSERT_TAIL(&(c->send_queue), nb, list); } @@ -100,10 +129,9 @@ kore_debug("net_send_stream(%p, %p, %zu)", c, data, len); - nb = kore_pool_get(&nb_pool); + nb = net_netbuf_get(); nb->cb = cb; nb->owner = c; - nb->s_off = 0; nb->buf = data; nb->b_len = len; nb->m_len = nb->b_len; @@ -111,23 +139,45 @@ nb->flags = NETBUF_IS_STREAM; TAILQ_INSERT_TAIL(&(c->send_queue), nb, list); + if (out != NULL) *out = nb; } void +net_send_fileref(struct connection *c, struct kore_fileref *ref) +{ + struct netbuf *nb; + + nb = net_netbuf_get(); + nb->owner = c; + nb->file_ref = ref; + nb->type = NETBUF_SEND; + nb->flags = NETBUF_IS_FILEREF; + +#if defined(KORE_USE_PLATFORM_SENDFILE) + nb->fd_off = 0; + nb->fd_len = ref->size; +#else + nb->buf = ref->base; + nb->b_len = ref->size; + nb->m_len = nb->b_len; + nb->flags |= NETBUF_IS_STREAM; +#endif + + TAILQ_INSERT_TAIL(&(c->send_queue), nb, list); +} + +void net_recv_reset(struct connection *c, size_t len, int (*cb)(struct netbuf *)) { kore_debug("net_recv_reset(): %p %zu", c, len); - if (c->rnb->type != NETBUF_RECV) - fatal("net_recv_reset(): wrong netbuf type"); - c->rnb->cb = cb; c->rnb->s_off = 0; c->rnb->b_len = len; - if (c->rnb->b_len <= c->rnb->m_len && + if (c->rnb->buf != NULL && c->rnb->b_len <= c->rnb->m_len && c->rnb->m_len < (NETBUF_SEND_PAYLOAD_MAX / 2)) return; @@ -143,15 +193,13 @@ kore_debug("net_recv_queue(): %p %zu %d", c, len, flags); if (c->rnb != NULL) - fatal("net_recv_queue(): called incorrectly for %p", c); + fatal("net_recv_queue(): called incorrectly"); - c->rnb = kore_pool_get(&nb_pool); + c->rnb = net_netbuf_get(); c->rnb->cb = cb; c->rnb->owner = c; - c->rnb->s_off = 0; c->rnb->b_len = len; c->rnb->m_len = len; - c->rnb->extra = NULL; c->rnb->flags = flags; c->rnb->type = NETBUF_RECV; c->rnb->buf = kore_malloc(c->rnb->b_len); @@ -162,9 +210,6 @@ { kore_debug("net_recv_expand(): %p %d", c, len); - if (c->rnb->type != NETBUF_RECV) - fatal("net_recv_expand(): wrong netbuf type"); - c->rnb->cb = cb; c->rnb->b_len += len; c->rnb->m_len = c->rnb->b_len; @@ -174,29 +219,33 @@ int net_send(struct connection *c) { - int r; - u_int32_t len, smin; + size_t r, len, smin; c->snb = TAILQ_FIRST(&(c->send_queue)); + +#if defined(KORE_USE_PLATFORM_SENDFILE) + if ((c->snb->flags & NETBUF_IS_FILEREF) && + !(c->snb->flags & NETBUF_IS_STREAM)) { + return (kore_platform_sendfile(c, c->snb)); + } +#endif + if (c->snb->b_len != 0) { smin = c->snb->b_len - c->snb->s_off; len = MIN(NETBUF_SEND_PAYLOAD_MAX, smin); if (!c->write(c, len, &r)) return (KORE_RESULT_ERROR); - if (!(c->flags & CONN_WRITE_POSSIBLE)) + if (!(c->evt.flags & KORE_EVENT_WRITE)) return (KORE_RESULT_OK); - kore_debug("net_send(%p/%d/%d bytes), progress with %d", - c->snb, c->snb->s_off, c->snb->b_len, r); - - c->snb->s_off += (size_t)r; + c->snb->s_off += r; c->snb->flags &= ~NETBUF_MUST_RESEND; } if (c->snb->s_off == c->snb->b_len || (c->snb->flags & NETBUF_FORCE_REMOVE)) { - net_remove_netbuf(&(c->send_queue), c->snb); + net_remove_netbuf(c, c->snb); c->snb = NULL; } @@ -209,13 +258,14 @@ kore_debug("net_send_flush(%p)", c); while (!TAILQ_EMPTY(&(c->send_queue)) && - (c->flags & CONN_WRITE_POSSIBLE)) { + (c->evt.flags & KORE_EVENT_WRITE)) { if (!net_send(c)) return (KORE_RESULT_ERROR); } - if ((c->flags & CONN_CLOSE_EMPTY) && TAILQ_EMPTY(&(c->send_queue))) + if ((c->flags & CONN_CLOSE_EMPTY) && TAILQ_EMPTY(&(c->send_queue))) { kore_connection_disconnect(c); + } return (KORE_RESULT_OK); } @@ -223,23 +273,23 @@ int net_recv_flush(struct connection *c) { - int r; + size_t r; kore_debug("net_recv_flush(%p)", c); if (c->rnb == NULL) return (KORE_RESULT_OK); - while (c->flags & CONN_READ_POSSIBLE) { + while (c->evt.flags & KORE_EVENT_READ) { + if (c->rnb->buf == NULL) + return (KORE_RESULT_OK); + if (!c->read(c, &r)) return (KORE_RESULT_ERROR); - if (!(c->flags & CONN_READ_POSSIBLE)) + if (!(c->evt.flags & KORE_EVENT_READ)) break; - kore_debug("net_recv(%ld/%ld bytes), progress with %d", - c->rnb->s_off, c->rnb->b_len, r); - - c->rnb->s_off += (size_t)r; + c->rnb->s_off += r; if (c->rnb->s_off == c->rnb->b_len || (c->rnb->flags & NETBUF_CALL_CB_ALWAYS)) { r = c->rnb->cb(c->rnb); @@ -252,9 +302,9 @@ } void -net_remove_netbuf(struct netbuf_head *list, struct netbuf *nb) +net_remove_netbuf(struct connection *c, struct netbuf *nb) { - kore_debug("net_remove_netbuf(%p, %p)", list, nb); + kore_debug("net_remove_netbuf(%p, %p)", c, nb); if (nb->type == NETBUF_RECV) fatal("net_remove_netbuf(): cannot remove recv netbuf"); @@ -271,16 +321,24 @@ (void)nb->cb(nb); } - TAILQ_REMOVE(list, nb, list); + if (nb->flags & NETBUF_IS_FILEREF) + kore_fileref_release(nb->file_ref); + + TAILQ_REMOVE(&(c->send_queue), nb, list); + kore_pool_put(&nb_pool, nb); } #if !defined(KORE_NO_TLS) int -net_write_ssl(struct connection *c, int len, int *written) +net_write_tls(struct connection *c, size_t len, size_t *written) { int r; + if (len > INT_MAX) + return (KORE_RESULT_ERROR); + + ERR_clear_error(); r = SSL_write(c->ssl, (c->snb->buf + c->snb->s_off), len); if (c->tls_reneg > 1) return (KORE_RESULT_ERROR); @@ -290,8 +348,8 @@ switch (r) { case SSL_ERROR_WANT_READ: case SSL_ERROR_WANT_WRITE: + c->evt.flags &= ~KORE_EVENT_WRITE; c->snb->flags |= NETBUF_MUST_RESEND; - c->flags &= ~CONN_WRITE_POSSIBLE; return (KORE_RESULT_OK); case SSL_ERROR_SYSCALL: switch (errno) { @@ -299,8 +357,8 @@ *written = 0; return (KORE_RESULT_OK); case EAGAIN: + c->evt.flags &= ~KORE_EVENT_WRITE; c->snb->flags |= NETBUF_MUST_RESEND; - c->flags &= ~CONN_WRITE_POSSIBLE; return (KORE_RESULT_OK); default: break; @@ -312,15 +370,17 @@ } } - *written = r; + *written = (size_t)r; + return (KORE_RESULT_OK); } int -net_read_ssl(struct connection *c, int *bytes) +net_read_tls(struct connection *c, size_t *bytes) { int r; + ERR_clear_error(); r = SSL_read(c->ssl, (c->rnb->buf + c->rnb->s_off), (c->rnb->b_len - c->rnb->s_off)); @@ -332,7 +392,7 @@ switch (r) { case SSL_ERROR_WANT_READ: case SSL_ERROR_WANT_WRITE: - c->flags &= ~CONN_READ_POSSIBLE; + c->evt.flags &= ~KORE_EVENT_READ; return (KORE_RESULT_OK); case SSL_ERROR_SYSCALL: switch (errno) { @@ -340,8 +400,8 @@ *bytes = 0; return (KORE_RESULT_OK); case EAGAIN: + c->evt.flags &= ~KORE_EVENT_READ; c->snb->flags |= NETBUF_MUST_RESEND; - c->flags &= ~CONN_WRITE_POSSIBLE; return (KORE_RESULT_OK); default: break; @@ -353,24 +413,25 @@ } } - *bytes = r; + *bytes = (size_t)r; + return (KORE_RESULT_OK); } #endif int -net_write(struct connection *c, int len, int *written) +net_write(struct connection *c, size_t len, size_t *written) { - int r; + ssize_t r; r = write(c->fd, (c->snb->buf + c->snb->s_off), len); - if (r <= -1) { + if (r == -1) { switch (errno) { case EINTR: *written = 0; return (KORE_RESULT_OK); case EAGAIN: - c->flags &= ~CONN_WRITE_POSSIBLE; + c->evt.flags &= ~KORE_EVENT_WRITE; return (KORE_RESULT_OK); default: kore_debug("write: %s", errno_s); @@ -378,24 +439,25 @@ } } - *written = r; + *written = (size_t)r; + return (KORE_RESULT_OK); } int -net_read(struct connection *c, int *bytes) +net_read(struct connection *c, size_t *bytes) { - int r; + ssize_t r; r = read(c->fd, (c->rnb->buf + c->rnb->s_off), (c->rnb->b_len - c->rnb->s_off)); - if (r <= 0) { + if (r == -1) { switch (errno) { case EINTR: *bytes = 0; return (KORE_RESULT_OK); case EAGAIN: - c->flags &= ~CONN_READ_POSSIBLE; + c->evt.flags &= ~KORE_EVENT_READ; return (KORE_RESULT_OK); default: kore_debug("read(): %s", errno_s); @@ -403,7 +465,14 @@ } } - *bytes = r; + if (r == 0) { + kore_connection_disconnect(c); + c->evt.flags &= ~KORE_EVENT_READ; + return (KORE_RESULT_OK); + } + + *bytes = (size_t)r; + return (KORE_RESULT_OK); } diff -Nru kore-2.0.0/src/pgsql.c kore-3.3.1/src/pgsql.c --- kore-2.0.0/src/pgsql.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/pgsql.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2016 Joris Vink + * Copyright (c) 2014-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -21,28 +21,33 @@ #include #include "kore.h" + +#if !defined(KORE_NO_HTTP) #include "http.h" +#endif + #include "pgsql.h" -struct pgsql_job { - struct http_request *req; +struct pgsql_wait { struct kore_pgsql *pgsql; - - TAILQ_ENTRY(pgsql_job) list; + TAILQ_ENTRY(pgsql_wait) list; }; -struct pgsql_wait { - struct http_request *req; - TAILQ_ENTRY(pgsql_wait) list; +struct pgsql_job { + struct kore_pgsql *pgsql; + TAILQ_ENTRY(pgsql_job) list; }; #define PGSQL_CONN_MAX 2 #define PGSQL_CONN_FREE 0x01 #define PGSQL_LIST_INSERTED 0x0100 +#define PGSQL_QUEUE_LIMIT 1000 static void pgsql_queue_wakeup(void); +static void pgsql_cancel(struct kore_pgsql *); static void pgsql_set_error(struct kore_pgsql *, const char *); -static void pgsql_queue_add(struct http_request *); +static void pgsql_queue_add(struct kore_pgsql *); +static void pgsql_queue_remove(struct kore_pgsql *); static void pgsql_conn_release(struct kore_pgsql *); static void pgsql_conn_cleanup(struct pgsql_conn *); static void pgsql_read_result(struct kore_pgsql *); @@ -51,21 +56,21 @@ static struct pgsql_conn *pgsql_conn_create(struct kore_pgsql *, struct pgsql_db *); static struct pgsql_conn *pgsql_conn_next(struct kore_pgsql *, - struct pgsql_db *, - struct http_request *); + struct pgsql_db *); static struct kore_pool pgsql_job_pool; static struct kore_pool pgsql_wait_pool; static TAILQ_HEAD(, pgsql_conn) pgsql_conn_free; static TAILQ_HEAD(, pgsql_wait) pgsql_wait_queue; static LIST_HEAD(, pgsql_db) pgsql_db_conn_strings; -static u_int16_t pgsql_conn_count; -u_int16_t pgsql_conn_max = PGSQL_CONN_MAX; + +u_int32_t pgsql_queue_count = 0; +u_int16_t pgsql_conn_max = PGSQL_CONN_MAX; +u_int32_t pgsql_queue_limit = PGSQL_QUEUE_LIMIT; void -kore_pgsql_init(void) +kore_pgsql_sys_init(void) { - pgsql_conn_count = 0; TAILQ_INIT(&pgsql_conn_free); TAILQ_INIT(&pgsql_wait_queue); LIST_INIT(&pgsql_db_conn_strings); @@ -73,26 +78,50 @@ kore_pool_init(&pgsql_job_pool, "pgsql_job_pool", sizeof(struct pgsql_job), 100); kore_pool_init(&pgsql_wait_pool, "pgsql_wait_pool", - sizeof(struct pgsql_wait), 100); + sizeof(struct pgsql_wait), pgsql_queue_limit); } -int -kore_pgsql_query_init(struct kore_pgsql *pgsql, struct http_request *req, - const char *dbname, int flags) +void +kore_pgsql_sys_cleanup(void) { - struct pgsql_db *db; + struct pgsql_conn *conn, *next; + + kore_pool_cleanup(&pgsql_job_pool); + kore_pool_cleanup(&pgsql_wait_pool); + + for (conn = TAILQ_FIRST(&pgsql_conn_free); conn != NULL; conn = next) { + next = TAILQ_NEXT(conn, list); + pgsql_conn_cleanup(conn); + } +} +void +kore_pgsql_init(struct kore_pgsql *pgsql) +{ memset(pgsql, 0, sizeof(*pgsql)); - pgsql->flags = flags; pgsql->state = KORE_PGSQL_STATE_INIT; +} + +int +kore_pgsql_setup(struct kore_pgsql *pgsql, const char *dbname, int flags) +{ + struct pgsql_db *db; - if ((req == NULL && (flags & KORE_PGSQL_ASYNC)) || - ((flags & KORE_PGSQL_ASYNC) && (flags & KORE_PGSQL_SYNC))) { + if ((flags & KORE_PGSQL_ASYNC) && (flags & KORE_PGSQL_SYNC)) { pgsql_set_error(pgsql, "invalid query init parameters"); return (KORE_RESULT_ERROR); } + if (flags & KORE_PGSQL_ASYNC) { + if (pgsql->req == NULL && pgsql->cb == NULL) { + pgsql_set_error(pgsql, "nothing was bound"); + return (KORE_RESULT_ERROR); + } + } + db = NULL; + pgsql->flags |= flags; + LIST_FOREACH(db, &pgsql_db_conn_strings, rlist) { if (!strcmp(db->name, dbname)) break; @@ -103,24 +132,47 @@ return (KORE_RESULT_ERROR); } - if ((pgsql->conn = pgsql_conn_next(pgsql, db, req)) == NULL) + if ((pgsql->conn = pgsql_conn_next(pgsql, db)) == NULL) return (KORE_RESULT_ERROR); if (pgsql->flags & KORE_PGSQL_ASYNC) { pgsql->conn->job = kore_pool_get(&pgsql_job_pool); - pgsql->conn->job->req = req; pgsql->conn->job->pgsql = pgsql; - - http_request_sleep(req); - pgsql->flags |= PGSQL_LIST_INSERTED; - LIST_INSERT_HEAD(&(req->pgsqls), pgsql, rlist); } return (KORE_RESULT_OK); } +#if !defined(KORE_NO_HTTP) +void +kore_pgsql_bind_request(struct kore_pgsql *pgsql, struct http_request *req) +{ + if (pgsql->req != NULL || pgsql->cb != NULL) + fatal("kore_pgsql_bind_request: already bound"); + + pgsql->req = req; + pgsql->flags |= PGSQL_LIST_INSERTED; + + LIST_INSERT_HEAD(&(req->pgsqls), pgsql, rlist); +} +#endif + +void +kore_pgsql_bind_callback(struct kore_pgsql *pgsql, + void (*cb)(struct kore_pgsql *, void *), void *arg) +{ + if (pgsql->req != NULL) + fatal("kore_pgsql_bind_callback: already bound"); + + if (pgsql->cb != NULL) + fatal("kore_pgsql_bind_callback: already bound"); + + pgsql->cb = cb; + pgsql->arg = arg; +} + int -kore_pgsql_query(struct kore_pgsql *pgsql, const char *query) +kore_pgsql_query(struct kore_pgsql *pgsql, const void *query) { if (pgsql->conn == NULL) { pgsql_set_error(pgsql, "no connection was set before query"); @@ -129,7 +181,6 @@ if (pgsql->flags & KORE_PGSQL_SYNC) { pgsql->result = PQexec(pgsql->conn->db, query); - if ((PQresultStatus(pgsql->result) != PGRES_TUPLES_OK) && (PQresultStatus(pgsql->result) != PGRES_COMMAND_OK)) { pgsql_set_error(pgsql, PQerrorMessage(pgsql->conn->db)); @@ -151,10 +202,10 @@ int kore_pgsql_v_query_params(struct kore_pgsql *pgsql, - const char *query, int result, u_int8_t count, va_list args) + const void *query, int binary, int count, va_list args) { - u_int8_t i; - char **values; + int i; + const char **values; int *lengths, *formats, ret; if (pgsql->conn == NULL) { @@ -169,7 +220,7 @@ for (i = 0; i < count; i++) { values[i] = va_arg(args, void *); - lengths[i] = va_arg(args, u_int32_t); + lengths[i] = va_arg(args, int); formats[i] = va_arg(args, int); } } else { @@ -178,54 +229,60 @@ values = NULL; } - ret = KORE_RESULT_ERROR; + ret = kore_pgsql_query_param_fields(pgsql, query, binary, count, + values, lengths, formats); + + kore_free(values); + kore_free(lengths); + kore_free(formats); + + return (ret); +} + +int +kore_pgsql_query_param_fields(struct kore_pgsql *pgsql, const void *query, + int binary, int count, const char **values, int *lengths, int *formats) +{ + if (pgsql->conn == NULL) { + pgsql_set_error(pgsql, "no connection was set before query"); + return (KORE_RESULT_ERROR); + } if (pgsql->flags & KORE_PGSQL_SYNC) { pgsql->result = PQexecParams(pgsql->conn->db, query, count, NULL, (const char * const *)values, lengths, formats, - result); + binary); if ((PQresultStatus(pgsql->result) != PGRES_TUPLES_OK) && (PQresultStatus(pgsql->result) != PGRES_COMMAND_OK)) { pgsql_set_error(pgsql, PQerrorMessage(pgsql->conn->db)); - goto cleanup; + return (KORE_RESULT_ERROR); } pgsql->state = KORE_PGSQL_STATE_DONE; } else { if (!PQsendQueryParams(pgsql->conn->db, query, count, NULL, - (const char * const *)values, lengths, formats, result)) { + (const char * const *)values, lengths, formats, binary)) { pgsql_set_error(pgsql, PQerrorMessage(pgsql->conn->db)); - goto cleanup; + return (KORE_RESULT_ERROR); } pgsql_schedule(pgsql); } - ret = KORE_RESULT_OK; - -cleanup: - kore_free(values); - kore_free(lengths); - kore_free(formats); - - return (ret); + return (KORE_RESULT_OK); } int kore_pgsql_query_params(struct kore_pgsql *pgsql, - const char *query, int result, u_int8_t count, ...) + const void *query, int binary, int count, ...) { int ret; va_list args; - if (count > 0) - va_start(args, count); - - ret = kore_pgsql_v_query_params(pgsql, query, result, count, args); - - if (count > 0) - va_end(args); + va_start(args, count); + ret = kore_pgsql_v_query_params(pgsql, query, binary, count, args); + va_end(args); return (ret); } @@ -242,6 +299,8 @@ pgsqldb = kore_malloc(sizeof(*pgsqldb)); pgsqldb->name = kore_strdup(dbname); + pgsqldb->conn_count = 0; + pgsqldb->conn_max = pgsql_conn_max; pgsqldb->conn_string = kore_strdup(connstring); LIST_INSERT_HEAD(&pgsql_db_conn_strings, pgsqldb, rlist); @@ -251,7 +310,6 @@ void kore_pgsql_handle(void *c, int err) { - struct http_request *req; struct kore_pgsql *pgsql; struct pgsql_conn *conn = (struct pgsql_conn *)c; @@ -260,9 +318,10 @@ return; } - req = conn->job->req; + if (!(conn->evt.flags & KORE_EVENT_READ)) + fatal("%s: read event not set", __func__); + pgsql = conn->job->pgsql; - kore_debug("kore_pgsql_handle: %p (%d)", req, pgsql->state); if (!PQconsumeInput(conn->db)) { pgsql->state = KORE_PGSQL_STATE_ERROR; @@ -272,18 +331,25 @@ } if (pgsql->state == KORE_PGSQL_STATE_WAIT) { - http_request_sleep(req); +#if !defined(KORE_NO_HTTP) + if (pgsql->req != NULL) + http_request_sleep(pgsql->req); +#endif + if (pgsql->cb != NULL) + pgsql->cb(pgsql, pgsql->arg); } else { - http_request_wakeup(req); +#if !defined(KORE_NO_HTTP) + if (pgsql->req != NULL) + http_request_wakeup(pgsql->req); +#endif + if (pgsql->cb != NULL) + pgsql->cb(pgsql, pgsql->arg); } } void -kore_pgsql_continue(struct http_request *req, struct kore_pgsql *pgsql) +kore_pgsql_continue(struct kore_pgsql *pgsql) { - kore_debug("kore_pgsql_continue: %p->%p (%d)", - req->owner, req, pgsql->state); - if (pgsql->error) { kore_free(pgsql->error); pgsql->error = NULL; @@ -299,11 +365,15 @@ case KORE_PGSQL_STATE_WAIT: break; case KORE_PGSQL_STATE_DONE: - http_request_wakeup(req); +#if !defined(KORE_NO_HTTP) + if (pgsql->req != NULL) + http_request_wakeup(pgsql->req); +#endif pgsql_conn_release(pgsql); break; case KORE_PGSQL_STATE_ERROR: case KORE_PGSQL_STATE_RESULT: + case KORE_PGSQL_STATE_NOTIFY: kore_pgsql_handle(pgsql->conn, 0); break; default: @@ -314,7 +384,7 @@ void kore_pgsql_cleanup(struct kore_pgsql *pgsql) { - kore_debug("kore_pgsql_cleanup(%p)", pgsql); + pgsql_queue_remove(pgsql); if (pgsql->result != NULL) PQclear(pgsql->result); @@ -349,39 +419,43 @@ } int +kore_pgsql_nfields(struct kore_pgsql *pgsql) +{ + return (PQnfields(pgsql->result)); +} + +int kore_pgsql_getlength(struct kore_pgsql *pgsql, int row, int col) { return (PQgetlength(pgsql->result, row, col)); } char * +kore_pgsql_fieldname(struct kore_pgsql *pgsql, int field) +{ + return (PQfname(pgsql->result, field)); +} + +char * kore_pgsql_getvalue(struct kore_pgsql *pgsql, int row, int col) { return (PQgetvalue(pgsql->result, row, col)); } -void -kore_pgsql_queue_remove(struct http_request *req) +int +kore_pgsql_column_binary(struct kore_pgsql *pgsql, int col) { - struct pgsql_wait *pgw, *next; - - for (pgw = TAILQ_FIRST(&pgsql_wait_queue); pgw != NULL; pgw = next) { - next = TAILQ_NEXT(pgw, list); - if (pgw->req != req) - continue; - - TAILQ_REMOVE(&pgsql_wait_queue, pgw, list); - kore_pool_put(&pgsql_wait_pool, pgw); - return; - } + return (PQfformat(pgsql->result, col)); } static struct pgsql_conn * -pgsql_conn_next(struct kore_pgsql *pgsql, struct pgsql_db *db, - struct http_request *req) +pgsql_conn_next(struct kore_pgsql *pgsql, struct pgsql_db *db) { - struct pgsql_conn *conn; + PGTransactionStatusType state; + struct pgsql_conn *conn; + struct kore_pgsql rollback; +rescan: conn = NULL; TAILQ_FOREACH(conn, &pgsql_conn_free, list) { @@ -391,10 +465,34 @@ break; } + if (conn != NULL) { + state = PQtransactionStatus(conn->db); + if (state == PQTRANS_INERROR) { + conn->flags &= ~PGSQL_CONN_FREE; + TAILQ_REMOVE(&pgsql_conn_free, conn, list); + + kore_pgsql_init(&rollback); + rollback.conn = conn; + rollback.flags = KORE_PGSQL_SYNC; + + if (!kore_pgsql_query(&rollback, "ROLLBACK")) { + kore_pgsql_logerror(&rollback); + kore_pgsql_cleanup(&rollback); + pgsql_conn_cleanup(conn); + } else { + kore_pgsql_cleanup(&rollback); + } + + goto rescan; + } + } + if (conn == NULL) { - if (pgsql_conn_count >= pgsql_conn_max) { - if (pgsql->flags & KORE_PGSQL_ASYNC) { - pgsql_queue_add(req); + if (db->conn_max != 0 && + db->conn_count >= db->conn_max) { + if ((pgsql->flags & KORE_PGSQL_ASYNC) && + pgsql_queue_count < pgsql_queue_limit) { + pgsql_queue_add(pgsql); } else { pgsql_set_error(pgsql, "no available connection"); @@ -434,35 +532,74 @@ kore_platform_schedule_read(fd, pgsql->conn); pgsql->state = KORE_PGSQL_STATE_WAIT; + pgsql->flags |= KORE_PGSQL_SCHEDULED; + +#if !defined(KORE_NO_HTTP) + if (pgsql->req != NULL) + http_request_sleep(pgsql->req); +#endif + if (pgsql->cb != NULL) + pgsql->cb(pgsql, pgsql->arg); } static void -pgsql_queue_add(struct http_request *req) +pgsql_queue_add(struct kore_pgsql *pgsql) { struct pgsql_wait *pgw; - http_request_sleep(req); +#if !defined(KORE_NO_HTTP) + if (pgsql->req != NULL) + http_request_sleep(pgsql->req); +#endif pgw = kore_pool_get(&pgsql_wait_pool); - pgw->req = req; - pgw->req->flags |= HTTP_REQUEST_PGSQL_QUEUE; + pgw->pgsql = pgsql; + pgsql_queue_count++; TAILQ_INSERT_TAIL(&pgsql_wait_queue, pgw, list); } static void -pgsql_queue_wakeup(void) +pgsql_queue_remove(struct kore_pgsql *pgsql) { struct pgsql_wait *pgw, *next; for (pgw = TAILQ_FIRST(&pgsql_wait_queue); pgw != NULL; pgw = next) { next = TAILQ_NEXT(pgw, list); - if (pgw->req->flags & HTTP_REQUEST_DELETE) + if (pgw->pgsql != pgsql) continue; - http_request_wakeup(pgw->req); - pgw->req->flags &= ~HTTP_REQUEST_PGSQL_QUEUE; + pgsql_queue_count--; + TAILQ_REMOVE(&pgsql_wait_queue, pgw, list); + kore_pool_put(&pgsql_wait_pool, pgw); + return; + } +} + +static void +pgsql_queue_wakeup(void) +{ + struct pgsql_wait *pgw, *next; + + for (pgw = TAILQ_FIRST(&pgsql_wait_queue); pgw != NULL; pgw = next) { + next = TAILQ_NEXT(pgw, list); + +#if !defined(KORE_NO_HTTP) + if (pgw->pgsql->req != NULL) { + if (pgw->pgsql->req->flags & HTTP_REQUEST_DELETE) { + pgsql_queue_count--; + TAILQ_REMOVE(&pgsql_wait_queue, pgw, list); + kore_pool_put(&pgsql_wait_pool, pgw); + continue; + } + + http_request_wakeup(pgw->pgsql->req); + } +#endif + if (pgw->pgsql->cb != NULL) + pgw->pgsql->cb(pgw->pgsql, pgw->pgsql->arg); + pgsql_queue_count--; TAILQ_REMOVE(&pgsql_wait_queue, pgw, list); kore_pool_put(&pgsql_wait_pool, pgw); return; @@ -477,9 +614,16 @@ if (db == NULL || db->conn_string == NULL) fatal("pgsql_conn_create: no connection string"); - pgsql_conn_count++; - conn = kore_malloc(sizeof(*conn)); - kore_debug("pgsql_conn_create(): %p", conn); + db->conn_count++; + + conn = kore_calloc(1, sizeof(*conn)); + conn->job = NULL; + conn->flags = PGSQL_CONN_FREE; + conn->name = kore_strdup(db->name); + TAILQ_INSERT_TAIL(&pgsql_conn_free, conn, list); + + conn->evt.type = KORE_TYPE_PGSQL_CONN; + conn->evt.handle = kore_pgsql_handle; conn->db = PQconnectdb(db->conn_string); if (conn->db == NULL || (PQstatus(conn->db) != CONNECTION_OK)) { @@ -488,12 +632,6 @@ return (NULL); } - conn->job = NULL; - conn->flags = PGSQL_CONN_FREE; - conn->type = KORE_TYPE_PGSQL_CONN; - conn->name = kore_strdup(db->name); - TAILQ_INSERT_TAIL(&pgsql_conn_free, conn, list); - return (conn); } @@ -501,22 +639,26 @@ pgsql_conn_release(struct kore_pgsql *pgsql) { int fd; + PGresult *result; if (pgsql->conn == NULL) return; /* Async query cleanup */ if (pgsql->flags & KORE_PGSQL_ASYNC) { - if (pgsql->conn != NULL) { + if (pgsql->flags & KORE_PGSQL_SCHEDULED) { fd = PQsocket(pgsql->conn->db); kore_platform_disable_read(fd); - kore_pool_put(&pgsql_job_pool, pgsql->conn->job); + + if (pgsql->state != KORE_PGSQL_STATE_DONE) + pgsql_cancel(pgsql); } + kore_pool_put(&pgsql_job_pool, pgsql->conn->job); } /* Drain just in case. */ - while (PQgetResult(pgsql->conn->db) != NULL) - ; + while ((result = PQgetResult(pgsql->conn->db)) != NULL) + PQclear(result); pgsql->conn->job = NULL; pgsql->conn->flags |= PGSQL_CONN_FREE; @@ -525,25 +667,27 @@ pgsql->conn = NULL; pgsql->state = KORE_PGSQL_STATE_COMPLETE; + if (pgsql->cb != NULL) + pgsql->cb(pgsql, pgsql->arg); + pgsql_queue_wakeup(); } static void pgsql_conn_cleanup(struct pgsql_conn *conn) { - struct http_request *req; struct kore_pgsql *pgsql; - - kore_debug("pgsql_conn_cleanup(): %p", conn); + struct pgsql_db *pgsqldb; if (conn->flags & PGSQL_CONN_FREE) TAILQ_REMOVE(&pgsql_conn_free, conn, list); if (conn->job) { - req = conn->job->req; pgsql = conn->job->pgsql; - http_request_wakeup(req); - +#if !defined(KORE_NO_HTTP) + if (pgsql->req != NULL) + http_request_wakeup(pgsql->req); +#endif pgsql->conn = NULL; pgsql_set_error(pgsql, PQerrorMessage(conn->db)); @@ -554,7 +698,13 @@ if (conn->db != NULL) PQfinish(conn->db); - pgsql_conn_count--; + LIST_FOREACH(pgsqldb, &pgsql_db_conn_strings, rlist) { + if (!strcmp(pgsqldb->name, conn->name)) { + pgsqldb->conn_count--; + break; + } + } + kore_free(conn->name); kore_free(conn); } @@ -562,11 +712,24 @@ static void pgsql_read_result(struct kore_pgsql *pgsql) { + PGnotify *notify; + if (PQisBusy(pgsql->conn->db)) { pgsql->state = KORE_PGSQL_STATE_WAIT; return; } + while ((notify = PQnotifies(pgsql->conn->db)) != NULL) { + pgsql->state = KORE_PGSQL_STATE_NOTIFY; + pgsql->notify.extra = notify->extra; + pgsql->notify.channel = notify->relname; + + if (pgsql->cb != NULL) + pgsql->cb(pgsql, pgsql->arg); + + PQfreemem(notify); + } + pgsql->result = PQgetResult(pgsql->conn->db); if (pgsql->result == NULL) { pgsql->state = KORE_PGSQL_STATE_DONE; @@ -595,3 +758,16 @@ break; } } + +static void +pgsql_cancel(struct kore_pgsql *pgsql) +{ + PGcancel *cancel; + char buf[256]; + + if ((cancel = PQgetCancel(pgsql->conn->db)) != NULL) { + if (!PQcancel(cancel, buf, sizeof(buf))) + kore_log(LOG_ERR, "failed to cancel: %s", buf); + PQfreeCancel(cancel); + } +} diff -Nru kore-2.0.0/src/pool.c kore-3.3.1/src/pool.c --- kore-2.0.0/src/pool.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/pool.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include @@ -21,6 +22,8 @@ #include "kore.h" +#define POOL_MIN_ELEMENTS 16 + #define POOL_ELEMENT_BUSY 0 #define POOL_ELEMENT_FREE 1 @@ -38,6 +41,9 @@ { kore_debug("kore_pool_init(%p, %s, %zu, %zu)", pool, name, len, elm); + if (elm < POOL_MIN_ELEMENTS) + elm = POOL_MIN_ELEMENTS; + if ((pool->name = strdup(name)) == NULL) fatal("kore_pool_init: strdup %s", errno_s); @@ -45,6 +51,7 @@ pool->elms = 0; pool->inuse = 0; pool->elen = len; + pool->growth = elm * 0.25f; pool->slen = pool->elen + sizeof(struct kore_pool_entry); LIST_INIT(&(pool->regions)); @@ -62,10 +69,8 @@ pool->elen = 0; pool->slen = 0; - if (pool->name != NULL) { - free(pool->name); - pool->name = NULL; - } + free(pool->name); + pool->name = NULL; pool_region_destroy(pool); } @@ -80,11 +85,8 @@ pool_lock(pool); #endif - if (LIST_EMPTY(&(pool->freelist))) { - kore_log(LOG_NOTICE, "pool %s is exhausted (%d/%d)", - pool->name, pool->inuse, pool->elms); - pool_region_create(pool, pool->elms); - } + if (LIST_EMPTY(&(pool->freelist))) + pool_region_create(pool, pool->growth); entry = LIST_FIRST(&(pool->freelist)); if (entry->state != POOL_ELEMENT_FREE) @@ -136,7 +138,7 @@ struct kore_pool_region *reg; struct kore_pool_entry *entry; - kore_debug("pool_region_create(%p, %d)", pool, elms); + kore_debug("pool_region_create(%p, %zu)", pool, elms); if ((reg = calloc(1, sizeof(struct kore_pool_region))) == NULL) fatal("pool_region_create: calloc: %s", errno_s); @@ -149,7 +151,7 @@ reg->length = elms * pool->slen; reg->start = mmap(NULL, reg->length, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (reg->start == NULL) + if (reg->start == MAP_FAILED) fatal("mmap: %s", errno_s); p = (u_int8_t *)reg->start; diff -Nru kore-2.0.0/src/python.c kore-3.3.1/src/python.c --- kore-2.0.0/src/python.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/src/python.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,4395 @@ +/* + * Copyright (c) 2016 Stanislav Yudin + * Copyright (c) 2017-2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "kore.h" +#include "http.h" + +#if defined(KORE_USE_PGSQL) +#include "pgsql.h" +#endif + +#if defined(KORE_USE_CURL) +#include "curl.h" +#endif + +#include "python_api.h" +#include "python_methods.h" + +static PyMODINIT_FUNC python_module_init(void); +static PyObject *python_import(const char *); +static PyObject *pyconnection_alloc(struct connection *); +static PyObject *python_callable(PyObject *, const char *); +static void python_split_arguments(char *, char **, size_t); + +static int pyhttp_response_sent(struct netbuf *); +static PyObject *pyhttp_file_alloc(struct http_file *); +static PyObject *pyhttp_request_alloc(const struct http_request *); + +static struct python_coro *python_coro_create(PyObject *, + struct http_request *); +static int python_coro_run(struct python_coro *); +static void python_coro_wakeup(struct python_coro *); +static void python_coro_suspend(struct python_coro *); + +static void pysocket_evt_handle(void *, int); +static void pysocket_op_timeout(void *, u_int64_t); +static PyObject *pysocket_op_create(struct pysocket *, + int, const void *, size_t); + +static struct pysocket *pysocket_alloc(void); +static PyObject *pysocket_async_recv(struct pysocket_op *); +static PyObject *pysocket_async_send(struct pysocket_op *); +static PyObject *pysocket_async_accept(struct pysocket_op *); +static PyObject *pysocket_async_connect(struct pysocket_op *); + +static void pylock_do_release(struct pylock *); + +static void pytimer_run(void *, u_int64_t); +static void pyproc_timeout(void *, u_int64_t); +static void pysuspend_wakeup(void *, u_int64_t); + +static void pygather_reap_coro(struct pygather_op *, + struct python_coro *); + +static int pyhttp_iterobj_chunk_sent(struct netbuf *); +static int pyhttp_iterobj_next(struct pyhttp_iterobj *); +static void pyhttp_iterobj_disconnect(struct connection *); + +#if defined(KORE_USE_PGSQL) +static PyObject *pykore_pgsql_alloc(struct http_request *, + const char *, const char *, PyObject *); +static int pykore_pgsql_params(struct pykore_pgsql *, PyObject *); +#endif + +#if defined(KORE_USE_CURL) +static void python_curl_callback(struct kore_curl *, void *); +static PyObject *pyhttp_client_request(struct pyhttp_client *, int, + PyObject *); +#endif + +static void python_append_path(const char *); +static void python_push_integer(PyObject *, const char *, long); +static void python_push_type(const char *, PyObject *, PyTypeObject *); + +static int python_validator_check(PyObject *); +static int python_runtime_http_request(void *, struct http_request *); +static int python_runtime_validator(void *, struct http_request *, + const void *); +static void python_runtime_wsmessage(void *, struct connection *, + u_int8_t, const void *, size_t); +static void python_runtime_execute(void *); +static int python_runtime_onload(void *, int); +static void python_runtime_configure(void *, int, char **); +static void python_runtime_connect(void *, struct connection *); + +static void python_module_load(struct kore_module *); +static void python_module_free(struct kore_module *); +static void python_module_reload(struct kore_module *); +static void *python_module_getsym(struct kore_module *, const char *); + +static void *python_malloc(void *, size_t); +static void *python_calloc(void *, size_t, size_t); +static void *python_realloc(void *, void *, size_t); +static void python_free(void *, void *); + +struct kore_module_functions kore_python_module = { + .free = python_module_free, + .load = python_module_load, + .getsym = python_module_getsym, + .reload = python_module_reload +}; + +struct kore_runtime kore_python_runtime = { + KORE_RUNTIME_PYTHON, + .http_request = python_runtime_http_request, + .validator = python_runtime_validator, + .wsconnect = python_runtime_connect, + .wsmessage = python_runtime_wsmessage, + .wsdisconnect = python_runtime_connect, + .onload = python_runtime_onload, + .connect = python_runtime_connect, + .execute = python_runtime_execute, + .configure = python_runtime_configure, +}; + +static struct { + const char *symbol; + int value; +} python_integers[] = { + { "LOG_ERR", LOG_ERR }, + { "LOG_INFO", LOG_INFO }, + { "LOG_NOTICE", LOG_NOTICE }, + { "RESULT_OK", KORE_RESULT_OK }, + { "RESULT_RETRY", KORE_RESULT_RETRY }, + { "RESULT_ERROR", KORE_RESULT_ERROR }, + { "MODULE_LOAD", KORE_MODULE_LOAD }, + { "MODULE_UNLOAD", KORE_MODULE_UNLOAD }, + { "TIMER_ONESHOT", KORE_TIMER_ONESHOT }, + { "CONN_PROTO_HTTP", CONN_PROTO_HTTP }, + { "CONN_PROTO_UNKNOWN", CONN_PROTO_UNKNOWN }, + { "CONN_PROTO_WEBSOCKET", CONN_PROTO_WEBSOCKET }, + { "CONN_STATE_ESTABLISHED", CONN_STATE_ESTABLISHED }, + { "HTTP_METHOD_GET", HTTP_METHOD_GET }, + { "HTTP_METHOD_PUT", HTTP_METHOD_PUT }, + { "HTTP_METHOD_HEAD", HTTP_METHOD_HEAD }, + { "HTTP_METHOD_POST", HTTP_METHOD_POST }, + { "HTTP_METHOD_DELETE", HTTP_METHOD_DELETE }, + { "HTTP_METHOD_OPTIONS", HTTP_METHOD_OPTIONS }, + { "HTTP_METHOD_PATCH", HTTP_METHOD_PATCH }, + { "WEBSOCKET_OP_TEXT", WEBSOCKET_OP_TEXT }, + { "WEBSOCKET_OP_BINARY", WEBSOCKET_OP_BINARY }, + { "WEBSOCKET_BROADCAST_LOCAL", WEBSOCKET_BROADCAST_LOCAL }, + { "WEBSOCKET_BROADCAST_GLOBAL", WEBSOCKET_BROADCAST_GLOBAL }, + { NULL, -1 } +}; + +static PyMemAllocatorEx allocator = { + .ctx = NULL, + .malloc = python_malloc, + .calloc = python_calloc, + .realloc = python_realloc, + .free = python_free +}; + +static TAILQ_HEAD(, pyproc) procs; + +static struct kore_pool coro_pool; +static struct kore_pool iterobj_pool; +static struct kore_pool queue_wait_pool; +static struct kore_pool gather_coro_pool; +static struct kore_pool queue_object_pool; +static struct kore_pool gather_result_pool; + +static u_int64_t coro_id; +static int coro_count; +static struct coro_list coro_runnable; +static struct coro_list coro_suspended; + +extern const char *__progname; + +/* XXX */ +static struct python_coro *coro_running = NULL; +static PyObject *python_tracer = NULL; + +void +kore_python_init(void) +{ + struct kore_runtime_call *rcall; + + coro_id = 0; + coro_count = 0; + + TAILQ_INIT(&procs); + TAILQ_INIT(&coro_runnable); + TAILQ_INIT(&coro_suspended); + + kore_pool_init(&coro_pool, "coropool", sizeof(struct python_coro), 100); + + kore_pool_init(&iterobj_pool, "iterobj_pool", + sizeof(struct pyhttp_iterobj), 100); + kore_pool_init(&queue_wait_pool, "queue_wait_pool", + sizeof(struct pyqueue_waiting), 100); + kore_pool_init(&gather_coro_pool, "gather_coro_pool", + sizeof(struct pygather_coro), 100); + kore_pool_init(&queue_object_pool, "queue_object_pool", + sizeof(struct pyqueue_object), 100); + kore_pool_init(&gather_result_pool, "gather_result_pool", + sizeof(struct pygather_result), 100); + + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocator); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocator); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &allocator); + PyMem_SetupDebugHooks(); + + if (PyImport_AppendInittab("kore", &python_module_init) == -1) + fatal("kore_python_init: failed to add new module"); + + rcall = kore_runtime_getcall("kore_python_preinit"); + if (rcall != NULL) { + kore_runtime_execute(rcall); + kore_free(rcall); + } + + Py_Initialize(); +} + +void +kore_python_cleanup(void) +{ + if (Py_IsInitialized()) { + PyErr_Clear(); + Py_Finalize(); + } +} + +void +kore_python_path(const char *path) +{ + python_append_path(path); +} + +void +kore_python_coro_run(void) +{ + struct pygather_op *op; + struct python_coro *coro; + + while ((coro = TAILQ_FIRST(&coro_runnable)) != NULL) { + if (coro->state != CORO_STATE_RUNNABLE) + fatal("non-runnable coro on coro_runnable"); + + if (python_coro_run(coro) == KORE_RESULT_OK) { + if (coro->gatherop != NULL) { + op = coro->gatherop; + if (op->coro->request != NULL) + http_request_wakeup(op->coro->request); + else + python_coro_wakeup(op->coro); + pygather_reap_coro(op, coro); + } else { + kore_python_coro_delete(coro); + } + } + } + + /* + * Let Kore do HTTP processing so awoken coroutines run asap without + * having to wait for a tick from the event loop. + * + * Maybe it is more beneficial that we track if something related + * to HTTP requests was awoken and only run if true? + */ + http_process(); +} + +void +kore_python_coro_delete(void *obj) +{ + struct python_coro *coro; + + coro = obj; + coro_count--; + + coro_running = coro; + Py_DECREF(coro->obj); + coro_running = NULL; + + if (coro->state == CORO_STATE_RUNNABLE) + TAILQ_REMOVE(&coro_runnable, coro, list); + else + TAILQ_REMOVE(&coro_suspended, coro, list); + + Py_XDECREF(coro->result); + kore_pool_put(&coro_pool, coro); +} + +void +kore_python_log_error(const char *function) +{ + const char *sval; + PyObject *ret, *repr, *type, *value, *traceback; + + if (!PyErr_Occurred() || PyErr_ExceptionMatches(PyExc_StopIteration)) + return; + + PyErr_Fetch(&type, &value, &traceback); + + if (type == NULL || value == NULL) { + kore_log(LOG_ERR, "unknown python exception in '%s'", function); + return; + } + + if (value == NULL || !PyObject_IsInstance(value, type)) + PyErr_NormalizeException(&type, &value, &traceback); + + /* + * If we're in an active coroutine and it was tied to a gather + * operation we have to make sure we can use the Exception that + * was thrown as the result value so we can propagate it via the + * return list of kore.gather(). + */ + if (coro_running != NULL && coro_running->gatherop != NULL) { + PyErr_SetObject(PyExc_StopIteration, value); + } else if (python_tracer != NULL) { + /* + * Call the user-supplied tracer callback. + */ + ret = PyObject_CallFunctionObjArgs(python_tracer, + type, value, traceback, NULL); + Py_XDECREF(ret); + } else { + if ((repr = PyObject_Repr(value)) == NULL) + sval = "unknown"; + else + sval = PyUnicode_AsUTF8(repr); + + kore_log(LOG_ERR, + "uncaught exception %s in '%s'", sval, function); + + Py_XDECREF(repr); + } + + Py_DECREF(type); + Py_DECREF(value); + Py_XDECREF(traceback); +} + +void +kore_python_proc_reap(void) +{ + struct pyproc *proc; + pid_t child; + int status; + + for (;;) { + if ((child = waitpid(-1, &status, WNOHANG)) == -1) { + if (errno == ECHILD) + return; + if (errno == EINTR) + continue; + kore_log(LOG_NOTICE, "waitpid: %s", errno_s); + return; + } + + if (child == 0) + return; + + proc = NULL; + + TAILQ_FOREACH(proc, &procs, list) { + if (proc->pid == child) + break; + } + + if (proc == NULL) + continue; + + proc->pid = -1; + proc->reaped = 1; + proc->status = status; + + if (proc->timer != NULL) { + kore_timer_remove(proc->timer); + proc->timer = NULL; + } + + if (proc->coro->request != NULL) + http_request_wakeup(proc->coro->request); + else + python_coro_wakeup(proc->coro); + } +} + +static void * +python_malloc(void *ctx, size_t len) +{ + return (kore_malloc(len)); +} + +static void * +python_calloc(void *ctx, size_t memb, size_t len) +{ + return (kore_calloc(memb, len)); +} + +static void * +python_realloc(void *ctx, void *ptr, size_t len) +{ + return (kore_realloc(ptr, len)); +} + +static void +python_free(void *ctx, void *ptr) +{ + kore_free(ptr); +} + +static void +python_module_free(struct kore_module *module) +{ + kore_free(module->path); + Py_DECREF(module->handle); + kore_free(module); +} + +static void +python_split_arguments(char *args, char **argv, size_t elm) +{ + size_t idx; + char *p, *line, *end; + + if (elm <= 1) + fatal("not enough elements (%zu)", elm); + + idx = 0; + line = args; + + for (p = line; *p != '\0'; p++) { + if (idx >= elm - 1) + break; + + if (*p == ' ') { + *p = '\0'; + if (*line != '\0') + argv[idx++] = line; + line = p + 1; + continue; + } + + if (*p != '"') + continue; + + line = p + 1; + if ((end = strchr(line, '"')) == NULL) + break; + + *end = '\0'; + argv[idx++] = line; + line = end + 1; + + while (isspace(*(unsigned char *)line)) + line++; + + p = line; + } + + if (idx < elm - 1 && *line != '\0') + argv[idx++] = line; + + argv[idx] = NULL; +} + +static void +python_module_reload(struct kore_module *module) +{ + PyObject *handle; + + PyErr_Clear(); + if ((handle = PyImport_ReloadModule(module->handle)) == NULL) { + kore_python_log_error("python_module_reload"); + return; + } + + Py_DECREF(module->handle); + module->handle = handle; +} + +static void +python_module_load(struct kore_module *module) +{ + module->handle = python_import(module->path); + if (module->handle == NULL) + fatal("%s: failed to import module", module->path); +} + +static void * +python_module_getsym(struct kore_module *module, const char *symbol) +{ + return (python_callable(module->handle, symbol)); +} + +static struct python_coro * +python_coro_create(PyObject *obj, struct http_request *req) +{ + struct python_coro *coro; + + if (!PyCoro_CheckExact(obj)) + fatal("%s: object is not a coroutine", __func__); + + coro = kore_pool_get(&coro_pool); + coro_count++; + + coro->result = NULL; + coro->sockop = NULL; + coro->gatherop = NULL; + coro->exception = NULL; + coro->exception_msg = NULL; + + coro->obj = obj; + coro->request = req; + coro->id = coro_id++; + coro->state = CORO_STATE_RUNNABLE; + + TAILQ_INSERT_HEAD(&coro_runnable, coro, list); + + if (coro->request != NULL) + http_request_sleep(coro->request); + + return (coro); +} + +static int +python_coro_run(struct python_coro *coro) +{ + PyObject *item; + PyObject *type, *traceback; + + if (coro->state != CORO_STATE_RUNNABLE) + fatal("non-runnable coro attempted to run"); + + coro_running = coro; + + for (;;) { + PyErr_Clear(); + + item = _PyGen_Send((PyGenObject *)coro->obj, NULL); + if (item == NULL) { + if (coro->gatherop == NULL && PyErr_Occurred() && + PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Fetch(&type, &coro->result, &traceback); + Py_DECREF(type); + Py_XDECREF(traceback); + } else { + kore_python_log_error("coroutine"); + } + + coro_running = NULL; + return (KORE_RESULT_OK); + } + + if (item == Py_None) { + Py_DECREF(item); + break; + } + + Py_DECREF(item); + } + + python_coro_suspend(coro); + coro_running = NULL; + + if (coro->request != NULL) + http_request_sleep(coro->request); + + return (KORE_RESULT_RETRY); +} + +static void +python_coro_wakeup(struct python_coro *coro) +{ + if (coro->state != CORO_STATE_SUSPENDED) + return; + + coro->state = CORO_STATE_RUNNABLE; + TAILQ_REMOVE(&coro_suspended, coro, list); + TAILQ_INSERT_HEAD(&coro_runnable, coro, list); +} + +static void +python_coro_suspend(struct python_coro *coro) +{ + if (coro->state != CORO_STATE_RUNNABLE) + return; + + coro->state = CORO_STATE_SUSPENDED; + TAILQ_REMOVE(&coro_runnable, coro, list); + TAILQ_INSERT_HEAD(&coro_suspended, coro, list); +} + +static void +pyconnection_dealloc(struct pyconnection *pyc) +{ + PyObject_Del((PyObject *)pyc); +} + +static void +pyhttp_dealloc(struct pyhttp_request *pyreq) +{ + Py_XDECREF(pyreq->data); + PyObject_Del((PyObject *)pyreq); +} + +static void +pyhttp_file_dealloc(struct pyhttp_file *pyfile) +{ + PyObject_Del((PyObject *)pyfile); +} + +static int +python_runtime_http_request(void *addr, struct http_request *req) +{ + PyObject *pyret, *pyreq, *args, *callable; + + if (req->py_coro != NULL) { + python_coro_wakeup(req->py_coro); + if (python_coro_run(req->py_coro) == KORE_RESULT_OK) { + kore_python_coro_delete(req->py_coro); + req->py_coro = NULL; + return (KORE_RESULT_OK); + } + return (KORE_RESULT_RETRY); + } + + callable = (PyObject *)addr; + + if ((pyreq = pyhttp_request_alloc(req)) == NULL) + fatal("python_runtime_http_request: pyreq alloc failed"); + + if ((args = PyTuple_New(1)) == NULL) + fatal("python_runtime_http_request: PyTuple_New failed"); + + if (PyTuple_SetItem(args, 0, pyreq) != 0) + fatal("python_runtime_http_request: PyTuple_SetItem failed"); + + PyErr_Clear(); + pyret = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + + if (pyret == NULL) { + kore_python_log_error("python_runtime_http_request"); + http_response(req, HTTP_STATUS_INTERNAL_ERROR, NULL, 0); + return (KORE_RESULT_OK); + } + + if (PyCoro_CheckExact(pyret)) { + req->py_coro = python_coro_create(pyret, req); + if (python_coro_run(req->py_coro) == KORE_RESULT_OK) { + http_request_wakeup(req); + kore_python_coro_delete(req->py_coro); + req->py_coro = NULL; + return (KORE_RESULT_OK); + } + return (KORE_RESULT_RETRY); + } + + if (pyret != Py_None) + fatal("python_runtime_http_request: unexpected return type"); + + Py_DECREF(pyret); + + return (KORE_RESULT_OK); +} + +static int +python_runtime_validator(void *addr, struct http_request *req, const void *data) +{ + int ret; + struct python_coro *coro; + PyObject *pyret, *pyreq, *args, *callable, *arg; + + if (req->py_coro != NULL) { + coro = req->py_coro; + python_coro_wakeup(coro); + if (python_coro_run(coro) == KORE_RESULT_OK) { + ret = python_validator_check(coro->result); + kore_python_coro_delete(coro); + req->py_coro = NULL; + return (ret); + } + + return (KORE_RESULT_RETRY); + } + + callable = (PyObject *)addr; + + if (req->flags & HTTP_VALIDATOR_IS_REQUEST) { + if ((arg = pyhttp_request_alloc(data)) == NULL) + fatal("%s: pyreq failed", __func__); + + if ((args = PyTuple_New(1)) == NULL) + fatal("%s: PyTuple_New failed", __func__); + + if (PyTuple_SetItem(args, 0, arg) != 0) + fatal("%s: PyTuple_SetItem failed", __func__); + } else { + if ((pyreq = pyhttp_request_alloc(req)) == NULL) + fatal("%s: pyreq alloc failed", __func__); + + if ((arg = PyUnicode_FromString(data)) == NULL) + fatal("python_runtime_validator: PyUnicode failed"); + + if ((args = PyTuple_New(2)) == NULL) + fatal("%s: PyTuple_New failed", __func__); + + if (PyTuple_SetItem(args, 0, pyreq) != 0 || + PyTuple_SetItem(args, 1, arg) != 0) + fatal("%s: PyTuple_SetItem failed", __func__); + } + + PyErr_Clear(); + pyret = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + + if (pyret == NULL) { + kore_python_log_error("python_runtime_validator"); + fatal("failed to execute python call"); + } + + if (PyCoro_CheckExact(pyret)) { + coro = python_coro_create(pyret, req); + req->py_coro = coro; + if (python_coro_run(coro) == KORE_RESULT_OK) { + http_request_wakeup(req); + ret = python_validator_check(coro->result); + kore_python_coro_delete(coro); + req->py_coro = NULL; + return (ret); + } + return (KORE_RESULT_RETRY); + } + + ret = python_validator_check(pyret); + Py_DECREF(pyret); + + return (ret); +} + +static int +python_validator_check(PyObject *obj) +{ + int ret; + + if (obj == NULL) + return (KORE_RESULT_ERROR); + + if (!PyLong_Check(obj)) { + kore_log(LOG_WARNING, + "invalid return value from authenticator (not an int)"); + ret = KORE_RESULT_ERROR; + } else { + ret = (int)PyLong_AsLong(obj); + } + + switch (ret) { + case KORE_RESULT_OK: + case KORE_RESULT_ERROR: + break; + default: + kore_log(LOG_WARNING, + "unsupported authenticator return value '%d'", ret); + ret = KORE_RESULT_ERROR; + break; + } + + return (ret); +} + +static void +python_runtime_wsmessage(void *addr, struct connection *c, u_int8_t op, + const void *data, size_t len) +{ + PyObject *callable, *args, *pyret, *pyc, *pyop, *pydata; + + callable = (PyObject *)addr; + + if ((pyc = pyconnection_alloc(c)) == NULL) + fatal("python_runtime_wsmessage: pyc alloc failed"); + + if ((pyop = PyLong_FromLong((long)op)) == NULL) + fatal("python_runtime_wsmessage: PyLong_FromLong failed"); + + switch (op) { + case WEBSOCKET_OP_TEXT: + if ((pydata = PyUnicode_FromStringAndSize(data, len)) == NULL) + fatal("wsmessage: PyUnicode_AsUTF8AndSize failed"); + break; + case WEBSOCKET_OP_BINARY: + if ((pydata = PyBytes_FromStringAndSize(data, len)) == NULL) + fatal("wsmessage: PyBytes_FromString failed"); + break; + default: + fatal("python_runtime_wsmessage: invalid op"); + } + + if ((args = PyTuple_New(3)) == NULL) + fatal("python_runtime_wsmessage: PyTuple_New failed"); + + if (PyTuple_SetItem(args, 0, pyc) != 0 || + PyTuple_SetItem(args, 1, pyop) != 0 || + PyTuple_SetItem(args, 2, pydata) != 0) + fatal("python_runtime_wsmessage: PyTuple_SetItem failed"); + + PyErr_Clear(); + pyret = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + + if (pyret == NULL) { + kore_python_log_error("python_runtime_wsconnect"); + fatal("failed to execute python call"); + } + + Py_DECREF(pyret); +} + +static void +python_runtime_execute(void *addr) +{ + PyObject *callable, *args, *pyret; + + callable = (PyObject *)addr; + + if ((args = PyTuple_New(0)) == NULL) + fatal("python_runtime_execute: PyTuple_New failed"); + + PyErr_Clear(); + pyret = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + + if (pyret == NULL) { + kore_python_log_error("python_runtime_execute"); + fatal("failed to execute python call"); + } + + Py_DECREF(pyret); +} + +static void +python_runtime_configure(void *addr, int argc, char **argv) +{ + int i; + PyObject *callable, *args, *pyret, *pyarg, *list; + + callable = (PyObject *)addr; + + if ((args = PyTuple_New(1)) == NULL) + fatal("python_runtime_configure: PyTuple_New failed"); + + if ((list = PyList_New(argc + 1)) == NULL) + fatal("python_runtime_configure: PyList_New failed"); + + if ((pyarg = PyUnicode_FromString(__progname)) == NULL) + fatal("python_runtime_configure: PyUnicode_FromString"); + + if (PyList_SetItem(list, 0, pyarg) == -1) + fatal("python_runtime_configure: PyList_SetItem"); + + for (i = 0; i < argc; i++) { + if ((pyarg = PyUnicode_FromString(argv[i])) == NULL) + fatal("python_runtime_configure: PyUnicode_FromString"); + + if (PyList_SetItem(list, i + 1, pyarg) == -1) + fatal("python_runtime_configure: PyList_SetItem"); + } + + if (PyTuple_SetItem(args, 0, list) != 0) + fatal("python_runtime_configure: PyTuple_SetItem"); + + PyErr_Clear(); + pyret = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + Py_DECREF(list); + + if (pyret == NULL) { + kore_python_log_error("python_runtime_configure"); + fatal("failed to call configure method: wrong args?"); + } + + Py_DECREF(pyret); +} + +static int +python_runtime_onload(void *addr, int action) +{ + int ret; + PyObject *pyret, *args, *pyact, *callable; + + callable = (PyObject *)addr; + + if ((pyact = PyLong_FromLong(action)) == NULL) + fatal("python_runtime_onload: PyLong_FromLong failed"); + + if ((args = PyTuple_New(1)) == NULL) + fatal("python_runtime_onload: PyTuple_New failed"); + + if (PyTuple_SetItem(args, 0, pyact) != 0) + fatal("python_runtime_onload: PyTuple_SetItem failed"); + + PyErr_Clear(); + pyret = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + + if (pyret == NULL) { + kore_python_log_error("python_runtime_onload"); + return (KORE_RESULT_ERROR); + } + + if (!PyLong_Check(pyret)) + fatal("python_runtime_onload: unexpected return type"); + + ret = (int)PyLong_AsLong(pyret); + Py_DECREF(pyret); + + return (ret); +} + +static void +python_runtime_connect(void *addr, struct connection *c) +{ + PyObject *pyc, *pyret, *args, *callable; + + callable = (PyObject *)addr; + + if ((pyc = pyconnection_alloc(c)) == NULL) + fatal("python_runtime_connect: pyc alloc failed"); + + if ((args = PyTuple_New(1)) == NULL) + fatal("python_runtime_connect: PyTuple_New failed"); + + if (PyTuple_SetItem(args, 0, pyc) != 0) + fatal("python_runtime_connect: PyTuple_SetItem failed"); + + PyErr_Clear(); + pyret = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + + if (pyret == NULL) { + kore_python_log_error("python_runtime_connect"); + kore_connection_disconnect(c); + } + + Py_DECREF(pyret); +} + +static PyMODINIT_FUNC +python_module_init(void) +{ + int i; + PyObject *pykore; + + if ((pykore = PyModule_Create(&pykore_module)) == NULL) + fatal("python_module_init: failed to setup pykore module"); + + python_push_type("pyproc", pykore, &pyproc_type); + python_push_type("pylock", pykore, &pylock_type); + python_push_type("pytimer", pykore, &pytimer_type); + python_push_type("pyqueue", pykore, &pyqueue_type); + python_push_type("pysocket", pykore, &pysocket_type); + python_push_type("pyconnection", pykore, &pyconnection_type); + +#if defined(KORE_USE_CURL) + python_push_type("pyhttpclient", pykore, &pyhttp_client_type); +#endif + + python_push_type("pyhttp_file", pykore, &pyhttp_file_type); + python_push_type("pyhttp_request", pykore, &pyhttp_request_type); + + for (i = 0; python_integers[i].symbol != NULL; i++) { + python_push_integer(pykore, python_integers[i].symbol, + python_integers[i].value); + } + + return (pykore); +} + +static void +python_append_path(const char *path) +{ + PyObject *mpath, *spath; + + if ((mpath = PyUnicode_FromString(path)) == NULL) + fatal("python_append_path: PyUnicode_FromString failed"); + + if ((spath = PySys_GetObject("path")) == NULL) + fatal("python_append_path: PySys_GetObject failed"); + + PyList_Append(spath, mpath); + Py_DECREF(mpath); +} + +static void +python_push_type(const char *name, PyObject *module, PyTypeObject *type) +{ + if (PyType_Ready(type) == -1) + fatal("python_push_type: failed to ready %s", name); + + Py_INCREF(type); + + if (PyModule_AddObject(module, name, (PyObject *)type) == -1) + fatal("python_push_type: failed to push %s", name); +} + +static void +python_push_integer(PyObject *module, const char *name, long value) +{ + int ret; + + if ((ret = PyModule_AddIntConstant(module, name, value)) == -1) + fatal("python_push_integer: failed to add %s", name); +} + +#if defined(KORE_USE_PGSQL) +static PyObject * +python_kore_pgsql_register(PyObject *self, PyObject *args) +{ + const char *db, *conninfo; + + if (!PyArg_ParseTuple(args, "ss", &db, &conninfo)) + return (NULL); + + (void)kore_pgsql_register(db, conninfo); + + Py_RETURN_TRUE; +} +#endif + +static PyObject * +python_kore_log(PyObject *self, PyObject *args) +{ + int prio; + const char *message; + + if (!PyArg_ParseTuple(args, "is", &prio, &message)) + return (NULL); + + kore_log(prio, "%s", message); + + Py_RETURN_TRUE; +} + +static PyObject * +python_kore_time(PyObject *self, PyObject *args) +{ + u_int64_t now; + + now = kore_time_ms(); + + return (PyLong_FromUnsignedLongLong(now)); +} + +static PyObject * +python_kore_bind(PyObject *self, PyObject *args) +{ + const char *ip, *port; + + if (!PyArg_ParseTuple(args, "ss", &ip, &port)) + return (NULL); + + if (!kore_server_bind(ip, port, NULL)) { + PyErr_SetString(PyExc_RuntimeError, "failed to listen"); + return (NULL); + } + + Py_RETURN_TRUE; +} + +static PyObject * +python_kore_bind_unix(PyObject *self, PyObject *args) +{ + const char *path; + + if (!PyArg_ParseTuple(args, "s", &path)) + return (NULL); + + if (!kore_server_bind_unix(path, NULL)) { + PyErr_SetString(PyExc_RuntimeError, "failed bind to path"); + return (NULL); + } + + Py_RETURN_TRUE; +} + +static PyObject * +python_kore_task_create(PyObject *self, PyObject *args) +{ + PyObject *obj; + + if (!PyArg_ParseTuple(args, "O", &obj)) + return (NULL); + + if (!PyCoro_CheckExact(obj)) + fatal("%s: object is not a coroutine", __func__); + + python_coro_create(obj, NULL); + Py_INCREF(obj); + + Py_RETURN_NONE; +} + +static PyObject * +python_kore_socket_wrap(PyObject *self, PyObject *args) +{ + struct pysocket *sock; + PyObject *pysock, *pyfd, *pyfam, *pyproto; + + sock = NULL; + pyfd = NULL; + pyfam = NULL; + pyproto = NULL; + + if (!PyArg_ParseTuple(args, "O", &pysock)) + return (NULL); + + if ((pyfd = PyObject_CallMethod(pysock, "fileno", NULL)) == NULL) + return (NULL); + + if ((pyfam = PyObject_GetAttrString(pysock, "family")) == NULL) + goto out; + + if ((pyproto = PyObject_GetAttrString(pysock, "proto")) == NULL) + goto out; + + if ((sock = pysocket_alloc()) == NULL) + goto out; + + sock->socket = pysock; + Py_INCREF(sock->socket); + + sock->fd = (int)PyLong_AsLong(pyfd); + sock->family = (int)PyLong_AsLong(pyfam); + sock->protocol = (int)PyLong_AsLong(pyproto); + + memset(&sock->addr, 0, sizeof(sock->addr)); + + switch (sock->family) { + case AF_INET: + case AF_UNIX: + break; + default: + PyErr_SetString(PyExc_RuntimeError, "unsupported family"); + Py_DECREF((PyObject *)sock); + sock = NULL; + goto out; + } + +out: + Py_XDECREF(pyfd); + Py_XDECREF(pyfam); + Py_XDECREF(pyproto); + + return ((PyObject *)sock); +} + +static PyObject * +python_kore_queue(PyObject *self, PyObject *args) +{ + struct pyqueue *queue; + + if ((queue = PyObject_New(struct pyqueue, &pyqueue_type)) == NULL) + return (NULL); + + TAILQ_INIT(&queue->objects); + TAILQ_INIT(&queue->waiting); + + return ((PyObject *)queue); +} + +static PyObject * +python_kore_tracer(PyObject *self, PyObject *args) +{ + PyObject *obj; + + if (python_tracer != NULL) { + PyErr_SetString(PyExc_RuntimeError, "tracer already set"); + return (NULL); + } + + if (!PyArg_ParseTuple(args, "O", &obj)) + return (NULL); + + if (!PyCallable_Check(obj)) { + PyErr_SetString(PyExc_RuntimeError, "object not callable"); + Py_DECREF(obj); + return (NULL); + } + + Py_INCREF(obj); + python_tracer = obj; + + Py_RETURN_TRUE; +} + +static PyObject * +python_kore_gather(PyObject *self, PyObject *args, PyObject *kwargs) +{ + struct pygather_op *op; + PyObject *obj; + struct pygather_coro *coro; + Py_ssize_t sz, idx; + int concurrency; + + if (coro_running == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "kore.gather only available in coroutines"); + return (NULL); + } + + sz = PyTuple_Size(args); + + if (sz > INT_MAX) { + PyErr_SetString(PyExc_TypeError, "too many arguments"); + return (NULL); + } + + if (kwargs != NULL && + (obj = PyDict_GetItemString(kwargs, "concurrency")) != NULL) { + if (!PyLong_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "concurrency level must be an integer"); + return (NULL); + } + + PyErr_Clear(); + concurrency = (int)PyLong_AsLong(obj); + if (concurrency == -1 && PyErr_Occurred()) + return (NULL); + + if (concurrency == 0) + concurrency = sz; + } else { + concurrency = sz; + } + + op = PyObject_New(struct pygather_op, &pygather_op_type); + if (op == NULL) + return (NULL); + + op->running = 0; + op->count = (int)sz; + op->coro = coro_running; + op->concurrency = concurrency; + + TAILQ_INIT(&op->results); + TAILQ_INIT(&op->coroutines); + + for (idx = 0; idx < sz; idx++) { + if ((obj = PyTuple_GetItem(args, idx)) == NULL) { + Py_DECREF((PyObject *)op); + return (NULL); + } + + if (!PyCoro_CheckExact(obj)) { + Py_DECREF((PyObject *)op); + PyErr_SetString(PyExc_TypeError, "not a coroutine"); + return (NULL); + } + + Py_INCREF(obj); + + coro = kore_pool_get(&gather_coro_pool); + coro->coro = python_coro_create(obj, NULL); + coro->coro->gatherop = op; + TAILQ_INSERT_TAIL(&op->coroutines, coro, list); + + if (idx > concurrency - 1) + python_coro_suspend(coro->coro); + else + op->running++; + } + + return ((PyObject *)op); +} + +static PyObject * +python_kore_lock(PyObject *self, PyObject *args) +{ + struct pylock *lock; + + if ((lock = PyObject_New(struct pylock, &pylock_type)) == NULL) + return (NULL); + + lock->owner = NULL; + TAILQ_INIT(&lock->ops); + + return ((PyObject *)lock); +} + +static PyObject * +python_kore_fatal(PyObject *self, PyObject *args) +{ + const char *reason; + + if (!PyArg_ParseTuple(args, "s", &reason)) + reason = "python_kore_fatal: PyArg_ParseTuple failed"; + + fatal("%s", reason); + + /* not reached */ + Py_RETURN_TRUE; +} + +static PyObject * +python_kore_fatalx(PyObject *self, PyObject *args) +{ + const char *reason; + + if (!PyArg_ParseTuple(args, "s", &reason)) + reason = "python_kore_fatalx: PyArg_ParseTuple failed"; + + fatalx("%s", reason); + + /* not reached */ + Py_RETURN_TRUE; +} + +static PyObject * +python_kore_suspend(PyObject *self, PyObject *args) +{ + struct pysuspend_op *op; + int delay; + + if (!PyArg_ParseTuple(args, "i", &delay)) + return (NULL); + + op = PyObject_New(struct pysuspend_op, &pysuspend_op_type); + if (op == NULL) + return (NULL); + + op->timer = NULL; + op->delay = delay; + op->coro = coro_running; + op->state = PYSUSPEND_OP_INIT; + + return ((PyObject *)op); +} + +static PyObject * +python_kore_shutdown(PyObject *self, PyObject *args) +{ + kore_shutdown(); + + Py_RETURN_TRUE; +} + +static PyObject * +python_kore_timer(PyObject *self, PyObject *args) +{ + u_int64_t ms; + PyObject *obj; + int flags; + struct pytimer *timer; + + if (!PyArg_ParseTuple(args, "OKi", &obj, &ms, &flags)) + return (NULL); + + if (flags & ~(KORE_TIMER_FLAGS)) { + PyErr_SetString(PyExc_RuntimeError, "invalid flags"); + return (NULL); + } + + if ((timer = PyObject_New(struct pytimer, &pytimer_type)) == NULL) + return (NULL); + + timer->flags = flags; + timer->callable = obj; + timer->run = kore_timer_add(pytimer_run, ms, timer, flags); + + Py_INCREF((PyObject *)timer); + Py_INCREF(timer->callable); + + return ((PyObject *)timer); +} + +static PyObject * +python_kore_proc(PyObject *self, PyObject *args) +{ + const char *cmd; + struct pyproc *proc; + char *copy, *argv[32]; + int timeo, in_pipe[2], out_pipe[2]; + + timeo = -1; + + if (coro_running == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "kore.proc only available in coroutines"); + return (NULL); + } + + if (!PyArg_ParseTuple(args, "s|i", &cmd, &timeo)) + return (NULL); + + if (pipe(in_pipe) == -1) { + PyErr_SetString(PyExc_RuntimeError, errno_s); + return (NULL); + } + + if (pipe(out_pipe) == -1) { + close(in_pipe[0]); + close(in_pipe[1]); + PyErr_SetString(PyExc_RuntimeError, errno_s); + return (NULL); + } + + if ((proc = PyObject_New(struct pyproc, &pyproc_type)) == NULL) { + close(in_pipe[0]); + close(in_pipe[1]); + close(out_pipe[0]); + close(out_pipe[1]); + return (NULL); + } + + proc->pid = -1; + proc->reaped = 0; + proc->status = 0; + proc->timer = NULL; + proc->coro = coro_running; + proc->in = pysocket_alloc(); + proc->out = pysocket_alloc(); + + if (proc->in == NULL || proc->out == NULL) { + Py_DECREF((PyObject *)proc); + return (NULL); + } + + TAILQ_INSERT_TAIL(&procs, proc, list); + + proc->pid = fork(); + if (proc->pid == -1) { + if (errno == ENOSYS) { + Py_DECREF((PyObject *)proc); + PyErr_SetString(PyExc_RuntimeError, errno_s); + return (NULL); + } + fatal("python_kore_proc: fork(): %s", errno_s); + } + + if (proc->pid == 0) { + close(in_pipe[1]); + close(out_pipe[0]); + + if (dup2(out_pipe[1], STDOUT_FILENO) == -1 || + dup2(out_pipe[1], STDERR_FILENO) == -1 || + dup2(in_pipe[0], STDIN_FILENO) == -1) + fatal("dup2: %s", errno_s); + + copy = kore_strdup(cmd); + python_split_arguments(copy, argv, 32); + (void)execve(argv[0], argv, NULL); + kore_log(LOG_ERR, "kore.proc failed to execute %s (%s)", + argv[0], errno_s); + exit(1); + } + + close(in_pipe[0]); + close(out_pipe[1]); + + if (!kore_connection_nonblock(in_pipe[1], 0) || + !kore_connection_nonblock(out_pipe[0], 0)) + fatal("failed to mark kore.proc pipes are non-blocking"); + + proc->in->fd = in_pipe[1]; + proc->out->fd = out_pipe[0]; + + if (timeo != -1) { + proc->timer = kore_timer_add(pyproc_timeout, + timeo, proc, KORE_TIMER_ONESHOT); + } + + return ((PyObject *)proc); +} + +static PyObject * +python_import(const char *path) +{ + struct stat st; + PyObject *module; + char *dir, *file, *copy, *p; + + if (stat(path, &st) == -1) + fatal("python_import: stat(%s): %s", path, errno_s); + + if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) + fatal("python_import: '%s' is not a file or directory", path); + + copy = kore_strdup(path); + + if ((file = basename(copy)) == NULL) + fatal("basename: %s: %s", path, errno_s); + if ((dir = dirname(copy)) == NULL) + fatal("dirname: %s: %s", path, errno_s); + + if ((p = strrchr(file, '.')) != NULL) + *p = '\0'; + + python_append_path(dir); + + if (S_ISDIR(st.st_mode)) + python_append_path(path); + + module = PyImport_ImportModule(file); + if (module == NULL) + PyErr_Print(); + + kore_free(copy); + + return (module); +} + +static PyObject * +python_callable(PyObject *module, const char *symbol) +{ + char *base, *method; + PyObject *res, *obj, *meth; + + res = NULL; + obj = NULL; + base = kore_strdup(symbol); + + if ((method = strchr(base, '.')) != NULL) + *(method)++ = '\0'; + + if ((obj = PyObject_GetAttrString(module, base)) == NULL) + goto out; + + if (method != NULL) { + if ((meth = PyObject_GetAttrString(obj, method)) == NULL) + goto out; + + Py_DECREF(obj); + obj = meth; + } + + if (!PyCallable_Check(obj)) + goto out; + + res = obj; + obj = NULL; + +out: + if (obj != NULL) + Py_DECREF(obj); + + kore_free(base); + + return (res); +} + +static PyObject * +pyconnection_alloc(struct connection *c) +{ + struct pyconnection *pyc; + + pyc = PyObject_New(struct pyconnection, &pyconnection_type); + if (pyc == NULL) + return (NULL); + + pyc->c = c; + + return ((PyObject *)pyc); +} + +static PyObject * +pyconnection_disconnect(struct pyconnection *pyc, PyObject *args) +{ + kore_connection_disconnect(pyc->c); + + Py_RETURN_TRUE; +} + +static PyObject * +pyconnection_get_fd(struct pyconnection *pyc, void *closure) +{ + PyObject *fd; + + if ((fd = PyLong_FromLong(pyc->c->fd)) == NULL) + return (PyErr_NoMemory()); + + return (fd); +} + +static PyObject * +pyconnection_get_addr(struct pyconnection *pyc, void *closure) +{ + void *ptr; + PyObject *result; + char addr[INET6_ADDRSTRLEN]; + + switch (pyc->c->family) { + case AF_INET: + ptr = &pyc->c->addr.ipv4.sin_addr; + break; + case AF_INET6: + ptr = &pyc->c->addr.ipv6.sin6_addr; + break; + default: + PyErr_SetString(PyExc_RuntimeError, "invalid family"); + return (NULL); + } + + if (inet_ntop(pyc->c->family, ptr, addr, sizeof(addr)) == NULL) { + PyErr_SetString(PyExc_RuntimeError, "inet_ntop failed"); + return (NULL); + } + + if ((result = PyUnicode_FromString(addr)) == NULL) + return (PyErr_NoMemory()); + + return (result); +} + +#if !defined(KORE_NO_TLS) +static PyObject * +pyconnection_get_peer_x509(struct pyconnection *pyc, void *closure) +{ + int len; + PyObject *bytes; + u_int8_t *der, *pp; + + if (pyc->c->cert == NULL) { + Py_RETURN_NONE; + } + + if ((len = i2d_X509(pyc->c->cert, NULL)) <= 0) { + PyErr_SetString(PyExc_RuntimeError, "i2d_X509 failed"); + return (NULL); + } + + der = kore_calloc(1, len); + pp = der; + + if (i2d_X509(pyc->c->cert, &pp) <= 0) { + kore_free(der); + PyErr_SetString(PyExc_RuntimeError, "i2d_X509 failed"); + return (NULL); + } + + bytes = PyBytes_FromStringAndSize((char *)der, len); + kore_free(der); + + return (bytes); +} +#endif + +static void +pytimer_run(void *arg, u_int64_t now) +{ + PyObject *ret; + struct pytimer *timer = arg; + + PyErr_Clear(); + ret = PyObject_CallObject(timer->callable, NULL); + Py_XDECREF(ret); + + kore_python_log_error("pytimer_run"); + + if (timer->flags & KORE_TIMER_ONESHOT) { + timer->run = NULL; + Py_DECREF((PyObject *)timer); + } +} + +static void +pytimer_dealloc(struct pytimer *timer) +{ + if (timer->run != NULL) { + kore_timer_remove(timer->run); + timer->run = NULL; + } + + if (timer->callable != NULL) { + Py_DECREF(timer->callable); + timer->callable = NULL; + } + + PyObject_Del((PyObject *)timer); +} + +static PyObject * +pytimer_close(struct pytimer *timer, PyObject *args) +{ + if (timer->run != NULL) { + kore_timer_remove(timer->run); + timer->run = NULL; + } + + if (timer->callable != NULL) { + Py_DECREF(timer->callable); + timer->callable = NULL; + } + + Py_INCREF((PyObject *)timer); + Py_RETURN_TRUE; +} + +static void +pysuspend_op_dealloc(struct pysuspend_op *op) +{ + if (op->timer != NULL) { + kore_timer_remove(op->timer); + op->timer = NULL; + } + + PyObject_Del((PyObject *)op); +} + +static PyObject * +pysuspend_op_await(PyObject *sop) +{ + Py_INCREF(sop); + return (sop); +} + +static PyObject * +pysuspend_op_iternext(struct pysuspend_op *op) +{ + switch (op->state) { + case PYSUSPEND_OP_INIT: + op->timer = kore_timer_add(pysuspend_wakeup, op->delay, + op, KORE_TIMER_ONESHOT); + op->state = PYSUSPEND_OP_WAIT; + break; + case PYSUSPEND_OP_WAIT: + break; + case PYSUSPEND_OP_CONTINUE: + PyErr_SetNone(PyExc_StopIteration); + return (NULL); + default: + fatal("unknown state %d for pysuspend_op", op->state); + } + + Py_RETURN_NONE; +} + +static void +pysuspend_wakeup(void *arg, u_int64_t now) +{ + struct pysuspend_op *op = arg; + + op->timer = NULL; + op->state = PYSUSPEND_OP_CONTINUE; + + if (op->coro->request != NULL) + http_request_wakeup(op->coro->request); + else + python_coro_wakeup(op->coro); +} + +static struct pysocket * +pysocket_alloc(void) +{ + struct pysocket *sock; + + if ((sock = PyObject_New(struct pysocket, &pysocket_type)) == NULL) + return (NULL); + + sock->fd = -1; + sock->family = -1; + sock->protocol = -1; + sock->scheduled = 0; + + sock->socket = NULL; + sock->recvop = NULL; + sock->sendop = NULL; + + sock->event.s = sock; + sock->event.evt.flags = 0; + sock->event.evt.type = KORE_TYPE_PYSOCKET; + sock->event.evt.handle = pysocket_evt_handle; + + return (sock); +} + +static void +pysocket_dealloc(struct pysocket *sock) +{ + if (sock->scheduled && sock->fd != -1) { + kore_platform_disable_read(sock->fd); +#if !defined(__linux__) + kore_platform_disable_write(sock->fd); +#endif + } + + if (sock->socket != NULL) { + Py_DECREF(sock->socket); + } else if (sock->fd != -1) { + (void)close(sock->fd); + } + + PyObject_Del((PyObject *)sock); +} + +static PyObject * +pysocket_send(struct pysocket *sock, PyObject *args) +{ + Py_buffer buf; + + if (!PyArg_ParseTuple(args, "y*", &buf)) + return (NULL); + + return (pysocket_op_create(sock, PYSOCKET_TYPE_SEND, buf.buf, buf.len)); +} + +static PyObject * +pysocket_sendto(struct pysocket *sock, PyObject *args) +{ + Py_buffer buf; + struct pysocket_op *op; + PyObject *ret; + int port; + const char *ip, *sockaddr; + + switch (sock->family) { + case AF_INET: + if (!PyArg_ParseTuple(args, "siy*", &ip, &port, &buf)) + return (NULL); + if (port <= 0 || port >= USHRT_MAX) { + PyErr_SetString(PyExc_RuntimeError, "invalid port"); + return (NULL); + } + break; + case AF_UNIX: + if (!PyArg_ParseTuple(args, "sy*", &sockaddr, &buf)) + return (NULL); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "unsupported family"); + return (NULL); + } + + ret = pysocket_op_create(sock, PYSOCKET_TYPE_SENDTO, buf.buf, buf.len); + + op = (struct pysocket_op *)ret; + + switch (sock->family) { + case AF_INET: + op->sendaddr.ipv4.sin_family = AF_INET; + op->sendaddr.ipv4.sin_port = htons(port); + op->sendaddr.ipv4.sin_addr.s_addr = inet_addr(ip); + break; + case AF_UNIX: + op->sendaddr.sun.sun_family = AF_UNIX; + if (kore_strlcpy(op->sendaddr.sun.sun_path, sockaddr, + sizeof(op->sendaddr.sun.sun_path)) >= + sizeof(op->sendaddr.sun.sun_path)) { + Py_DECREF(ret); + PyErr_SetString(PyExc_RuntimeError, + "unix socket path too long"); + return (NULL); + } + break; + default: + Py_DECREF(ret); + PyErr_SetString(PyExc_RuntimeError, "unsupported family"); + return (NULL); + } + + return (ret); +} + +static PyObject * +pysocket_recv(struct pysocket *sock, PyObject *args) +{ + Py_ssize_t len; + struct pysocket_op *op; + PyObject *obj; + int timeo; + + timeo = -1; + + if (!PyArg_ParseTuple(args, "n|i", &len, &timeo)) + return (NULL); + + obj = pysocket_op_create(sock, PYSOCKET_TYPE_RECV, NULL, len); + if (obj == NULL) + return (NULL); + + op = (struct pysocket_op *)obj; + + if (timeo != -1) { + op->timer = kore_timer_add(pysocket_op_timeout, + timeo, op, KORE_TIMER_ONESHOT); + } + + return (obj); +} + +static PyObject * +pysocket_recvfrom(struct pysocket *sock, PyObject *args) +{ + Py_ssize_t len; + + if (!PyArg_ParseTuple(args, "n", &len)) + return (NULL); + + return (pysocket_op_create(sock, PYSOCKET_TYPE_RECVFROM, NULL, len)); +} + +static PyObject * +pysocket_accept(struct pysocket *sock, PyObject *args) +{ + return (pysocket_op_create(sock, PYSOCKET_TYPE_ACCEPT, NULL, 0)); +} + +static PyObject * +pysocket_connect(struct pysocket *sock, PyObject *args) +{ + const char *host; + int port, len; + + port = 0; + + if (!PyArg_ParseTuple(args, "s|i", &host, &port)) + return (NULL); + + if (port < 0 || port > USHRT_MAX) { + PyErr_SetString(PyExc_RuntimeError, "invalid port number"); + return (NULL); + } + + switch (sock->family) { + case AF_INET: + sock->addr.ipv4.sin_family = AF_INET; + sock->addr.ipv4.sin_port = htons(port); + if (inet_pton(sock->family, host, + &sock->addr.ipv4.sin_addr) == -1) { + PyErr_SetString(PyExc_RuntimeError, "invalid host"); + return (NULL); + } + sock->addr_len = sizeof(sock->addr.ipv4); + break; + case AF_UNIX: + sock->addr.sun.sun_family = AF_UNIX; + len = snprintf(sock->addr.sun.sun_path, + sizeof(sock->addr.sun.sun_path), "%s", host); + if (len == -1 || + (size_t)len >= sizeof(sock->addr.sun.sun_path)) { + PyErr_SetString(PyExc_RuntimeError, "path too long"); + return (NULL); + } +#if defined(__linux__) + /* Assume abstract socket if prefixed with '@'. */ + if (sock->addr.sun.sun_path[0] == '@') + sock->addr.sun.sun_path[0] = '\0'; +#endif + sock->addr_len = sizeof(sock->addr.sun.sun_family) + len; + break; + default: + fatal("unsupported socket family %d", sock->family); + } + + return (pysocket_op_create(sock, PYSOCKET_TYPE_CONNECT, NULL, 0)); +} + +static PyObject * +pysocket_close(struct pysocket *sock, PyObject *args) +{ + if (sock->scheduled) { + sock->scheduled = 0; + kore_platform_disable_read(sock->fd); +#if !defined(__linux__) + kore_platform_disable_write(sock->fd); +#endif + } + + if (sock->socket != NULL) { + Py_DECREF(sock->socket); + sock->socket = NULL; + } else if (sock->fd != -1) { + (void)close(sock->fd); + } + + sock->fd = -1; + + Py_RETURN_TRUE; +} + +static void +pysocket_op_dealloc(struct pysocket_op *op) +{ + if (op->type == PYSOCKET_TYPE_RECV || + op->type == PYSOCKET_TYPE_RECVFROM || + op->type == PYSOCKET_TYPE_SEND) + kore_buf_cleanup(&op->buffer); + + switch (op->type) { + case PYSOCKET_TYPE_RECV: + case PYSOCKET_TYPE_ACCEPT: + case PYSOCKET_TYPE_RECVFROM: + if (op->socket->recvop != op) + fatal("recvop mismatch"); + op->socket->recvop = NULL; + break; + case PYSOCKET_TYPE_SEND: + case PYSOCKET_TYPE_SENDTO: + case PYSOCKET_TYPE_CONNECT: + if (op->socket->sendop != op) + fatal("sendop mismatch"); + op->socket->sendop = NULL; + break; + } + + if (op->timer != NULL) { + kore_timer_remove(op->timer); + op->timer = NULL; + } + + op->coro->sockop = NULL; + Py_DECREF(op->socket); + + PyObject_Del((PyObject *)op); +} + +static PyObject * +pysocket_op_create(struct pysocket *sock, int type, const void *ptr, size_t len) +{ + struct pysocket_op *op; + + if (coro_running->sockop != NULL) + fatal("pysocket_op_create: coro has active socketop"); + + switch (type) { + case PYSOCKET_TYPE_RECV: + case PYSOCKET_TYPE_ACCEPT: + case PYSOCKET_TYPE_RECVFROM: + if (sock->recvop != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "only one recv operation can be done per socket"); + return (NULL); + } + break; + case PYSOCKET_TYPE_SEND: + case PYSOCKET_TYPE_SENDTO: + case PYSOCKET_TYPE_CONNECT: + if (sock->sendop != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "only one send operation can be done per socket"); + return (NULL); + } + break; + default: + fatal("unknown pysocket_op type %u", type); + } + + op = PyObject_New(struct pysocket_op, &pysocket_op_type); + if (op == NULL) + return (NULL); + + op->eof = 0; + op->self = op; + op->type = type; + op->timer = NULL; + op->socket = sock; + op->coro = coro_running; + + coro_running->sockop = op; + Py_INCREF(op->socket); + + switch (type) { + case PYSOCKET_TYPE_RECV: + case PYSOCKET_TYPE_RECVFROM: + sock->recvop = op; + kore_buf_init(&op->buffer, len); + break; + case PYSOCKET_TYPE_SEND: + case PYSOCKET_TYPE_SENDTO: + sock->sendop = op; + kore_buf_init(&op->buffer, len); + kore_buf_append(&op->buffer, ptr, len); + kore_buf_reset(&op->buffer); + break; + case PYSOCKET_TYPE_ACCEPT: + sock->recvop = op; + break; + case PYSOCKET_TYPE_CONNECT: + sock->sendop = op; + break; + default: + fatal("unknown pysocket_op type %u", type); + } + + if (sock->scheduled == 0) { + sock->scheduled = 1; + kore_platform_event_all(sock->fd, &sock->event); + } + + return ((PyObject *)op); +} + +static PyObject * +pysocket_op_await(PyObject *obj) +{ + Py_INCREF(obj); + return (obj); +} + +static PyObject * +pysocket_op_iternext(struct pysocket_op *op) +{ + PyObject *ret; + + if (op->socket->fd == -1) { + PyErr_SetNone(PyExc_StopIteration); + return (NULL); + } + + if (op->eof) { + if (op->coro->exception != NULL) { + PyErr_SetString(op->coro->exception, + op->coro->exception_msg); + op->coro->exception = NULL; + return (NULL); + } + + if (op->type != PYSOCKET_TYPE_RECV) { + PyErr_SetString(PyExc_RuntimeError, "socket EOF"); + return (NULL); + } + + /* Drain the recv socket. */ + op->socket->event.evt.flags |= KORE_EVENT_READ; + return (pysocket_async_recv(op)); + } + + switch (op->type) { + case PYSOCKET_TYPE_CONNECT: + ret = pysocket_async_connect(op); + break; + case PYSOCKET_TYPE_ACCEPT: + ret = pysocket_async_accept(op); + break; + case PYSOCKET_TYPE_RECV: + case PYSOCKET_TYPE_RECVFROM: + ret = pysocket_async_recv(op); + break; + case PYSOCKET_TYPE_SEND: + case PYSOCKET_TYPE_SENDTO: + ret = pysocket_async_send(op); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "invalid op type"); + return (NULL); + } + + return (ret); +} + +static void +pysocket_op_timeout(void *arg, u_int64_t now) +{ + struct pysocket_op *op = arg; + + op->eof = 1; + op->timer = NULL; + + op->coro->exception = PyExc_TimeoutError; + op->coro->exception_msg = "timeout before operation completed"; + + if (op->coro->request != NULL) + http_request_wakeup(op->coro->request); + else + python_coro_wakeup(op->coro); +} + +static PyObject * +pysocket_async_connect(struct pysocket_op *op) +{ + if (connect(op->socket->fd, (struct sockaddr *)&op->socket->addr, + op->socket->addr_len) == -1) { + if (errno != EALREADY && errno != EINPROGRESS && + errno != EISCONN && errno != EAGAIN) { + PyErr_SetString(PyExc_RuntimeError, errno_s); + return (NULL); + } + + if (errno != EISCONN) { + Py_RETURN_NONE; + } + } + + PyErr_SetNone(PyExc_StopIteration); + return (NULL); +} + +static PyObject * +pysocket_async_accept(struct pysocket_op *op) +{ + int fd; + struct pysocket *sock; + + if (!(op->socket->event.evt.flags & KORE_EVENT_READ)) { + Py_RETURN_NONE; + } + + if ((sock = pysocket_alloc()) == NULL) + return (NULL); + + sock->addr_len = sizeof(sock->addr); + + if ((fd = accept(op->socket->fd, + (struct sockaddr *)&sock->addr, &sock->addr_len)) == -1) { + Py_DECREF((PyObject *)sock); + if (errno == EAGAIN || errno == EWOULDBLOCK) { + op->socket->event.evt.flags &= ~KORE_EVENT_READ; + Py_RETURN_NONE; + } + PyErr_SetString(PyExc_RuntimeError, errno_s); + return (NULL); + } + + if (!kore_connection_nonblock(fd, 0)) { + Py_DECREF((PyObject *)sock); + PyErr_SetString(PyExc_RuntimeError, errno_s); + return (NULL); + } + + sock->fd = fd; + sock->socket = NULL; + sock->family = op->socket->family; + sock->protocol = op->socket->protocol; + + PyErr_SetObject(PyExc_StopIteration, (PyObject *)sock); + Py_DECREF((PyObject *)sock); + + return (NULL); +} + +static PyObject * +pysocket_async_recv(struct pysocket_op *op) +{ + ssize_t ret; + size_t len; + u_int16_t port; + socklen_t socklen; + struct sockaddr *sendaddr; + const char *ptr, *ip; + PyObject *bytes, *result, *tuple; + + if (!(op->socket->event.evt.flags & KORE_EVENT_READ)) { + Py_RETURN_NONE; + } + + for (;;) { + if (op->type == PYSOCKET_TYPE_RECV) { + ret = read(op->socket->fd, op->buffer.data, + op->buffer.length); + } else { + sendaddr = (struct sockaddr *)&op->sendaddr; + switch (op->socket->family) { + case AF_INET: + socklen = sizeof(op->sendaddr.ipv4); + break; + case AF_UNIX: + socklen = sizeof(op->sendaddr.sun); + break; + default: + fatal("non AF_INET/AF_UNIX in %s", __func__); + } + + memset(sendaddr, 0, socklen); + ret = recvfrom(op->socket->fd, op->buffer.data, + op->buffer.length, 0, sendaddr, &socklen); + } + + if (ret == -1) { + if (errno == EINTR) + continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) { + op->socket->event.evt.flags &= ~KORE_EVENT_READ; + Py_RETURN_NONE; + } + PyErr_SetString(PyExc_RuntimeError, errno_s); + return (NULL); + } + + break; + } + + op->coro->exception = NULL; + op->coro->exception_msg = NULL; + + if (op->timer != NULL) { + kore_timer_remove(op->timer); + op->timer = NULL; + } + + if (op->type == PYSOCKET_TYPE_RECV && ret == 0) { + PyErr_SetNone(PyExc_StopIteration); + return (NULL); + } + + ptr = (const char *)op->buffer.data; + if ((bytes = PyBytes_FromStringAndSize(ptr, ret)) == NULL) + return (NULL); + + if (op->type == PYSOCKET_TYPE_RECV) { + PyErr_SetObject(PyExc_StopIteration, bytes); + Py_DECREF(bytes); + return (NULL); + } + + switch(op->socket->family) { + case AF_INET: + port = ntohs(op->sendaddr.ipv4.sin_port); + ip = inet_ntoa(op->sendaddr.ipv4.sin_addr); + + if ((tuple = Py_BuildValue("(sHN)", ip, port, bytes)) == NULL) + return (NULL); + break; + case AF_UNIX: + len = strlen(op->sendaddr.sun.sun_path); +#if defined(__linux__) + if (len == 0 && socklen > 0) { + len = socklen - sizeof(sa_family_t); + op->sendaddr.sun.sun_path[0] = '@'; + op->sendaddr.sun.sun_path[len] = '\0'; + } +#endif + if (len == 0) { + if ((tuple = Py_BuildValue("(ON)", + Py_None, bytes)) == NULL) + return (NULL); + break; + } + + if ((tuple = Py_BuildValue("(sN)", + op->sendaddr.sun.sun_path, bytes)) == NULL) + return (NULL); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Unsupported family"); + return (NULL); + } + + result = PyObject_CallFunctionObjArgs(PyExc_StopIteration, tuple, NULL); + if (result == NULL) { + Py_DECREF(tuple); + return (NULL); + } + + Py_DECREF(tuple); + PyErr_SetObject(PyExc_StopIteration, result); + Py_DECREF(result); + + return (NULL); +} + +static PyObject * +pysocket_async_send(struct pysocket_op *op) +{ + ssize_t ret; + socklen_t socklen; + const struct sockaddr *sendaddr; + + if (!(op->socket->event.evt.flags & KORE_EVENT_WRITE)) { + Py_RETURN_NONE; + } + + for (;;) { + if (op->type == PYSOCKET_TYPE_SEND) { + ret = write(op->socket->fd, + op->buffer.data + op->buffer.offset, + op->buffer.length - op->buffer.offset); + } else { + sendaddr = (const struct sockaddr *)&op->sendaddr; + + switch (op->socket->family) { + case AF_INET: + socklen = sizeof(op->sendaddr.ipv4); + break; + case AF_UNIX: + socklen = sizeof(op->sendaddr.sun); +#if defined(__linux__) + if (op->sendaddr.sun.sun_path[0] == '@') { + socklen = sizeof(sa_family_t) + + strlen(op->sendaddr.sun.sun_path); + op->sendaddr.sun.sun_path[0] = '\0'; + } +#endif + break; + default: + fatal("non AF_INET/AF_UNIX in %s", __func__); + } + + ret = sendto(op->socket->fd, + op->buffer.data + op->buffer.offset, + op->buffer.length - op->buffer.offset, + 0, sendaddr, socklen); + } + + if (ret == -1) { + if (errno == EINTR) + continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) { + op->socket->event.evt.flags &= + ~KORE_EVENT_WRITE; + Py_RETURN_NONE; + } + PyErr_SetString(PyExc_RuntimeError, errno_s); + return (NULL); + } + break; + } + + op->buffer.offset += (size_t)ret; + + if (op->buffer.offset == op->buffer.length) { + PyErr_SetNone(PyExc_StopIteration); + return (NULL); + } + + Py_RETURN_NONE; +} + +static void +pysocket_evt_handle(void *arg, int eof) +{ + struct pysocket_event *event = arg; + struct pysocket *socket = event->s; + + if ((eof || (event->evt.flags & KORE_EVENT_READ)) && + socket->recvop != NULL) { + if (socket->recvop->coro->request != NULL) + http_request_wakeup(socket->recvop->coro->request); + else + python_coro_wakeup(socket->recvop->coro); + socket->recvop->eof = eof; + } + + if ((eof || (event->evt.flags & KORE_EVENT_WRITE)) && + socket->sendop != NULL) { + if (socket->sendop->coro->request != NULL) + http_request_wakeup(socket->sendop->coro->request); + else + python_coro_wakeup(socket->sendop->coro); + socket->sendop->eof = eof; + } +} + +static void +pyqueue_dealloc(struct pyqueue *queue) +{ + struct pyqueue_object *object; + struct pyqueue_waiting *waiting; + + while ((object = TAILQ_FIRST(&queue->objects)) != NULL) { + TAILQ_REMOVE(&queue->objects, object, list); + Py_DECREF(object->obj); + kore_pool_put(&queue_object_pool, object); + } + + while ((waiting = TAILQ_FIRST(&queue->waiting)) != NULL) { + TAILQ_REMOVE(&queue->waiting, waiting, list); + if (waiting->op != NULL) + waiting->op->waiting = NULL; + kore_pool_put(&queue_wait_pool, waiting); + } + + PyObject_Del((PyObject *)queue); +} + +static PyObject * +pyqueue_pop(struct pyqueue *queue, PyObject *args) +{ + struct pyqueue_op *op; + + if ((op = PyObject_New(struct pyqueue_op, &pyqueue_op_type)) == NULL) + return (NULL); + + op->queue = queue; + op->waiting = kore_pool_get(&queue_wait_pool); + op->waiting->op = op; + + op->waiting->coro = coro_running; + TAILQ_INSERT_TAIL(&queue->waiting, op->waiting, list); + + Py_INCREF((PyObject *)queue); + + return ((PyObject *)op); +} + +static PyObject * +pyqueue_popnow(struct pyqueue *queue, PyObject *args) +{ + PyObject *obj; + struct pyqueue_object *object; + + if ((object = TAILQ_FIRST(&queue->objects)) == NULL) { + Py_RETURN_NONE; + } + + TAILQ_REMOVE(&queue->objects, object, list); + + obj = object->obj; + kore_pool_put(&queue_object_pool, object); + + return (obj); +} + +static PyObject * +pyqueue_push(struct pyqueue *queue, PyObject *args) +{ + PyObject *obj; + struct pyqueue_object *object; + struct pyqueue_waiting *waiting; + + if (!PyArg_ParseTuple(args, "O", &obj)) + return (NULL); + + Py_INCREF(obj); + + object = kore_pool_get(&queue_object_pool); + object->obj = obj; + + TAILQ_INSERT_TAIL(&queue->objects, object, list); + + /* Wakeup first in line if any. */ + if ((waiting = TAILQ_FIRST(&queue->waiting)) != NULL) { + TAILQ_REMOVE(&queue->waiting, waiting, list); + + /* wakeup HTTP request if one is tied. */ + if (waiting->coro->request != NULL) + http_request_wakeup(waiting->coro->request); + else + python_coro_wakeup(waiting->coro); + + waiting->op->waiting = NULL; + kore_pool_put(&queue_wait_pool, waiting); + } + + Py_RETURN_TRUE; +} + +static void +pyqueue_op_dealloc(struct pyqueue_op *op) +{ + if (op->waiting != NULL) { + TAILQ_REMOVE(&op->queue->waiting, op->waiting, list); + kore_pool_put(&queue_wait_pool, op->waiting); + op->waiting = NULL; + } + + Py_DECREF((PyObject *)op->queue); + PyObject_Del((PyObject *)op); +} + +static PyObject * +pyqueue_op_await(PyObject *obj) +{ + Py_INCREF(obj); + return (obj); +} + +static PyObject * +pyqueue_op_iternext(struct pyqueue_op *op) +{ + PyObject *obj; + struct pyqueue_object *object; + struct pyqueue_waiting *waiting; + + if ((object = TAILQ_FIRST(&op->queue->objects)) == NULL) { + Py_RETURN_NONE; + } + + TAILQ_REMOVE(&op->queue->objects, object, list); + + obj = object->obj; + kore_pool_put(&queue_object_pool, object); + + TAILQ_FOREACH(waiting, &op->queue->waiting, list) { + if (waiting->coro->id == coro_running->id) { + TAILQ_REMOVE(&op->queue->waiting, waiting, list); + waiting->op->waiting = NULL; + kore_pool_put(&queue_wait_pool, waiting); + break; + } + } + + PyErr_SetObject(PyExc_StopIteration, obj); + Py_DECREF(obj); + + return (NULL); +} + +static void +pylock_dealloc(struct pylock *lock) +{ + struct pylock_op *op; + + while ((op = TAILQ_FIRST(&lock->ops)) != NULL) { + TAILQ_REMOVE(&lock->ops, op, list); + op->active = 0; + Py_DECREF((PyObject *)op); + } + + PyObject_Del((PyObject *)lock); +} + +static PyObject * +pylock_aenter(struct pylock *lock, PyObject *args) +{ + struct pylock_op *op; + + if (lock->owner != NULL && lock->owner->id == coro_running->id) { + PyErr_SetString(PyExc_RuntimeError, "recursive lock detected"); + return (NULL); + } + + if ((op = PyObject_New(struct pylock_op, &pylock_op_type)) == NULL) + return (NULL); + + op->active = 1; + op->lock = lock; + op->locking = 1; + op->coro = coro_running; + + Py_INCREF((PyObject *)op); + Py_INCREF((PyObject *)lock); + + TAILQ_INSERT_TAIL(&lock->ops, op, list); + + return ((PyObject *)op); +} + +static PyObject * +pylock_aexit(struct pylock *lock, PyObject *args) +{ + struct pylock_op *op; + + if (lock->owner == NULL || lock->owner->id != coro_running->id) { + PyErr_SetString(PyExc_RuntimeError, "invalid lock owner"); + return (NULL); + } + + if ((op = PyObject_New(struct pylock_op, &pylock_op_type)) == NULL) + return (NULL); + + op->active = 1; + op->lock = lock; + op->locking = 0; + op->coro = coro_running; + + Py_INCREF((PyObject *)op); + Py_INCREF((PyObject *)lock); + + TAILQ_INSERT_TAIL(&lock->ops, op, list); + + return ((PyObject *)op); +} + +static void +pylock_do_release(struct pylock *lock) +{ + struct pylock_op *op; + + lock->owner = NULL; + + TAILQ_FOREACH(op, &lock->ops, list) { + if (op->locking == 0) + continue; + + op->active = 0; + TAILQ_REMOVE(&op->lock->ops, op, list); + + if (op->coro->request != NULL) + http_request_wakeup(op->coro->request); + else + python_coro_wakeup(op->coro); + + Py_DECREF((PyObject *)op); + break; + } +} + +static void +pylock_op_dealloc(struct pylock_op *op) +{ + if (op->active) { + TAILQ_REMOVE(&op->lock->ops, op, list); + op->active = 0; + } + + Py_DECREF((PyObject *)op->lock); + PyObject_Del((PyObject *)op); +} + +static PyObject * +pylock_op_await(PyObject *obj) +{ + Py_INCREF(obj); + return (obj); +} + +static PyObject * +pylock_op_iternext(struct pylock_op *op) +{ + if (op->locking == 0) { + if (op->lock->owner == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "no lock owner set"); + return (NULL); + } + + if (op->lock->owner->id != coro_running->id) { + PyErr_SetString(PyExc_RuntimeError, + "lock not owned by caller"); + return (NULL); + } + + pylock_do_release(op->lock); + } else { + if (op->lock->owner != NULL) { + /* + * We could be beat by another coroutine that grabbed + * the lock even if we were the one woken up for it. + */ + if (op->active == 0) { + op->active = 1; + TAILQ_INSERT_HEAD(&op->lock->ops, op, list); + Py_INCREF((PyObject *)op); + } + Py_RETURN_NONE; + } + + op->lock->owner = coro_running; + } + + if (op->active) { + op->active = 0; + TAILQ_REMOVE(&op->lock->ops, op, list); + Py_DECREF((PyObject *)op); + } + + PyErr_SetNone(PyExc_StopIteration); + + return (NULL); +} + +static void +pyproc_timeout(void *arg, u_int64_t now) +{ + struct pyproc *proc = arg; + + proc->timer = NULL; + + if (proc->coro->sockop != NULL) + proc->coro->sockop->eof = 1; + + proc->coro->exception = PyExc_TimeoutError; + proc->coro->exception_msg = "timeout before process exited"; + + if (proc->coro->request != NULL) + http_request_wakeup(proc->coro->request); + else + python_coro_wakeup(proc->coro); +} + +static void +pyproc_dealloc(struct pyproc *proc) +{ + int status; + + TAILQ_REMOVE(&procs, proc, list); + + if (proc->timer != NULL) { + kore_timer_remove(proc->timer); + proc->timer = NULL; + } + + if (proc->pid != -1) { + if (kill(proc->pid, SIGKILL) == -1) { + kore_log(LOG_NOTICE, + "kore.proc failed to send SIGKILL %d (%s)", + proc->pid, errno_s); + } + + for (;;) { + if (waitpid(proc->pid, &status, 0) == -1) { + if (errno == EINTR) + continue; + kore_log(LOG_NOTICE, + "kore.proc failed to wait for %d (%s)", + proc->pid, errno_s); + } + break; + } + } + + if (proc->in != NULL) { + Py_DECREF((PyObject *)proc->in); + proc->in = NULL; + } + + if (proc->out != NULL) { + Py_DECREF((PyObject *)proc->out); + proc->out = NULL; + } + + PyObject_Del((PyObject *)proc); +} + +static PyObject * +pyproc_kill(struct pyproc *proc, PyObject *args) +{ + if (proc->pid != -1 && kill(proc->pid, SIGKILL) == -1) + kore_log(LOG_NOTICE, "kill(%d): %s", proc->pid, errno_s); + + Py_RETURN_TRUE; +} + +static PyObject * +pyproc_reap(struct pyproc *proc, PyObject *args) +{ + struct pyproc_op *op; + + if (proc->timer != NULL) { + kore_timer_remove(proc->timer); + proc->timer = NULL; + } + + if ((op = PyObject_New(struct pyproc_op, &pyproc_op_type)) == NULL) + return (NULL); + + op->proc = proc; + + Py_INCREF((PyObject *)proc); + + return ((PyObject *)op); +} + +static PyObject * +pyproc_recv(struct pyproc *proc, PyObject *args) +{ + Py_ssize_t len; + struct pysocket_op *op; + PyObject *obj; + int timeo; + + timeo = -1; + + if (proc->out == NULL) { + PyErr_SetString(PyExc_RuntimeError, "stdout closed"); + return (NULL); + } + + if (!PyArg_ParseTuple(args, "n|i", &len, &timeo)) + return (NULL); + + obj = pysocket_op_create(proc->out, PYSOCKET_TYPE_RECV, NULL, len); + if (obj == NULL) + return (NULL); + + op = (struct pysocket_op *)obj; + + if (timeo != -1) { + op->timer = kore_timer_add(pysocket_op_timeout, + timeo, op, KORE_TIMER_ONESHOT); + } + + return (obj); +} + +static PyObject * +pyproc_send(struct pyproc *proc, PyObject *args) +{ + Py_buffer buf; + PyObject *ret; + + if (proc->in == NULL) { + PyErr_SetString(PyExc_RuntimeError, "stdin closed"); + return (NULL); + } + + if (!PyArg_ParseTuple(args, "y*", &buf)) + return (NULL); + + ret = pysocket_op_create(proc->in, + PYSOCKET_TYPE_SEND, buf.buf, buf.len); + + return (ret); +} + +static PyObject * +pyproc_close_stdin(struct pyproc *proc, PyObject *args) +{ + if (proc->in != NULL) { + Py_DECREF((PyObject *)proc->in); + proc->in = NULL; + } + + Py_RETURN_TRUE; +} + +static void +pyproc_op_dealloc(struct pyproc_op *op) +{ + Py_DECREF((PyObject *)op->proc); + PyObject_Del((PyObject *)op); +} + +static PyObject * +pyproc_op_await(PyObject *sop) +{ + Py_INCREF(sop); + return (sop); +} + +static PyObject * +pyproc_op_iternext(struct pyproc_op *op) +{ + int ret; + PyObject *res; + + if (op->proc->coro->exception != NULL) { + PyErr_SetString(op->proc->coro->exception, + op->proc->coro->exception_msg); + op->proc->coro->exception = NULL; + return (NULL); + } + + if (op->proc->reaped == 0) + Py_RETURN_NONE; + + if (WIFSTOPPED(op->proc->status)) { + op->proc->reaped = 0; + Py_RETURN_NONE; + } + + if (WIFEXITED(op->proc->status)) { + ret = WEXITSTATUS(op->proc->status); + } else { + ret = op->proc->status; + } + + if ((res = PyLong_FromLong(ret)) == NULL) + return (NULL); + + PyErr_SetObject(PyExc_StopIteration, res); + Py_DECREF(res); + + return (NULL); +} + +static void +pygather_reap_coro(struct pygather_op *op, struct python_coro *reap) +{ + struct pygather_coro *coro; + struct pygather_result *result; + + TAILQ_FOREACH(coro, &op->coroutines, list) { + if (coro->coro->id == reap->id) + break; + } + + if (coro == NULL) + fatal("coroutine %" PRIu64 " not found in gather", reap->id); + + op->running--; + if (op->running < 0) + fatal("gatherop: running miscount (%d)", op->running); + + result = kore_pool_get(&gather_result_pool); + result->obj = NULL; + + if (_PyGen_FetchStopIterationValue(&result->obj) == -1) { + result->obj = Py_None; + Py_INCREF(Py_None); + } + + TAILQ_INSERT_TAIL(&op->results, result, list); + + TAILQ_REMOVE(&op->coroutines, coro, list); + kore_pool_put(&gather_coro_pool, coro); + + kore_python_coro_delete(reap); +} + +static void +pygather_op_dealloc(struct pygather_op *op) +{ + struct python_coro *old; + struct pygather_coro *coro, *next; + struct pygather_result *res, *rnext; + + /* + * Since we are calling kore_python_coro_delete() on all the + * remaining coroutines in this gather op we must remember the + * original coroutine that is running as the removal will end + * up setting coro_running to NULL. + */ + old = coro_running; + + for (coro = TAILQ_FIRST(&op->coroutines); coro != NULL; coro = next) { + next = TAILQ_NEXT(coro, list); + TAILQ_REMOVE(&op->coroutines, coro, list); + + /* Make sure we don't end up in pygather_reap_coro(). */ + coro->coro->gatherop = NULL; + + kore_python_coro_delete(coro->coro); + kore_pool_put(&gather_coro_pool, coro); + } + + coro_running = old; + + for (res = TAILQ_FIRST(&op->results); res != NULL; res = rnext) { + rnext = TAILQ_NEXT(res, list); + TAILQ_REMOVE(&op->results, res, list); + + Py_DECREF(res->obj); + kore_pool_put(&gather_result_pool, res); + } + + PyObject_Del((PyObject *)op); +} + +static PyObject * +pygather_op_await(PyObject *obj) +{ + Py_INCREF(obj); + return (obj); +} + +static PyObject * +pygather_op_iternext(struct pygather_op *op) +{ + int idx; + struct pygather_coro *coro; + struct pygather_result *res, *next; + PyObject *list, *obj; + + if (!TAILQ_EMPTY(&op->coroutines)) { + if (op->running > 0) + Py_RETURN_NONE; + + TAILQ_FOREACH(coro, &op->coroutines, list) { + if (op->running >= op->concurrency) + break; + python_coro_wakeup(coro->coro); + op->running++; + } + + Py_RETURN_NONE; + } + + if ((list = PyList_New(op->count)) == NULL) + return (NULL); + + idx = 0; + + for (res = TAILQ_FIRST(&op->results); res != NULL; res = next) { + next = TAILQ_NEXT(res, list); + TAILQ_REMOVE(&op->results, res, list); + + obj = res->obj; + res->obj = NULL; + kore_pool_put(&gather_result_pool, res); + + if (PyList_SetItem(list, idx++, obj) != 0) { + Py_DECREF(list); + return (NULL); + } + } + + PyErr_SetObject(PyExc_StopIteration, list); + Py_DECREF(list); + + return (NULL); +} + +static PyObject * +pyhttp_request_alloc(const struct http_request *req) +{ + union { const void *cp; void *p; } ptr; + struct pyhttp_request *pyreq; + + pyreq = PyObject_New(struct pyhttp_request, &pyhttp_request_type); + if (pyreq == NULL) + return (NULL); + + /* + * Hack around all http apis taking a non-const pointer and us having + * a const pointer for the req data structure. This is because we + * could potentially be called from a validator where the argument + * is a http_request pointer. + */ + ptr.cp = req; + pyreq->req = ptr.p; + pyreq->data = NULL; + + return ((PyObject *)pyreq); +} + +static PyObject * +pyhttp_file_alloc(struct http_file *file) +{ + struct pyhttp_file *pyfile; + + pyfile = PyObject_New(struct pyhttp_file, &pyhttp_file_type); + if (pyfile == NULL) + return (NULL); + + pyfile->file = file; + + return ((PyObject *)pyfile); +} + +static PyObject * +pyhttp_response(struct pyhttp_request *pyreq, PyObject *args) +{ + struct connection *c; + char *ptr; + Py_ssize_t length; + int status; + struct pyhttp_iterobj *iterobj; + PyObject *obj, *iterator; + + length = -1; + + if (!PyArg_ParseTuple(args, "iO", &status, &obj)) + return (NULL); + + if (PyBytes_CheckExact(obj)) { + if (PyBytes_AsStringAndSize(obj, &ptr, &length) == -1) + return (NULL); + + if (length < 0) { + PyErr_SetString(PyExc_TypeError, "invalid length"); + return (NULL); + } + + Py_INCREF(obj); + + http_response_stream(pyreq->req, status, ptr, length, + pyhttp_response_sent, obj); + } else if (obj == Py_None) { + http_response(pyreq->req, status, NULL, 0); + } else { + if ((iterator = PyObject_GetIter(obj)) == NULL) + return (NULL); + + c = pyreq->req->owner; + + iterobj = kore_pool_get(&iterobj_pool); + iterobj->iterator = iterator; + iterobj->connection = c; + iterobj->remove = 0; + + kore_buf_init(&iterobj->buf, 4096); + + c->hdlr_extra = iterobj; + c->flags |= CONN_IS_BUSY; + c->disconnect = pyhttp_iterobj_disconnect; + + pyreq->req->flags |= HTTP_REQUEST_NO_CONTENT_LENGTH; + http_response_header(pyreq->req, "transfer-encoding", + "chunked"); + + http_response(pyreq->req, status, NULL, 0); + pyhttp_iterobj_next(iterobj); + } + + Py_RETURN_TRUE; +} + +static int +pyhttp_response_sent(struct netbuf *nb) +{ + PyObject *data; + + data = nb->extra; + Py_DECREF(data); + + return (KORE_RESULT_OK); +} + +static int +pyhttp_iterobj_next(struct pyhttp_iterobj *iterobj) +{ + struct netbuf *nb; + PyObject *obj; + const char *ptr; + Py_ssize_t length; + + PyErr_Clear(); + + if ((obj = PyIter_Next(iterobj->iterator)) == NULL) { + if (PyErr_Occurred()) { + kore_python_log_error("pyhttp_iterobj_next"); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); + } + + if ((ptr = PyUnicode_AsUTF8AndSize(obj, &length)) == NULL) { + kore_python_log_error("pyhttp_iterobj_next"); + return (KORE_RESULT_ERROR); + } + + kore_buf_reset(&iterobj->buf); + kore_buf_appendf(&iterobj->buf, "%x\r\n", length); + kore_buf_append(&iterobj->buf, ptr, length); + kore_buf_appendf(&iterobj->buf, "\r\n"); + + Py_DECREF(obj); + + net_send_stream(iterobj->connection, iterobj->buf.data, + iterobj->buf.offset, pyhttp_iterobj_chunk_sent, &nb); + + nb->extra = iterobj; + + return (KORE_RESULT_RETRY); +} + +static int +pyhttp_iterobj_chunk_sent(struct netbuf *nb) +{ + int ret; + struct pyhttp_iterobj *iterobj; + + iterobj = nb->extra; + + if (iterobj->remove) { + ret = KORE_RESULT_ERROR; + } else { + ret = pyhttp_iterobj_next(iterobj); + } + + if (ret != KORE_RESULT_RETRY) { + iterobj->connection->hdlr_extra = NULL; + iterobj->connection->disconnect = NULL; + iterobj->connection->flags &= ~CONN_IS_BUSY; + + if (iterobj->remove == 0) + http_start_recv(iterobj->connection); + + kore_buf_reset(&iterobj->buf); + kore_buf_appendf(&iterobj->buf, "0\r\n\r\n"); + net_send_queue(iterobj->connection, + iterobj->buf.data, iterobj->buf.offset); + + Py_DECREF(iterobj->iterator); + + kore_buf_cleanup(&iterobj->buf); + kore_pool_put(&iterobj_pool, iterobj); + } else { + ret = KORE_RESULT_OK; + } + + return (ret); +} + +static void +pyhttp_iterobj_disconnect(struct connection *c) +{ + struct pyhttp_iterobj *iterobj; + + iterobj = c->hdlr_extra; + + iterobj->remove = 1; +} + +static PyObject * +pyhttp_response_header(struct pyhttp_request *pyreq, PyObject *args) +{ + const char *header, *value; + + if (!PyArg_ParseTuple(args, "ss", &header, &value)) + return (NULL); + + http_response_header(pyreq->req, header, value); + + Py_RETURN_TRUE; +} + +static PyObject * +pyhttp_request_header(struct pyhttp_request *pyreq, PyObject *args) +{ + const char *value; + const char *header; + PyObject *result; + + if (!PyArg_ParseTuple(args, "s", &header)) + return (NULL); + + if (!http_request_header(pyreq->req, header, &value)) { + Py_RETURN_NONE; + } + + if ((result = PyUnicode_FromString(value)) == NULL) + return (PyErr_NoMemory()); + + return (result); +} + +static PyObject * +pyhttp_body_read(struct pyhttp_request *pyreq, PyObject *args) +{ + ssize_t ret; + size_t len; + Py_ssize_t pylen; + PyObject *result; + u_int8_t buf[1024]; + + if (!PyArg_ParseTuple(args, "n", &pylen) || pylen < 0) + return (NULL); + + len = (size_t)pylen; + if (len > sizeof(buf)) { + PyErr_SetString(PyExc_RuntimeError, "len > sizeof(buf)"); + return (NULL); + } + + ret = http_body_read(pyreq->req, buf, len); + if (ret == -1) { + PyErr_SetString(PyExc_RuntimeError, "http_body_read() failed"); + return (NULL); + } + + if (ret > INT_MAX) { + PyErr_SetString(PyExc_RuntimeError, "ret > INT_MAX"); + return (NULL); + } + + result = Py_BuildValue("ny#", ret, buf, (int)ret); + if (result == NULL) + return (PyErr_NoMemory()); + + return (result); +} + +static PyObject * +pyhttp_populate_get(struct pyhttp_request *pyreq, PyObject *args) +{ + http_populate_get(pyreq->req); + Py_RETURN_TRUE; +} + +static PyObject * +pyhttp_populate_post(struct pyhttp_request *pyreq, PyObject *args) +{ + http_populate_post(pyreq->req); + Py_RETURN_TRUE; +} + +static PyObject * +pyhttp_populate_multi(struct pyhttp_request *pyreq, PyObject *args) +{ + http_populate_multipart_form(pyreq->req); + Py_RETURN_TRUE; +} + +static PyObject * +pyhttp_populate_cookies(struct pyhttp_request *pyreq, PyObject *args) +{ + http_populate_cookies(pyreq->req); + Py_RETURN_TRUE; +} + +static PyObject * +pyhttp_argument(struct pyhttp_request *pyreq, PyObject *args) +{ + const char *name; + PyObject *value; + char *string; + + if (!PyArg_ParseTuple(args, "s", &name)) + return (NULL); + + if (!http_argument_get_string(pyreq->req, name, &string)) { + Py_RETURN_NONE; + } + + if ((value = PyUnicode_FromString(string)) == NULL) + return (PyErr_NoMemory()); + + return (value); +} + +static PyObject * +pyhttp_cookie(struct pyhttp_request *pyreq, PyObject *args) +{ + const char *name; + PyObject *value; + char *string; + + if (!PyArg_ParseTuple(args, "s", &name)) + return (NULL); + + if (!http_request_cookie(pyreq->req, name, &string)) { + Py_RETURN_NONE; + } + + if ((value = PyUnicode_FromString(string)) == NULL) + return (PyErr_NoMemory()); + + return (value); +} + +static PyObject * +pyhttp_file_lookup(struct pyhttp_request *pyreq, PyObject *args) +{ + const char *name; + struct http_file *file; + PyObject *pyfile; + + if (!PyArg_ParseTuple(args, "s", &name)) + return (NULL); + + if ((file = http_file_lookup(pyreq->req, name)) == NULL) { + Py_RETURN_NONE; + } + + if ((pyfile = pyhttp_file_alloc(file)) == NULL) + return (PyErr_NoMemory()); + + return (pyfile); +} + +static PyObject * +pyhttp_file_read(struct pyhttp_file *pyfile, PyObject *args) +{ + ssize_t ret; + size_t len; + Py_ssize_t pylen; + PyObject *result; + u_int8_t buf[1024]; + + if (!PyArg_ParseTuple(args, "n", &pylen) || pylen < 0) + return (NULL); + + len = (size_t)pylen; + if (len > sizeof(buf)) { + PyErr_SetString(PyExc_RuntimeError, "len > sizeof(buf)"); + return (NULL); + } + + ret = http_file_read(pyfile->file, buf, len); + if (ret == -1) { + PyErr_SetString(PyExc_RuntimeError, "http_file_read() failed"); + return (NULL); + } + + if (ret > INT_MAX) { + PyErr_SetString(PyExc_RuntimeError, "ret > INT_MAX"); + return (NULL); + } + + result = Py_BuildValue("ny#", ret, buf, (int)ret); + if (result == NULL) + return (PyErr_NoMemory()); + + return (result); +} + +static PyObject * +pyhttp_websocket_handshake(struct pyhttp_request *pyreq, PyObject *args) +{ + const char *onconnect, *onmsg, *ondisconnect; + + if (!PyArg_ParseTuple(args, "sss", &onconnect, &onmsg, &ondisconnect)) + return (NULL); + + kore_websocket_handshake(pyreq->req, onconnect, onmsg, ondisconnect); + + Py_RETURN_TRUE; +} + +static PyObject * +pyconnection_websocket_send(struct pyconnection *pyc, PyObject *args) +{ + const char *data; + int op, len; + + if (pyc->c->proto != CONN_PROTO_WEBSOCKET) { + PyErr_SetString(PyExc_TypeError, "not a websocket connection"); + return (NULL); + } + + len = -1; + + if (!PyArg_ParseTuple(args, "iy#", &op, &data, &len)) + return (NULL); + + if (len < 0) { + PyErr_SetString(PyExc_TypeError, "invalid length"); + return (NULL); + } + + switch (op) { + case WEBSOCKET_OP_TEXT: + case WEBSOCKET_OP_BINARY: + break; + default: + PyErr_SetString(PyExc_TypeError, "invalid op parameter"); + return (NULL); + } + + kore_websocket_send(pyc->c, op, data, len); + + Py_RETURN_TRUE; +} + +static PyObject * +python_websocket_broadcast(PyObject *self, PyObject *args) +{ + struct connection *c; + struct pyconnection *pyc; + const char *data; + PyObject *pysrc; + int op, broadcast, len; + + len = -1; + + if (!PyArg_ParseTuple(args, "Oiy#i", &pysrc, &op, &data, &len, + &broadcast)) + return (NULL); + + if (len < 0) { + PyErr_SetString(PyExc_TypeError, "invalid length"); + return (NULL); + } + + switch (op) { + case WEBSOCKET_OP_TEXT: + case WEBSOCKET_OP_BINARY: + break; + default: + PyErr_SetString(PyExc_TypeError, "invalid op parameter"); + return (NULL); + } + + if (pysrc == Py_None) { + c = NULL; + } else { + if (!PyObject_TypeCheck(pysrc, &pyconnection_type)) + return (NULL); + pyc = (struct pyconnection *)pysrc; + c = pyc->c; + } + + kore_websocket_broadcast(c, op, data, len, broadcast); + + Py_RETURN_TRUE; +} + +static PyObject * +pyhttp_get_host(struct pyhttp_request *pyreq, void *closure) +{ + PyObject *host; + + if ((host = PyUnicode_FromString(pyreq->req->host)) == NULL) + return (PyErr_NoMemory()); + + return (host); +} + +static PyObject * +pyhttp_get_path(struct pyhttp_request *pyreq, void *closure) +{ + PyObject *path; + + if ((path = PyUnicode_FromString(pyreq->req->path)) == NULL) + return (PyErr_NoMemory()); + + return (path); +} + +static PyObject * +pyhttp_get_body(struct pyhttp_request *pyreq, void *closure) +{ + ssize_t ret; + struct kore_buf buf; + PyObject *body; + u_int8_t data[BUFSIZ]; + + kore_buf_init(&buf, 1024); + if (!http_body_rewind(pyreq->req)) { + PyErr_SetString(PyExc_RuntimeError, + "http_body_rewind() failed"); + return (NULL); + } + + for (;;) { + ret = http_body_read(pyreq->req, data, sizeof(data)); + if (ret == -1) { + kore_buf_cleanup(&buf); + PyErr_SetString(PyExc_RuntimeError, + "http_body_read() failed"); + return (NULL); + } + + if (ret == 0) + break; + + kore_buf_append(&buf, data, (size_t)ret); + } + + body = PyBytes_FromStringAndSize((char *)buf.data, buf.offset); + kore_buf_free(&buf); + + if (body == NULL) + return (PyErr_NoMemory()); + + return (body); +} + +static PyObject * +pyhttp_get_agent(struct pyhttp_request *pyreq, void *closure) +{ + PyObject *agent; + + if (pyreq->req->agent == NULL) { + Py_RETURN_NONE; + } + + if ((agent = PyUnicode_FromString(pyreq->req->path)) == NULL) + return (PyErr_NoMemory()); + + return (agent); +} + +static PyObject * +pyhttp_get_method(struct pyhttp_request *pyreq, void *closure) +{ + PyObject *method; + + if ((method = PyLong_FromUnsignedLong(pyreq->req->method)) == NULL) + return (PyErr_NoMemory()); + + return (method); +} + +static PyObject * +pyhttp_get_body_path(struct pyhttp_request *pyreq, void *closure) +{ + PyObject *path; + + if (pyreq->req->http_body_path == NULL) { + Py_RETURN_NONE; + } + + if ((path = PyUnicode_FromString(pyreq->req->http_body_path)) == NULL) + return (PyErr_NoMemory()); + + return (path); +} + +static PyObject * +pyhttp_get_connection(struct pyhttp_request *pyreq, void *closure) +{ + PyObject *pyc; + + if (pyreq->req->owner == NULL) { + Py_RETURN_NONE; + } + + if ((pyc = pyconnection_alloc(pyreq->req->owner)) == NULL) + return (PyErr_NoMemory()); + + return (pyc); +} + +static PyObject * +pyhttp_file_get_name(struct pyhttp_file *pyfile, void *closure) +{ + PyObject *name; + + if ((name = PyUnicode_FromString(pyfile->file->name)) == NULL) + return (PyErr_NoMemory()); + + return (name); +} + +static PyObject * +pyhttp_file_get_filename(struct pyhttp_file *pyfile, void *closure) +{ + PyObject *name; + + if ((name = PyUnicode_FromString(pyfile->file->filename)) == NULL) + return (PyErr_NoMemory()); + + return (name); +} + +#if defined(KORE_USE_PGSQL) +static void +pykore_pgsql_dealloc(struct pykore_pgsql *pysql) +{ + Py_ssize_t i; + + kore_free(pysql->db); + kore_free(pysql->query); + kore_pgsql_cleanup(&pysql->sql); + + if (pysql->result != NULL) + Py_DECREF(pysql->result); + + for (i = 0; i < pysql->param.count; i++) + Py_XDECREF(pysql->param.objs[i]); + + kore_free(pysql->param.objs); + kore_free(pysql->param.values); + kore_free(pysql->param.lengths); + kore_free(pysql->param.formats); + + PyObject_Del((PyObject *)pysql); +} + +static PyObject * +pykore_pgsql_alloc(struct http_request *req, const char *db, const char *query, + PyObject *kwargs) +{ + PyObject *obj; + struct pykore_pgsql *pysql; + + pysql = PyObject_New(struct pykore_pgsql, &pykore_pgsql_type); + if (pysql == NULL) + return (NULL); + + pysql->req = req; + pysql->result = NULL; + pysql->db = kore_strdup(db); + pysql->query = kore_strdup(query); + pysql->state = PYKORE_PGSQL_PREINIT; + + pysql->binary = 0; + pysql->param.count = 0; + pysql->param.objs = NULL; + pysql->param.values = NULL; + pysql->param.lengths = NULL; + pysql->param.formats = NULL; + + memset(&pysql->sql, 0, sizeof(pysql->sql)); + + if (kwargs != NULL) { + if ((obj = PyDict_GetItemString(kwargs, "params")) != NULL) { + if (!pykore_pgsql_params(pysql, obj)) { + Py_DECREF((PyObject *)pysql); + return (NULL); + } + } + + if ((obj = PyDict_GetItemString(kwargs, "binary")) != NULL) { + if (obj == Py_True) { + pysql->binary = 1; + } else if (obj == Py_False) { + pysql->binary = 0; + } else { + Py_DECREF((PyObject *)pysql); + PyErr_SetString(PyExc_RuntimeError, + "pgsql: binary not True or False"); + return (NULL); + } + } + } + + return ((PyObject *)pysql); +} + +static int +pykore_pgsql_params(struct pykore_pgsql *op, PyObject *list) +{ + union { const char *cp; char *p; } ptr; + PyObject *item; + int format; + Py_ssize_t i, len, vlen; + + if (!PyList_CheckExact(list)) { + if (list == Py_None) + return (KORE_RESULT_OK); + + PyErr_SetString(PyExc_RuntimeError, + "pgsql: params keyword must be a list"); + return (KORE_RESULT_ERROR); + } + + len = PyList_Size(list); + if (len == 0) + return (KORE_RESULT_OK); + + if (len > INT_MAX) { + PyErr_SetString(PyExc_RuntimeError, + "pgsql: list length too large"); + return (KORE_RESULT_ERROR); + } + + op->param.count = len; + op->param.lengths = kore_calloc(len, sizeof(int)); + op->param.formats = kore_calloc(len, sizeof(int)); + op->param.values = kore_calloc(len, sizeof(char *)); + op->param.objs = kore_calloc(len, sizeof(PyObject *)); + + for (i = 0; i < len; i++) { + if ((item = PyList_GetItem(list, i)) == NULL) + return (KORE_RESULT_ERROR); + + if (PyUnicode_CheckExact(item)) { + format = 0; + ptr.cp = PyUnicode_AsUTF8AndSize(item, &vlen); + } else if (PyBytes_CheckExact(item)) { + format = 1; + if (PyBytes_AsStringAndSize(item, &ptr.p, &vlen) == -1) + ptr.p = NULL; + } else { + PyErr_Format(PyExc_RuntimeError, + "pgsql: item %zu is not a string or bytes", i); + return (KORE_RESULT_ERROR); + } + + if (ptr.cp == NULL) + return (KORE_RESULT_ERROR); + + op->param.lengths[i] = vlen; + op->param.values[i] = ptr.cp; + op->param.formats[i] = format; + + /* Hold on to it since we are directly referencing its data. */ + op->param.objs[i] = item; + Py_INCREF(item); + } + + return (KORE_RESULT_OK); +} + +static PyObject * +pykore_pgsql_iternext(struct pykore_pgsql *pysql) +{ + switch (pysql->state) { + case PYKORE_PGSQL_PREINIT: + kore_pgsql_init(&pysql->sql); + kore_pgsql_bind_request(&pysql->sql, pysql->req); + pysql->state = PYKORE_PGSQL_INITIALIZE; + /* fallthrough */ + case PYKORE_PGSQL_INITIALIZE: + if (!kore_pgsql_setup(&pysql->sql, pysql->db, + KORE_PGSQL_ASYNC)) { + if (pysql->sql.state == KORE_PGSQL_STATE_INIT) + break; + PyErr_Format(PyExc_RuntimeError, "pgsql error: %s", + pysql->sql.error); + return (NULL); + } + /* fallthrough */ + case PYKORE_PGSQL_QUERY: + if (!kore_pgsql_query_param_fields(&pysql->sql, + pysql->query, pysql->binary, + pysql->param.count, pysql->param.values, + pysql->param.lengths, pysql->param.formats)) { + PyErr_Format(PyExc_RuntimeError, + "pgsql error: %s", pysql->sql.error); + return (NULL); + } + pysql->state = PYKORE_PGSQL_WAIT; + break; +wait_again: + case PYKORE_PGSQL_WAIT: + switch (pysql->sql.state) { + case KORE_PGSQL_STATE_WAIT: + break; + case KORE_PGSQL_STATE_COMPLETE: + PyErr_SetNone(PyExc_StopIteration); + if (pysql->result != NULL) { + PyErr_SetObject(PyExc_StopIteration, + pysql->result); + Py_DECREF(pysql->result); + } else { + PyErr_SetObject(PyExc_StopIteration, Py_None); + } + return (NULL); + case KORE_PGSQL_STATE_ERROR: + PyErr_Format(PyExc_RuntimeError, + "failed to perform query: %s", pysql->sql.error); + return (NULL); + case KORE_PGSQL_STATE_RESULT: + if (!pykore_pgsql_result(pysql)) + return (NULL); + goto wait_again; + default: + kore_pgsql_continue(&pysql->sql); + goto wait_again; + } + break; + default: + PyErr_SetString(PyExc_RuntimeError, "bad pykore_pgsql state"); + return (NULL); + } + + /* tell caller to wait. */ + Py_RETURN_NONE; +} + +static PyObject * +pykore_pgsql_await(PyObject *obj) +{ + Py_INCREF(obj); + return (obj); +} + +static int +pykore_pgsql_result(struct pykore_pgsql *pysql) +{ + const char *val; + char key[64]; + PyObject *list, *pyrow, *pyval; + int rows, row, field, fields, len; + + if ((list = PyList_New(0)) == NULL) { + PyErr_SetNone(PyExc_MemoryError); + return (KORE_RESULT_ERROR); + } + + rows = kore_pgsql_ntuples(&pysql->sql); + fields = kore_pgsql_nfields(&pysql->sql); + + for (row = 0; row < rows; row++) { + if ((pyrow = PyDict_New()) == NULL) { + Py_DECREF(list); + PyErr_SetNone(PyExc_MemoryError); + return (KORE_RESULT_ERROR); + } + + for (field = 0; field < fields; field++) { + val = kore_pgsql_getvalue(&pysql->sql, row, field); + len = kore_pgsql_getlength(&pysql->sql, row, field); + + if (kore_pgsql_column_binary(&pysql->sql, field)) { + pyval = PyBytes_FromStringAndSize(val, len); + } else { + pyval = PyUnicode_FromString(val); + } + + if (pyval == NULL) { + Py_DECREF(pyrow); + Py_DECREF(list); + PyErr_SetNone(PyExc_MemoryError); + return (KORE_RESULT_ERROR); + } + + (void)snprintf(key, sizeof(key), "%s", + kore_pgsql_fieldname(&pysql->sql, field)); + + if (PyDict_SetItemString(pyrow, key, pyval) == -1) { + Py_DECREF(pyval); + Py_DECREF(pyrow); + Py_DECREF(list); + PyErr_SetString(PyExc_RuntimeError, + "failed to add new value to row"); + return (KORE_RESULT_ERROR); + } + + Py_DECREF(pyval); + } + + if (PyList_Insert(list, row, pyrow) == -1) { + Py_DECREF(pyrow); + Py_DECREF(list); + PyErr_SetString(PyExc_RuntimeError, + "failed to add new row to list"); + return (KORE_RESULT_ERROR); + } + + Py_DECREF(pyrow); + } + + pysql->result = list; + kore_pgsql_continue(&pysql->sql); + + return (KORE_RESULT_OK); +} + +static PyObject * +pyhttp_pgsql(struct pyhttp_request *pyreq, PyObject *args, PyObject *kwargs) +{ + PyObject *obj; + const char *db, *query; + + if (!PyArg_ParseTuple(args, "ss", &db, &query)) + return (NULL); + + if ((obj = pykore_pgsql_alloc(pyreq->req, db, query, kwargs)) == NULL) + return (PyErr_NoMemory()); + + Py_INCREF(obj); + pyreq->data = obj; + + return ((PyObject *)obj); +} +#endif + +#if defined(KORE_USE_CURL) +static PyObject * +python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *obj; + struct pyhttp_client *client; + const char *url, *v; + + if (!PyArg_ParseTuple(args, "s", &url)) + return (NULL); + + client = PyObject_New(struct pyhttp_client, &pyhttp_client_type); + if (client == NULL) + return (NULL); + + client->unix = NULL; + client->tlskey = NULL; + client->tlscert = NULL; + client->cabundle = NULL; + + client->tlsverify = 1; + client->url = kore_strdup(url); + + if (kwargs != NULL) { + if ((obj = PyDict_GetItemString(kwargs, "tlscert")) != NULL) { + if ((v = PyUnicode_AsUTF8(obj)) == NULL) { + Py_DECREF((PyObject *)client); + return (NULL); + } + + client->tlscert = kore_strdup(v); + } + + if ((obj = PyDict_GetItemString(kwargs, "tlskey")) != NULL) { + if ((v = PyUnicode_AsUTF8(obj)) == NULL) { + Py_DECREF((PyObject *)client); + return (NULL); + } + + client->tlskey = kore_strdup(v); + } + + if ((obj = PyDict_GetItemString(kwargs, "cabundle")) != NULL) { + if ((v = PyUnicode_AsUTF8(obj)) == NULL) { + Py_DECREF((PyObject *)client); + return (NULL); + } + + client->cabundle = kore_strdup(v); + } + + if ((obj = PyDict_GetItemString(kwargs, "unix")) != NULL) { + if ((v = PyUnicode_AsUTF8(obj)) == NULL) { + Py_DECREF((PyObject *)client); + return (NULL); + } + + client->unix = kore_strdup(v); + } + + if ((obj = PyDict_GetItemString(kwargs, "tlsverify")) != NULL) { + if (obj == Py_True) { + client->tlsverify = 1; + } else if (obj == Py_False) { + client->tlsverify = 0; + } else { + Py_DECREF((PyObject *)client); + PyErr_SetString(PyExc_RuntimeError, + "tlsverify not True or False"); + return (NULL); + } + } + } + + if ((client->tlscert != NULL && client->tlskey == NULL) || + (client->tlskey != NULL && client->tlscert == NULL)) { + Py_DECREF((PyObject *)client); + PyErr_SetString(PyExc_RuntimeError, + "invalid TLS client configuration"); + return (NULL); + } + + return ((PyObject *)client); +} + +static void +pyhttp_client_dealloc(struct pyhttp_client *client) +{ + kore_free(client->url); + kore_free(client->unix); + kore_free(client->tlskey); + kore_free(client->tlscert); + kore_free(client->cabundle); + + PyObject_Del((PyObject *)client); +} + +static PyObject * +pyhttp_client_get(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_GET, kwargs)); +} + +static PyObject * +pyhttp_client_put(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_PUT, kwargs)); +} + +static PyObject * +pyhttp_client_post(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_POST, kwargs)); +} + +static PyObject * +pyhttp_client_head(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_HEAD, kwargs)); +} + +static PyObject * +pyhttp_client_patch(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_PATCH, kwargs)); +} + +static PyObject * +pyhttp_client_delete(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_DELETE, kwargs)); +} + +static PyObject * +pyhttp_client_options(struct pyhttp_client *client, PyObject *args, + PyObject *kwargs) +{ + return (pyhttp_client_request(client, HTTP_METHOD_OPTIONS, kwargs)); +} + +static PyObject * +pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) +{ + struct pyhttp_client_op *op; + char *ptr; + const char *k, *v; + Py_ssize_t length, idx; + PyObject *data, *headers, *key, *item; + + ptr = NULL; + length = 0; + headers = NULL; + + if (kwargs != NULL && + ((headers = PyDict_GetItemString(kwargs, "headers")) != NULL)) { + if (!PyDict_CheckExact(headers)) { + PyErr_SetString(PyExc_RuntimeError, + "header keyword must be a dict"); + return (NULL); + } + } + + switch (m) { + case HTTP_METHOD_GET: + case HTTP_METHOD_HEAD: + case HTTP_METHOD_DELETE: + case HTTP_METHOD_OPTIONS: + break; + case HTTP_METHOD_PUT: + case HTTP_METHOD_POST: + case HTTP_METHOD_PATCH: + length = -1; + + if (kwargs == NULL) { + PyErr_Format(PyExc_RuntimeError, + "no keyword arguments given, but body expected ", + http_method_text(m)); + return (NULL); + } + + if ((data = PyDict_GetItemString(kwargs, "body")) == NULL) + return (NULL); + + if (PyBytes_AsStringAndSize(data, &ptr, &length) == -1) + return (NULL); + + if (length < 0) { + PyErr_SetString(PyExc_TypeError, "invalid length"); + return (NULL); + } + break; + default: + fatal("%s: unknown method %d", __func__, m); + } + + op = PyObject_New(struct pyhttp_client_op, &pyhttp_client_op_type); + if (op == NULL) + return (NULL); + + if (!kore_curl_init(&op->curl, client->url)) { + Py_DECREF((PyObject *)op); + PyErr_SetString(PyExc_RuntimeError, "failed to setup call"); + return (NULL); + } + + op->headers = 0; + op->coro = coro_running; + op->state = PYHTTP_CLIENT_OP_RUN; + + Py_INCREF(client); + + kore_curl_http_setup(&op->curl, m, ptr, length); + kore_curl_bind_callback(&op->curl, python_curl_callback, op); + + /* Go in with our own bare hands. */ + if (client->unix != NULL) { +#if defined(__linux__) + if (client->unix[0] == '@') { + curl_easy_setopt(op->curl.handle, + CURLOPT_ABSTRACT_UNIX_SOCKET, client->unix + 1); + } else { + curl_easy_setopt(op->curl.handle, + CURLOPT_UNIX_SOCKET_PATH, client->unix); + } +#else + curl_easy_setopt(op->curl.handle, CURLOPT_UNIX_SOCKET_PATH, + client->unix); +#endif + } + + if (client->tlskey != NULL && client->tlscert != NULL) { + curl_easy_setopt(op->curl.handle, CURLOPT_SSLCERT, + client->tlscert); + curl_easy_setopt(op->curl.handle, CURLOPT_SSLKEY, + client->tlskey); + } + + if (client->tlsverify == 0) { + curl_easy_setopt(op->curl.handle, CURLOPT_SSL_VERIFYHOST, 0); + curl_easy_setopt(op->curl.handle, CURLOPT_SSL_VERIFYPEER, 0); + } + + if (client->cabundle != NULL) { + curl_easy_setopt(op->curl.handle, CURLOPT_CAINFO, + client->cabundle); + } + + if (headers != NULL) { + idx = 0; + while (PyDict_Next(headers, &idx, &key, &item)) { + if ((k = PyUnicode_AsUTF8(key)) == NULL) { + Py_DECREF((PyObject *)op); + return (NULL); + } + + if ((v = PyUnicode_AsUTF8(item)) == NULL) { + Py_DECREF((PyObject *)op); + return (NULL); + } + + kore_curl_http_set_header(&op->curl, k, v); + } + } + + if (kwargs != NULL && + ((item = PyDict_GetItemString(kwargs, "return_headers")) != NULL)) { + if (item == Py_True) { + op->headers = 1; + } else if (item == Py_False) { + op->headers = 0; + } else { + Py_DECREF((PyObject *)op); + PyErr_SetString(PyExc_RuntimeError, + "return_headers not True or False"); + return (NULL); + } + } + + return ((PyObject *)op); +} + +static void +pyhttp_client_op_dealloc(struct pyhttp_client_op *op) +{ + kore_curl_cleanup(&op->curl); + PyObject_Del((PyObject *)op); +} + +static PyObject * +pyhttp_client_op_await(PyObject *op) +{ + Py_INCREF(op); + return (op); +} + +static PyObject * +pyhttp_client_op_iternext(struct pyhttp_client_op *op) +{ + size_t len; + struct http_header *hdr; + const u_int8_t *response; + PyObject *result, *tuple, *dict, *value; + + if (op->state == PYHTTP_CLIENT_OP_RUN) { + kore_curl_run(&op->curl); + op->state = PYHTTP_CLIENT_OP_RESULT; + Py_RETURN_NONE; + } + + if (!kore_curl_success(&op->curl)) { + PyErr_Format(PyExc_RuntimeError, "request to '%s' failed: %s", + op->curl.url, kore_curl_strerror(&op->curl)); + return (NULL); + } + + kore_curl_response_as_bytes(&op->curl, &response, &len); + + if (op->headers) { + kore_curl_http_parse_headers(&op->curl); + + if ((dict = PyDict_New()) == NULL) + return (NULL); + + TAILQ_FOREACH(hdr, &op->curl.http.resp_hdrs, list) { + value = PyUnicode_FromString(hdr->value); + if (value == NULL) { + Py_DECREF(dict); + return (NULL); + } + + if (PyDict_SetItemString(dict, + hdr->header, value) == -1) { + Py_DECREF(dict); + Py_DECREF(value); + return (NULL); + } + + Py_DECREF(value); + } + + if ((tuple = Py_BuildValue("(iOy#)", op->curl.http.status, + dict, (const char *)response, len)) == NULL) + return (NULL); + + Py_DECREF(dict); + } else { + if ((tuple = Py_BuildValue("(iy#)", op->curl.http.status, + (const char *)response, len)) == NULL) + return (NULL); + } + + result = PyObject_CallFunctionObjArgs(PyExc_StopIteration, tuple, NULL); + if (result == NULL) { + Py_DECREF(tuple); + return (NULL); + } + + Py_DECREF(tuple); + PyErr_SetObject(PyExc_StopIteration, result); + Py_DECREF(result); + + return (NULL); +} + +static void +python_curl_callback(struct kore_curl *curl, void *arg) +{ + struct pyhttp_client_op *op = arg; + + if (op->coro->request != NULL) + http_request_wakeup(op->coro->request); + else + python_coro_wakeup(op->coro); +} +#endif diff -Nru kore-2.0.0/src/runtime.c kore-3.3.1/src/runtime.c --- kore-2.0.0/src/runtime.c 1970-01-01 00:00:00.000000000 +0000 +++ kore-3.3.1/src/runtime.c 2019-06-03 13:29:24.000000000 +0000 @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2019 Joris Vink + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include + +#include "kore.h" + +#if !defined(KORE_NO_HTTP) +#include "http.h" +#endif + +#if defined(KORE_USE_PYTHON) +#include "python_api.h" +#endif + +static void native_runtime_execute(void *); +static int native_runtime_onload(void *, int); +static void native_runtime_connect(void *, struct connection *); +static void native_runtime_configure(void *, int, char **); +#if !defined(KORE_NO_HTTP) +static int native_runtime_http_request(void *, struct http_request *); +static int native_runtime_validator(void *, struct http_request *, + const void *); + +static void native_runtime_wsmessage(void *, struct connection *, u_int8_t, + const void *, size_t); +#endif + +struct kore_runtime kore_native_runtime = { + KORE_RUNTIME_NATIVE, +#if !defined(KORE_NO_HTTP) + .http_request = native_runtime_http_request, + .validator = native_runtime_validator, + .wsconnect = native_runtime_connect, + .wsmessage = native_runtime_wsmessage, + .wsdisconnect = native_runtime_connect, +#endif + .onload = native_runtime_onload, + .connect = native_runtime_connect, + .execute = native_runtime_execute, + .configure = native_runtime_configure +}; + +struct kore_runtime_call * +kore_runtime_getcall(const char *symbol) +{ + void *ptr; + struct kore_runtime_call *rcall; + struct kore_runtime *runtime; + + ptr = kore_module_getsym(symbol, &runtime); + if (ptr == NULL) + return (NULL); + + rcall = kore_malloc(sizeof(*rcall)); + rcall->addr = ptr; + rcall->runtime = runtime; + + return (rcall); +} + +void +kore_runtime_execute(struct kore_runtime_call *rcall) +{ + rcall->runtime->execute(rcall->addr); +} + +void +kore_runtime_configure(struct kore_runtime_call *rcall, int argc, char **argv) +{ + rcall->runtime->configure(rcall->addr, argc, argv); +} + +int +kore_runtime_onload(struct kore_runtime_call *rcall, int action) +{ + return (rcall->runtime->onload(rcall->addr, action)); +} + +void +kore_runtime_connect(struct kore_runtime_call *rcall, struct connection *c) +{ + rcall->runtime->connect(rcall->addr, c); +} + +#if !defined(KORE_NO_HTTP) +int +kore_runtime_http_request(struct kore_runtime_call *rcall, + struct http_request *req) +{ + return (rcall->runtime->http_request(rcall->addr, req)); +} + +int +kore_runtime_validator(struct kore_runtime_call *rcall, + struct http_request *req, const void *data) +{ + return (rcall->runtime->validator(rcall->addr, req, data)); +} + +void +kore_runtime_wsconnect(struct kore_runtime_call *rcall, struct connection *c) +{ + rcall->runtime->wsconnect(rcall->addr, c); +} + +void +kore_runtime_wsmessage(struct kore_runtime_call *rcall, struct connection *c, + u_int8_t op, const void *data, size_t len) +{ + rcall->runtime->wsmessage(rcall->addr, c, op, data, len); +} + +void +kore_runtime_wsdisconnect(struct kore_runtime_call *rcall, struct connection *c) +{ + rcall->runtime->wsdisconnect(rcall->addr, c); +} +#endif + +static void +native_runtime_execute(void *addr) +{ + void (*cb)(void); + + *(void **)&(cb) = addr; + cb(); +} + +static void +native_runtime_configure(void *addr, int argc, char **argv) +{ + void (*cb)(int, char **); + + *(void **)&(cb) = addr; + cb(argc, argv); +} + +static void +native_runtime_connect(void *addr, struct connection *c) +{ + void (*cb)(struct connection *); + + *(void **)&(cb) = addr; + cb(c); +} + +static int +native_runtime_onload(void *addr, int action) +{ + int (*cb)(int); + + *(void **)&(cb) = addr; + return (cb(action)); +} + +#if !defined(KORE_NO_HTTP) +static int +native_runtime_http_request(void *addr, struct http_request *req) +{ + int (*cb)(struct http_request *); + + *(void **)&(cb) = addr; + return (cb(req)); +} + +static int +native_runtime_validator(void *addr, struct http_request *req, const void *data) +{ + int (*cb)(struct http_request *, const void *); + + *(void **)&(cb) = addr; + return (cb(req, data)); +} + +static void +native_runtime_wsmessage(void *addr, struct connection *c, u_int8_t op, + const void *data, size_t len) +{ + void (*cb)(struct connection *, u_int8_t, const void *, size_t); + + *(void **)&(cb) = addr; + cb(c, op, data, len); + +} +#endif diff -Nru kore-2.0.0/src/tasks.c kore-3.3.1/src/tasks.c --- kore-2.0.0/src/tasks.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/tasks.c 2019-06-03 13:29:24.000000000 +0000 @@ -59,8 +59,10 @@ #if !defined(KORE_NO_HTTP) t->req = NULL; #endif + t->evt.type = KORE_TYPE_TASK; + t->evt.handle = kore_task_handle; + t->entry = entry; - t->type = KORE_TYPE_TASK; t->state = KORE_TASK_STATE_CREATED; pthread_rwlock_init(&(t->lock), NULL); @@ -157,7 +159,6 @@ kore_task_finish(struct kore_task *t) { kore_debug("kore_task_finished: %p (%d)", t, t->result); - pthread_rwlock_wrlock(&(t->lock)); if (t->fds[1] != -1) { @@ -174,6 +175,7 @@ int fd; kore_debug("kore_task_channel_write: %p <- %p (%ld)", t, data, len); + THREAD_FD_ASSIGN(t->thread->tid, fd, t->fds[1], t->fds[0]); task_channel_write(fd, &len, sizeof(len)); task_channel_write(fd, data, len); @@ -201,8 +203,10 @@ } void -kore_task_handle(struct kore_task *t, int finished) +kore_task_handle(void *arg, int finished) { + struct kore_task *t = arg; + kore_debug("kore_task_handle: %p, %d", t, finished); #if !defined(KORE_NO_HTTP) diff -Nru kore-2.0.0/src/timer.c kore-3.3.1/src/timer.c --- kore-2.0.0/src/timer.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/timer.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Joris Vink + * Copyright (c) 2016-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include #include "kore.h" @@ -60,18 +61,32 @@ } u_int64_t +kore_timer_next_run(u_int64_t now) +{ + struct kore_timer *timer; + + if ((timer = TAILQ_FIRST(&kore_timers)) != NULL) { + if (timer->nextrun > now) + return (timer->nextrun - now); + return (0); + } + + return (KORE_WAIT_INFINITE); +} + +void kore_timer_run(u_int64_t now) { - struct kore_timer *timer, *t; - u_int64_t next_timer; + struct kore_timer *timer, *t, *prev; - next_timer = 100; + prev = NULL; while ((timer = TAILQ_FIRST(&kore_timers)) != NULL) { - if (timer->nextrun > now) { - next_timer = timer->nextrun - now; + if (timer == prev) + break; + + if (timer->nextrun > now) break; - } TAILQ_REMOVE(&kore_timers, timer, list); timer->cb(timer->arg, now); @@ -79,6 +94,7 @@ if (timer->flags & KORE_TIMER_ONESHOT) { kore_free(timer); } else { + prev = timer; timer->nextrun = now + timer->interval; TAILQ_FOREACH(t, &kore_timers, list) { if (t->nextrun > timer->nextrun) { @@ -91,9 +107,4 @@ TAILQ_INSERT_TAIL(&kore_timers, timer, list); } } - - if (next_timer > 1) - next_timer -= 1; - - return (next_timer); } diff -Nru kore-2.0.0/src/utils.c kore-3.3.1/src/utils.c --- kore-2.0.0/src/utils.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/utils.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include @@ -45,8 +46,11 @@ { NULL, 0 }, }; +static void fatal_log(const char *, va_list); + static char b64table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +#if defined(KORE_DEBUG) void kore_debug_internal(char *file, int line, const char *fmt, ...) { @@ -59,6 +63,7 @@ printf("[%d] %s:%d - %s\n", kore_pid, file, line, buf); } +#endif void kore_log_init(void) @@ -223,8 +228,40 @@ return ((sign) ? (u_int64_t)ll : l); } +double +kore_strtodouble(const char *str, long double min, long double max, int *err) +{ + double d; + char *ep; + + if (min > max) { + *err = KORE_RESULT_ERROR; + return (0); + } + + errno = 0; + d = strtod(str, &ep); + if (d == 0 || errno == ERANGE || str == ep || *ep != '\0') { + *err = KORE_RESULT_ERROR; + return (0); + } + + if (d < min) { + *err = KORE_RESULT_ERROR; + return (0); + } + + if (d > max) { + *err = KORE_RESULT_ERROR; + return (0); + } + + *err = KORE_RESULT_OK; + return (d); +} + int -kore_split_string(char *input, char *delim, char **out, size_t ele) +kore_split_string(char *input, const char *delim, char **out, size_t ele) { int count; char **ap; @@ -266,7 +303,7 @@ } time_t -kore_date_to_time(char *http_date) +kore_date_to_time(const char *http_date) { time_t t; int err, i; @@ -368,83 +405,82 @@ u_int64_t kore_time_ms(void) { - struct timeval tv; + struct timespec ts; - if (gettimeofday(&tv, NULL) == -1) - return (0); + (void)clock_gettime(CLOCK_MONOTONIC, &ts); - return (tv.tv_sec * 1000 + (tv.tv_usec / 1000)); + return ((u_int64_t)(ts.tv_sec * 1000 + (ts.tv_nsec / 1000000))); } int -kore_base64_encode(u_int8_t *data, size_t len, char **out) +kore_base64_encode(const void *data, size_t len, char **out) { - u_int32_t b; - struct kore_buf *res; - size_t plen, idx; - u_int8_t n, *pdata; - int i, padding; - - if ((len % 3) != 0) { - padding = 3 - (len % 3); - plen = len + padding; - if (plen < len) - fatal("plen wrapped"); - - pdata = kore_malloc(plen); - memcpy(pdata, data, len); - memset(pdata + len, 0, padding); - } else { - plen = len; - padding = 0; - pdata = data; - } - - res = kore_buf_alloc(plen); - - i = 2; - b = 0; - for (idx = 0; idx < plen; idx++) { - b |= (pdata[idx] << (i * 8)); - if (i-- == 0) { - for (i = 3; i >= 0; i--) { - n = (b >> (6 * i)) & 0x3f; - if (n >= sizeof(b64table)) { - kore_debug("unable to encode %d", n); - kore_buf_free(res); - return (KORE_RESULT_ERROR); - } - - if (idx >= len && i < padding) - break; + u_int8_t n; + size_t nb; + const u_int8_t *ptr; + u_int32_t bytes; + struct kore_buf result; + + nb = 0; + ptr = data; + kore_buf_init(&result, (len / 3) * 4); + + while (len > 0) { + if (len > 2) { + nb = 3; + bytes = *ptr++ << 16; + bytes |= *ptr++ << 8; + bytes |= *ptr++; + } else if (len > 1) { + nb = 2; + bytes = *ptr++ << 16; + bytes |= *ptr++ << 8; + } else if (len == 1) { + nb = 1; + bytes = *ptr++ << 16; + } else { + kore_buf_cleanup(&result); + return (KORE_RESULT_ERROR); + } - kore_buf_append(res, &(b64table[n]), 1); + n = (bytes >> 18) & 0x3f; + kore_buf_append(&result, &(b64table[n]), 1); + n = (bytes >> 12) & 0x3f; + kore_buf_append(&result, &(b64table[n]), 1); + if (nb > 1) { + n = (bytes >> 6) & 0x3f; + kore_buf_append(&result, &(b64table[n]), 1); + if (nb > 2) { + n = bytes & 0x3f; + kore_buf_append(&result, &(b64table[n]), 1); } - - b = 0; - i = 2; } - } - for (i = 0; i < padding; i++) - kore_buf_append(res, (u_int8_t *)"=", 1); - - if (pdata != data) - kore_free(pdata); + len -= nb; + } - pdata = kore_buf_release(res, &plen); - if ((plen + 1) < plen) - fatal("plen wrapped"); + switch (nb) { + case 1: + kore_buf_appendf(&result, "=="); + break; + case 2: + kore_buf_appendf(&result, "="); + break; + case 3: + break; + default: + kore_buf_cleanup(&result); + return (KORE_RESULT_ERROR); + } - *out = kore_malloc(plen + 1); - (void)kore_strlcpy(*out, (char *)pdata, plen + 1); - kore_free(pdata); + /* result.data gets taken over so no need to cleanup result. */ + *out = kore_buf_stringify(&result, NULL); return (KORE_RESULT_OK); } int -kore_base64_decode(char *in, u_int8_t **out, size_t *olen) +kore_base64_decode(const char *in, u_int8_t **out, size_t *olen) { int i, c; struct kore_buf *res; @@ -508,12 +544,12 @@ } void * -kore_mem_find(void *src, size_t slen, void *needle, size_t len) +kore_mem_find(void *src, size_t slen, const void *needle, size_t len) { size_t pos; for (pos = 0; pos < slen; pos++) { - if ( *((u_int8_t *)src + pos) != *(u_int8_t *)needle) + if ( *((u_int8_t *)src + pos) != *(const u_int8_t *)needle) continue; if ((slen - pos) < len) @@ -535,10 +571,10 @@ return (string); end = (string + len) - 1; - while (isspace(*string) && string < end) + while (isspace(*(unsigned char *)string) && string < end) string++; - while (isspace(*end) && end > string) + while (isspace(*(unsigned char *)end) && end > string) *(end)-- = '\0'; return (string); @@ -555,7 +591,7 @@ p = in; in[strcspn(in, "\n")] = '\0'; - while (isspace(*p)) + while (isspace(*(unsigned char *)p)) p++; if (p[0] == '#' || p[0] == '\0') { @@ -574,22 +610,46 @@ void fatal(const char *fmt, ...) { - va_list args; - char buf[2048]; - extern const char *__progname; + va_list args; va_start(args, fmt); - (void)vsnprintf(buf, sizeof(buf), fmt, args); + fatal_log(fmt, args); va_end(args); + exit(1); +} + +void +fatalx(const char *fmt, ...) +{ + va_list args; + + /* In case people call fatalx() from the parent context. */ + if (worker != NULL) + kore_msg_send(KORE_MSG_PARENT, KORE_MSG_SHUTDOWN, NULL, 0); + + va_start(args, fmt); + fatal_log(fmt, args); + va_end(args); + + exit(1); +} + +static void +fatal_log(const char *fmt, va_list args) +{ + char buf[2048]; + extern const char *kore_progname; + + (void)vsnprintf(buf, sizeof(buf), fmt, args); + if (!foreground) kore_log(LOG_ERR, "%s", buf); #if !defined(KORE_NO_TLS) if (worker != NULL && worker->id == KORE_WORKER_KEYMGR) - kore_keymgr_cleanup(); + kore_keymgr_cleanup(1); #endif - printf("%s: %s\n", __progname, buf); - exit(1); + printf("%s: %s\n", kore_progname, buf); } diff -Nru kore-2.0.0/src/validator.c kore-3.3.1/src/validator.c --- kore-2.0.0/src/validator.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/validator.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -14,6 +14,8 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include + #include "kore.h" TAILQ_HEAD(, kore_validator) validators; @@ -27,6 +29,7 @@ int kore_validator_add(const char *name, u_int8_t type, const char *arg) { + int ret; struct kore_validator *val; val = kore_malloc(sizeof(*val)); @@ -34,16 +37,18 @@ switch (val->type) { case KORE_VALIDATOR_TYPE_REGEX: - if (regcomp(&(val->rctx), arg, REG_EXTENDED | REG_NOSUB)) { + ret = regcomp(&(val->rctx), arg, REG_EXTENDED | REG_NOSUB); + if (ret) { kore_free(val); kore_log(LOG_NOTICE, - "validator %s has bad regex %s", name, arg); + "validator %s has bad regex %s (%d)", + name, arg, ret); return (KORE_RESULT_ERROR); } break; case KORE_VALIDATOR_TYPE_FUNCTION: - *(void **)(&val->func) = kore_module_getsym(arg); - if (val->func == NULL) { + val->rcall = kore_runtime_getcall(arg); + if (val->rcall == NULL) { kore_free(val); kore_log(LOG_NOTICE, "validator %s has undefined callback %s", @@ -80,7 +85,7 @@ int kore_validator_check(struct http_request *req, struct kore_validator *val, - void *data) + const void *data) { int r; @@ -92,7 +97,7 @@ r = KORE_RESULT_ERROR; break; case KORE_VALIDATOR_TYPE_FUNCTION: - r = val->func(req, data); + r = kore_runtime_validator(val->rcall, req, data); break; default: r = KORE_RESULT_ERROR; @@ -113,9 +118,10 @@ if (val->type != KORE_VALIDATOR_TYPE_FUNCTION) continue; - *(void **)&(val->func) = kore_module_getsym(val->arg); - if (val->func == NULL) - fatal("no function for validator %s found", val->name); + kore_free(val->rcall); + val->rcall = kore_runtime_getcall(val->arg); + if (val->rcall == NULL) + fatal("no function for validator %s found", val->arg); } } diff -Nru kore-2.0.0/src/websocket.c kore-3.3.1/src/websocket.c --- kore-2.0.0/src/websocket.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/websocket.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Joris Vink + * Copyright (c) 2014-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,6 +15,7 @@ */ #include +#include #include @@ -49,11 +50,13 @@ const void *, size_t); void -kore_websocket_handshake(struct http_request *req, struct kore_wscbs *wscbs) +kore_websocket_handshake(struct http_request *req, const char *onconnect, + const char *onmessage, const char *ondisconnect) { SHA_CTX sctx; struct kore_buf *buf; - char *key, *base64, *version; + char *base64; + const char *key, *version; u_int8_t digest[SHA_DIGEST_LENGTH]; if (!http_request_header(req, "sec-websocket-key", &key)) { @@ -102,24 +105,61 @@ req->owner->disconnect = websocket_disconnect; req->owner->rnb->flags &= ~NETBUF_CALL_CB_ALWAYS; - req->owner->wscbs = wscbs; + req->owner->http_timeout = 0; req->owner->idle_timer.start = kore_time_ms(); req->owner->idle_timer.length = kore_websocket_timeout; - if (wscbs->connect != NULL) - wscbs->connect(req->owner); + if (onconnect != NULL) { + req->owner->ws_connect = kore_runtime_getcall(onconnect); + if (req->owner->ws_connect == NULL) + fatal("no symbol '%s' for ws_connect", onconnect); + } else { + req->owner->ws_connect = NULL; + } + + if (onmessage != NULL) { + req->owner->ws_message = kore_runtime_getcall(onmessage); + if (req->owner->ws_message == NULL) + fatal("no symbol '%s' for ws_message", onmessage); + } else { + req->owner->ws_message = NULL; + } + + if (ondisconnect != NULL) { + req->owner->ws_disconnect = kore_runtime_getcall(ondisconnect); + if (req->owner->ws_disconnect == NULL) + fatal("no symbol '%s' for ws_disconnect", ondisconnect); + } else { + req->owner->ws_disconnect = NULL; + } + + if (req->owner->ws_connect != NULL) + kore_runtime_wsconnect(req->owner->ws_connect, req->owner); +} + +int +kore_websocket_send_clean(struct netbuf *nb) +{ + kore_free(nb->buf); + return (0); } void kore_websocket_send(struct connection *c, u_int8_t op, const void *data, size_t len) { - struct kore_buf *frame; + struct kore_buf frame; - frame = kore_buf_alloc(len); - websocket_frame_build(frame, op, data, len); - net_send_queue(c, frame->data, frame->offset); - kore_buf_free(frame); + kore_buf_init(&frame, len); + websocket_frame_build(&frame, op, data, len); + net_send_stream(c, frame.data, frame.offset, + kore_websocket_send_clean, NULL); + + /* net_send_stream() takes over the buffer data pointer. */ + frame.data = NULL; + kore_buf_cleanup(&frame); + + net_send_flush(c); } void @@ -156,7 +196,7 @@ u_int64_t len64; if (len > WEBSOCKET_PAYLOAD_SINGLE) { - if (len < USHRT_MAX) + if (len <= USHRT_MAX) len_1 = WEBSOCKET_PAYLOAD_EXTEND_1; else len_1 = WEBSOCKET_PAYLOAD_EXTEND_2; @@ -183,7 +223,8 @@ } } - kore_buf_append(frame, data, len); + if (data != NULL && len > 0) + kore_buf_append(frame, data, len); } static int @@ -198,7 +239,7 @@ } if (WEBSOCKET_RSV(nb->buf[0], 1) || WEBSOCKET_RSV(nb->buf[0], 2) || - WEBSOCKET_RSV(nb->buf[0], 2)) { + WEBSOCKET_RSV(nb->buf[0], 3)) { kore_debug("%p: RSV bits are not zero", c); return (KORE_RESULT_ERROR); } @@ -245,13 +286,10 @@ { struct connection *c; int ret; - struct kore_wscbs *wscbs; u_int64_t len, i, total; u_int8_t op, moff, extra; c = nb->owner; - wscbs = c->wscbs; - op = nb->buf[0] & WEBSOCKET_OPCODE_MASK; len = WEBSOCKET_FRAME_LENGTH(nb->buf[1]); @@ -293,17 +331,26 @@ ret = KORE_RESULT_OK; switch (op) { - case WEBSOCKET_OP_CONT: case WEBSOCKET_OP_PONG: + break; + case WEBSOCKET_OP_CONT: ret = KORE_RESULT_ERROR; - kore_log(LOG_ERR, "%p: we do not support op 0x%02x yet", c, op); + kore_log(LOG_ERR, + "%p: we do not support op 0x%02x yet", (void *)c, op); break; case WEBSOCKET_OP_TEXT: case WEBSOCKET_OP_BINARY: - if (wscbs->message != NULL) - wscbs->message(c, op, &nb->buf[moff + 4], len); + if (c->ws_message != NULL) { + kore_runtime_wsmessage(c->ws_message, + c, op, &nb->buf[moff + 4], len); + } break; case WEBSOCKET_OP_CLOSE: + c->evt.flags &= ~KORE_EVENT_READ; + if (!(c->flags & CONN_WS_CLOSE_SENT)) { + c->flags |= CONN_WS_CLOSE_SENT; + kore_websocket_send(c, WEBSOCKET_OP_CLOSE, NULL, 0); + } kore_connection_disconnect(c); break; case WEBSOCKET_OP_PING: @@ -316,14 +363,19 @@ } net_recv_reset(c, WEBSOCKET_FRAME_HDR, websocket_recv_opcode); + return (ret); } static void websocket_disconnect(struct connection *c) { - struct kore_wscbs *wscbs = c->wscbs; + if (c->ws_disconnect != NULL) + kore_runtime_wsdisconnect(c->ws_disconnect, c); - if (wscbs->disconnect != NULL) - wscbs->disconnect(c); + if (!(c->flags & CONN_WS_CLOSE_SENT)) { + c->flags |= CONN_WS_CLOSE_SENT; + c->evt.flags &= ~KORE_EVENT_READ; + kore_websocket_send(c, WEBSOCKET_OP_CLOSE, NULL, 0); + } } diff -Nru kore-2.0.0/src/worker.c kore-3.3.1/src/worker.c --- kore-2.0.0/src/worker.c 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/src/worker.c 2019-06-03 13:29:24.000000000 +0000 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Joris Vink + * Copyright (c) 2013-2019 Joris Vink * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,12 +15,17 @@ */ #include +#include #include #include #include #include #include +#if !defined(KORE_NO_TLS) +#include +#endif + #include #include #include @@ -40,18 +45,19 @@ #include "tasks.h" #endif -#if defined(WORKER_DEBUG) -#define worker_debug(fmt, ...) printf(fmt, ##__VA_ARGS__) -#else -#define worker_debug(fmt, ...) +#if defined(KORE_USE_PYTHON) +#include "python_api.h" #endif #if !defined(WAIT_ANY) #define WAIT_ANY (-1) #endif -#define KORE_SHM_KEY 15000 -#define WORKER_LOCK_TIMEOUT 500 +#if !defined(KORE_NO_TLS) +#define WORKER_SOLO_COUNT 2 +#else +#define WORKER_SOLO_COUNT 1 +#endif #define WORKER(id) \ (struct kore_worker *)((u_int8_t *)kore_workers + \ @@ -65,29 +71,42 @@ static int worker_trylock(void); static void worker_unlock(void); -static inline int kore_worker_acceptlock_obtain(void); -static inline void kore_worker_acceptlock_release(void); +static inline int worker_acceptlock_obtain(void); +static inline void worker_acceptlock_release(void); +static void worker_accept_avail(struct kore_msg *, const void *); + +#if !defined(KORE_NO_TLS) +static void worker_entropy_recv(struct kore_msg *, const void *); +static void worker_keymgr_response(struct kore_msg *, const void *); +static int worker_keymgr_response_verify(struct kore_msg *, const void *, + struct kore_domain **); +#endif +static int accept_avail; static struct kore_worker *kore_workers; +static int worker_no_lock; static int shm_accept_key; static struct wlock *accept_lock; -extern volatile sig_atomic_t sig_recv; struct kore_worker *worker = NULL; u_int8_t worker_set_affinity = 1; -u_int32_t worker_accept_threshold = 0; -u_int32_t worker_rlimit_nofiles = 1024; -u_int32_t worker_max_connections = 250; +u_int32_t worker_accept_threshold = 16; +u_int32_t worker_rlimit_nofiles = 768; +u_int32_t worker_max_connections = 512; u_int32_t worker_active_connections = 0; +int worker_policy = KORE_WORKER_POLICY_RESTART; void kore_worker_init(void) { size_t len; + struct kore_worker *kw; u_int16_t i, cpu; + worker_no_lock = 0; + if (worker_count == 0) - worker_count = 1; + worker_count = cpu_count; #if !defined(KORE_NO_TLS) /* account for the key manager. */ @@ -117,6 +136,12 @@ kore_debug("kore_worker_init(): more workers than cpu's"); } + /* Setup log buffers. */ + for (i = 0; i < worker_count; i++) { + kw = WORKER(i); + kw->lb.offset = 0; + } + cpu = 0; for (i = 0; i < worker_count; i++) { kore_worker_spawn(i, cpu++); @@ -167,16 +192,30 @@ kore_worker_shutdown(void) { struct kore_worker *kw; + pid_t pid; + int status; u_int16_t id, done; - kore_log(LOG_NOTICE, "waiting for workers to drain and shutdown"); + if (!kore_quiet) { + kore_log(LOG_NOTICE, + "waiting for workers to drain and shutdown"); + } + for (;;) { + for (id = 0; id < worker_count; id++) { + kw = WORKER(id); + if (kw->pid != 0) { + pid = waitpid(kw->pid, &status, 0); + if (pid == -1) + continue; + kw->pid = 0; + } + } + done = 0; for (id = 0; id < worker_count; id++) { kw = WORKER(id); - if (kw->pid != 0) - kore_worker_wait(1); - else + if (kw->pid == 0) done++; } @@ -205,29 +244,37 @@ } void -kore_worker_privdrop(void) +kore_worker_privdrop(const char *runas, const char *root) { rlim_t fd; struct rlimit rl; struct passwd *pw = NULL; + if (root == NULL) + fatalx("no root directory for kore_worker_privdrop"); + /* Must happen before chroot. */ if (skip_runas == 0) { - pw = getpwnam(runas_user); + if (runas == NULL) + fatalx("no runas user given and -r not specified"); + pw = getpwnam(runas); if (pw == NULL) { - fatal("cannot getpwnam(\"%s\") runas user: %s", - runas_user, errno_s); + fatalx("cannot getpwnam(\"%s\") for user: %s", + runas, errno_s); } } if (skip_chroot == 0) { - if (chroot(chroot_path) == -1) { - fatal("cannot chroot(\"%s\"): %s", - chroot_path, errno_s); + if (chroot(root) == -1) { + fatalx("cannot chroot(\"%s\"): %s", + root, errno_s); } if (chdir("/") == -1) - fatal("cannot chdir(\"/\"): %s", errno_s); + fatalx("cannot chdir(\"/\"): %s", errno_s); + } else { + if (chdir(root) == -1) + fatalx("cannot chdir(\"%s\"): %s", root, errno_s); } if (getrlimit(RLIMIT_NOFILE, &rl) == -1) { @@ -243,7 +290,7 @@ rl.rlim_cur = worker_rlimit_nofiles; rl.rlim_max = worker_rlimit_nofiles; if (setrlimit(RLIMIT_NOFILE, &rl) == -1) { - kore_log(LOG_ERR, "setrlimit(RLIMIT_NOFILE, %d): %s", + kore_log(LOG_ERR, "setrlimit(RLIMIT_NOFILE, %u): %s", worker_rlimit_nofiles, errno_s); } @@ -256,26 +303,32 @@ setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) || setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid)) #endif - fatal("cannot drop privileges"); + fatalx("cannot drop privileges"); } + +#if defined(KORE_USE_PLATFORM_PLEDGE) + kore_platform_pledge(); +#endif + } void kore_worker_entry(struct kore_worker *kw) { - char buf[16]; - int quit, had_lock, r; - u_int64_t now, idle_check, next_lock, netwait; -#if defined(KORE_SINGLE_BINARY) - void (*onload)(void); + struct kore_runtime_call *rcall; + char buf[16]; + int quit, had_lock; + u_int64_t netwait, now, next_prune; +#if !defined(KORE_NO_TLS) + u_int64_t last_seed; #endif worker = kw; - (void)snprintf(buf, sizeof(buf), "kore [wrk %d]", kw->id); + (void)snprintf(buf, sizeof(buf), "[wrk %d]", kw->id); #if !defined(KORE_NO_TLS) if (kw->id == KORE_WORKER_KEYMGR) - (void)snprintf(buf, sizeof(buf), "kore [keymgr]"); + (void)snprintf(buf, sizeof(buf), "[keymgr]"); #endif kore_platform_proctitle(buf); @@ -284,16 +337,7 @@ kore_pid = kw->pid; - sig_recv = 0; - signal(SIGHUP, kore_signal); - signal(SIGQUIT, kore_signal); - signal(SIGTERM, kore_signal); - signal(SIGPIPE, SIG_IGN); - - if (foreground) - signal(SIGINT, kore_signal); - else - signal(SIGINT, SIG_IGN); + kore_signal_setup(); #if !defined(KORE_NO_TLS) if (kw->id == KORE_WORKER_KEYMGR) { @@ -301,69 +345,81 @@ exit(0); } #endif + net_init(); + kore_connection_init(); + kore_platform_event_init(); + kore_msg_worker_init(); - kore_worker_privdrop(); + kore_worker_privdrop(kore_runas_user, kore_root_path); - net_init(); #if !defined(KORE_NO_HTTP) http_init(); + kore_filemap_resolve_paths(); kore_accesslog_worker_init(); #endif kore_timer_init(); - kore_connection_init(); - kore_domain_load_crl(); + kore_fileref_init(); kore_domain_keymgr_init(); quit = 0; had_lock = 0; - next_lock = 0; - idle_check = 0; - kore_platform_event_init(); - kore_msg_worker_init(); + next_prune = 0; + accept_avail = 1; + worker_active_connections = 0; #if defined(KORE_USE_PGSQL) - kore_pgsql_init(); + kore_pgsql_sys_init(); #endif #if defined(KORE_USE_TASKS) kore_task_init(); #endif - kore_log(LOG_NOTICE, "worker %d started (cpu#%d)", kw->id, kw->cpu); +#if !defined(KORE_NO_TLS) + last_seed = 0; + kore_msg_register(KORE_MSG_CRL, worker_keymgr_response); + kore_msg_register(KORE_MSG_ENTROPY_RESP, worker_entropy_recv); + kore_msg_register(KORE_MSG_CERTIFICATE, worker_keymgr_response); + if (worker->restarted) { + kore_msg_send(KORE_WORKER_KEYMGR, + KORE_MSG_CERTIFICATE_REQ, NULL, 0); + } +#endif + + kore_msg_register(KORE_MSG_ACCEPT_AVAILABLE, worker_accept_avail); + + if (nlisteners == 0) + worker_no_lock = 1; + + if (!kore_quiet) { + kore_log(LOG_NOTICE, + "worker %d started (cpu#%d, pid#%d)", + kw->id, kw->cpu, kw->pid); + } + + rcall = kore_runtime_getcall("kore_worker_configure"); + if (rcall != NULL) { + kore_runtime_execute(rcall); + kore_free(rcall); + } -#if defined(KORE_SINGLE_BINARY) - *(void **)&(onload) = kore_module_getsym("kore_onload"); - if (onload != NULL) - onload(); -#else kore_module_onload(); -#endif + worker->restarted = 0; for (;;) { - if (sig_recv != 0) { - switch (sig_recv) { - case SIGHUP: -#if !defined(KORE_SINGLE_BINARY) - kore_module_reload(1); -#endif - break; - case SIGQUIT: - case SIGINT: - case SIGTERM: - quit = 1; - break; - default: - break; - } + now = kore_time_ms(); - sig_recv = 0; +#if !defined(KORE_NO_TLS) + if ((now - last_seed) > KORE_RESEED_TIME) { + kore_msg_send(KORE_WORKER_KEYMGR, + KORE_MSG_ENTROPY_REQ, NULL, 0); + last_seed = now; } +#endif - now = kore_time_ms(); - netwait = kore_timer_run(now); - - if (now > next_lock) { - if (kore_worker_acceptlock_obtain()) { + if (!worker->has_lock && accept_avail) { + accept_avail = 0; + if (worker_acceptlock_obtain()) { if (had_lock == 0) { kore_platform_enable_accept(); had_lock = 1; @@ -371,6 +427,17 @@ } } + netwait = kore_timer_next_run(now); + + if (netwait == KORE_WAIT_INFINITE && http_request_count > 0) + netwait = 100; + + kore_platform_event_wait(netwait); + now = kore_time_ms(); + + if (worker->has_lock) + worker_acceptlock_release(); + if (!worker->has_lock) { if (had_lock == 1) { had_lock = 0; @@ -378,25 +445,51 @@ } } - r = kore_platform_event_wait(netwait); - if (worker->has_lock && r > 0) { - kore_worker_acceptlock_release(); - next_lock = now + WORKER_LOCK_TIMEOUT; + if (sig_recv != 0) { + switch (sig_recv) { + case SIGHUP: + kore_module_reload(1); + break; + case SIGQUIT: + case SIGINT: + case SIGTERM: + quit = 1; + break; + case SIGCHLD: +#if defined(KORE_USE_PYTHON) + kore_python_proc_reap(); +#endif + break; + default: + break; + } + + sig_recv = 0; } + if (quit) + break; + + kore_timer_run(now); + #if !defined(KORE_NO_HTTP) http_process(); #endif +#if defined(KORE_USE_PYTHON) + kore_python_coro_run(); +#endif - if ((now - idle_check) >= 10000) { - idle_check = now; - kore_connection_check_timeout(); + if (next_prune <= now) { + kore_connection_check_timeout(now); + kore_connection_prune(KORE_CONNECTION_PRUNE_DISCONNECT); + next_prune = now + 500; } + } - kore_connection_prune(KORE_CONNECTION_PRUNE_DISCONNECT); - - if (quit) - break; + rcall = kore_runtime_getcall("kore_worker_teardown"); + if (rcall != NULL) { + kore_runtime_execute(rcall); + kore_free(rcall); } kore_platform_event_cleanup(); @@ -408,26 +501,43 @@ #endif net_cleanup(); +#if defined(KORE_USE_PYTHON) + kore_python_cleanup(); +#endif + +#if defined(KORE_USE_PGSQL) + kore_pgsql_sys_cleanup(); +#endif + kore_debug("worker %d shutting down", kw->id); + + kore_mem_cleanup(); exit(0); } void -kore_worker_wait(int final) +kore_worker_reap(void) { u_int16_t id; pid_t pid; struct kore_worker *kw; + const char *func; int status; - if (final) - pid = waitpid(WAIT_ANY, &status, 0); - else + for (;;) { pid = waitpid(WAIT_ANY, &status, WNOHANG); - if (pid == -1) { - kore_debug("waitpid(): %s", errno_s); - return; + if (pid == -1) { + if (errno == ECHILD) + return; + if (errno == EINTR) + continue; + kore_log(LOG_ERR, + "failed to wait for children: %s", errno_s); + return; + } + + break; } if (pid == 0) @@ -438,81 +548,119 @@ if (kw->pid != pid) continue; - kore_log(LOG_NOTICE, "worker %d (%d)-> status %d", - kw->id, pid, status); - - if (final) { - kw->pid = 0; - break; + if (!kore_quiet) { + kore_log(LOG_NOTICE, + "worker %d (%d) exited with status %d", + kw->id, pid, status); } - if (WEXITSTATUS(status) || WTERMSIG(status) || - WCOREDUMP(status)) { - kore_log(LOG_NOTICE, - "worker %d (pid: %d) (hdlr: %s) gone", - kw->id, kw->pid, - (kw->active_hdlr != NULL) ? kw->active_hdlr->func : - "none"); + func = "none"; +#if !defined(KORE_NO_HTTP) + if (kw->active_hdlr != NULL) + func = kw->active_hdlr->func; +#endif + kore_log(LOG_NOTICE, + "worker %d (pid: %d) (hdlr: %s) gone", + kw->id, kw->pid, func); #if !defined(KORE_NO_TLS) - if (id == KORE_WORKER_KEYMGR) { - kore_log(LOG_CRIT, "keymgr gone, stopping"); - kw->pid = 0; - if (raise(SIGTERM) != 0) { - kore_log(LOG_WARNING, - "failed to raise SIGTERM signal"); - } - break; + if (id == KORE_WORKER_KEYMGR) { + kore_log(LOG_CRIT, "keymgr gone, stopping"); + kw->pid = 0; + if (raise(SIGTERM) != 0) { + kore_log(LOG_WARNING, + "failed to raise SIGTERM signal"); } + break; + } #endif - if (kw->pid == accept_lock->current) - worker_unlock(); + if (kw->pid == accept_lock->current && + worker_no_lock == 0) + worker_unlock(); - if (kw->active_hdlr != NULL) { - kw->active_hdlr->errors++; - kore_log(LOG_NOTICE, - "hdlr %s has caused %d error(s)", - kw->active_hdlr->func, - kw->active_hdlr->errors); - } +#if !defined(KORE_NO_HTTP) + if (kw->active_hdlr != NULL) { + kw->active_hdlr->errors++; + kore_log(LOG_NOTICE, + "hdlr %s has caused %d error(s)", + kw->active_hdlr->func, + kw->active_hdlr->errors); + } +#endif - kore_log(LOG_NOTICE, "restarting worker %d", kw->id); - kore_msg_parent_remove(kw); - kore_worker_spawn(kw->id, kw->cpu); - kore_msg_parent_add(kw); - } else { + if (worker_policy == KORE_WORKER_POLICY_TERMINATE) { + kw->pid = 0; kore_log(LOG_NOTICE, - "worker %d (pid: %d) signaled us (%d)", - kw->id, kw->pid, status); + "worker policy is 'terminate', stopping"); + if (raise(SIGTERM) != 0) { + kore_log(LOG_WARNING, + "failed to raise SIGTERM signal"); + } + break; } + kore_log(LOG_NOTICE, "restarting worker %d", kw->id); + kw->restarted = 1; + kore_msg_parent_remove(kw); + kore_worker_spawn(kw->id, kw->cpu); + kore_msg_parent_add(kw); + break; } } +void +kore_worker_make_busy(void) +{ + if (worker_count == WORKER_SOLO_COUNT || worker_no_lock == 1) + return; + + if (worker->has_lock) { + worker_unlock(); + worker->has_lock = 0; + kore_msg_send(KORE_MSG_WORKER_ALL, + KORE_MSG_ACCEPT_AVAILABLE, NULL, 0); + } +} + static inline void -kore_worker_acceptlock_release(void) +worker_acceptlock_release(void) { - if (worker_count == 1) + if (worker_count == WORKER_SOLO_COUNT || worker_no_lock == 1) return; if (worker->has_lock != 1) return; + if (worker_active_connections < worker_max_connections) { +#if !defined(KORE_NO_HTTP) + if (http_request_count < http_request_limit) + return; +#else + return; +#endif + } + +#if defined(WORKER_DEBUG) + kore_log(LOG_DEBUG, "worker busy, releasing lock"); +#endif + worker_unlock(); worker->has_lock = 0; + + kore_msg_send(KORE_MSG_WORKER_ALL, KORE_MSG_ACCEPT_AVAILABLE, NULL, 0); } static inline int -kore_worker_acceptlock_obtain(void) +worker_acceptlock_obtain(void) { int r; if (worker->has_lock == 1) return (1); - if (worker_count == 1) { + if (worker_count == WORKER_SOLO_COUNT || worker_no_lock == 1) { worker->has_lock = 1; return (1); } @@ -520,10 +668,18 @@ if (worker_active_connections >= worker_max_connections) return (0); +#if !defined(KORE_NO_HTTP) + if (http_request_count >= http_request_limit) + return (0); +#endif + r = 0; if (worker_trylock()) { r = 1; worker->has_lock = 1; +#if defined(WORKER_DEBUG) + kore_log(LOG_DEBUG, "got lock"); +#endif } return (r); @@ -535,8 +691,6 @@ if (!__sync_bool_compare_and_swap(&(accept_lock->lock), 0, 1)) return (0); - worker_debug("wrk#%d grabbed lock (%d/%d)\n", worker->id, - worker_active_connections, worker_max_connections); accept_lock->current = worker->pid; return (1); @@ -549,3 +703,92 @@ if (!__sync_bool_compare_and_swap(&(accept_lock->lock), 1, 0)) kore_log(LOG_NOTICE, "worker_unlock(): wasnt locked"); } + +static void +worker_accept_avail(struct kore_msg *msg, const void *data) +{ + accept_avail = 1; +} + +#if !defined(KORE_NO_TLS) +static void +worker_entropy_recv(struct kore_msg *msg, const void *data) +{ + if (msg->length != 1024) { + kore_log(LOG_WARNING, + "invalid entropy response (got:%zu - wanted:1024)", + msg->length); + } + + RAND_poll(); + RAND_seed(data, msg->length); +} + +static void +worker_keymgr_response(struct kore_msg *msg, const void *data) +{ + struct kore_domain *dom; + const struct kore_x509_msg *req; + + if (!worker_keymgr_response_verify(msg, data, &dom)) + return; + + req = (const struct kore_x509_msg *)data; + + switch (msg->id) { + case KORE_MSG_CERTIFICATE: + kore_domain_tlsinit(dom, req->data, req->data_len); + break; + case KORE_MSG_CRL: + kore_domain_crl_add(dom, req->data, req->data_len); + break; + default: + kore_log(LOG_WARNING, "unknown keymgr request %u", msg->id); + break; + } +} + +static int +worker_keymgr_response_verify(struct kore_msg *msg, const void *data, + struct kore_domain **out) +{ + struct kore_domain *dom; + const struct kore_x509_msg *req; + + if (msg->length < sizeof(*req)) { + kore_log(LOG_WARNING, + "short keymgr message (%zu)", msg->length); + return (KORE_RESULT_ERROR); + } + + req = (const struct kore_x509_msg *)data; + if (msg->length != (sizeof(*req) + req->data_len)) { + kore_log(LOG_WARNING, + "invalid keymgr payload (%zu)", msg->length); + return (KORE_RESULT_ERROR); + } + + if (req->domain_len > KORE_DOMAINNAME_LEN) { + kore_log(LOG_WARNING, + "invalid keymgr domain (%u)", + req->domain_len); + return (KORE_RESULT_ERROR); + } + + dom = NULL; + TAILQ_FOREACH(dom, &domains, list) { + if (!strncmp(dom->domain, req->domain, req->domain_len)) + break; + } + + if (dom == NULL) { + kore_log(LOG_WARNING, + "got keymgr response for domain that does not exist"); + return (KORE_RESULT_ERROR); + } + + *out = dom; + + return (KORE_RESULT_OK); +} +#endif diff -Nru kore-2.0.0/.travis.yml kore-3.3.1/.travis.yml --- kore-2.0.0/.travis.yml 2016-08-01 07:56:52.000000000 +0000 +++ kore-3.3.1/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -language: c -sudo: false -compiler: - - clang - - gcc -addons: - apt: - packages: - - libssl-dev -script: - - make