diff -Nru kafkacat-1.5.0/avro.c kafkacat-1.6.0/avro.c --- kafkacat-1.5.0/avro.c 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/avro.c 2020-07-21 09:00:07.000000000 +0000 @@ -120,35 +120,35 @@ */ char *kc_avro_to_json (const void *data, size_t data_len, char *errstr, size_t errstr_size) { - avro_value_t avro; - serdes_schema_t *schema; - char *json; - serdes_err_t err; + avro_value_t avro; + serdes_schema_t *schema; + char *json; + serdes_err_t err; - err = serdes_deserialize_avro(serdes, &avro, &schema, data, data_len, - errstr, errstr_size); - if (err) { - if (err == SERDES_ERR_FRAMING_INVALID || - strstr(errstr, "Invalid CP1 magic byte")) { - static const char badframing[] = - ": message not produced with " - "Schema-Registry Avro framing"; - int len = strlen(errstr); + err = serdes_deserialize_avro(serdes, &avro, &schema, data, data_len, + errstr, errstr_size); + if (err) { + if (err == SERDES_ERR_FRAMING_INVALID || + strstr(errstr, "Invalid CP1 magic byte")) { + static const char badframing[] = + ": message not produced with " + "Schema-Registry Avro framing"; + int len = strlen(errstr); - if (len + sizeof(badframing) < errstr_size) - snprintf(errstr+len, errstr_size-len, - "%s", badframing); - } - return NULL; - } + if (len + sizeof(badframing) < errstr_size) + snprintf(errstr+len, errstr_size-len, + "%s", badframing); + } + return NULL; + } - if (avro_value_to_json(&avro, 1/*one-line*/, &json)) { - snprintf(errstr, errstr_size, "Failed to encode Avro as JSON"); - avro_value_decref(&avro); - return NULL; - } + if (avro_value_to_json(&avro, 1/*one-line*/, &json)) { + snprintf(errstr, errstr_size, "Failed to encode Avro as JSON"); + avro_value_decref(&avro); + return NULL; + } - avro_value_decref(&avro); + avro_value_decref(&avro); - return json; + return json; } diff -Nru kafkacat-1.5.0/bootstrap.sh kafkacat-1.6.0/bootstrap.sh --- kafkacat-1.5.0/bootstrap.sh 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/bootstrap.sh 2020-07-21 09:00:07.000000000 +0000 @@ -11,7 +11,7 @@ set -o errexit -o nounset -o pipefail -: "${LIBRDKAFKA_VERSION:=v1.1.0}" +: "${LIBRDKAFKA_VERSION:=v1.5.0}" function download { local url=$1 @@ -29,9 +29,18 @@ local dl='curl -s -L' fi + local tar_args= + + # Newer Mac tar's will try to restore metadata/attrs from + # certain tar files (avroc in this case), which fails for whatever reason. + if [[ $(uname -s) == "Darwin" ]] && + tar --no-mac-metadata -h >/dev/null 2>&1; then + tar_args="--no-mac-metadata" + fi + mkdir -p "$dir" pushd "$dir" > /dev/null - ($dl "$url" | tar -xzf - --strip-components 1) || exit 1 + ($dl "$url" | tar -xz $tar_args -f - --strip-components 1) || exit 1 popd > /dev/null } @@ -108,7 +117,7 @@ build librdkafka "([ -f config.h ] || ./configure --prefix=$DEST --install-deps --disable-lz4-ext) && make -j && make install" || (echo "Failed to build librdkafka: bootstrap failed" ; false) github_download "edenhill/yajl" "edenhill" "libyajl" -build libyajl "([ -d build ] || ./configure --prefix $DEST) && make distro && make install" || (echo "Failed to build libyajl: JSON support will probably be disabled" ; true) +build libyajl "([ -d build ] || ./configure --prefix $DEST) && make install" || (echo "Failed to build libyajl: JSON support will probably be disabled" ; true) download http://www.digip.org/jansson/releases/jansson-2.12.tar.gz libjansson build libjansson "([[ -f config.status ]] || ./configure --enable-static --prefix=$DEST) && make && make install" || (echo "Failed to build libjansson: AVRO support will probably be disabled" ; true) diff -Nru kafkacat-1.5.0/CHANGELOG.md kafkacat-1.6.0/CHANGELOG.md --- kafkacat-1.5.0/CHANGELOG.md 1970-01-01 00:00:00.000000000 +0000 +++ kafkacat-1.6.0/CHANGELOG.md 2020-07-21 09:00:07.000000000 +0000 @@ -0,0 +1,35 @@ +# kafkacat v1.6.0 + + * Transactional Producer support (see below). + * Honour `-k ` when producing files (#197). + * Honour `-o ` in `-G` high-level consumer mode (#231). + * Added `-m ` argument to set metadata/query/transaction timeouts. + * Allow `schema.registry.url` to be configured in config file and + not only by `-r` (#220). + * Print broker-id message was produced to (if `-v`), + or was consumed from (if `-J`). + +## Apache Kafke EOS / Transactional Producer support + +Messages can now be produced in a single transaction if `-X transactional.id=..` +is passed to the producer in `-P` mode. + +If kafkacat is terminated through Ctrl-C (or other signal) the transaction +will be aborted, while normal termination (due to stdin closing or after reading +all supplied files) will commit the transaction. + +```bash +$ kafkacat -b $BROKERS -P -t mytopic -X transactional.id=myproducerapp +% Using transactional producer +This is a transactional message +And so is this +:) +[Press Ctrl-D] +% Committing transaction +% Transaction successfully committed +``` + + +# Older releases + +See https://github.com/edenhill/kafkacat/releases diff -Nru kafkacat-1.5.0/configure kafkacat-1.6.0/configure --- kafkacat-1.5.0/configure 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/configure 2020-07-21 09:00:07.000000000 +0000 @@ -81,7 +81,7 @@ shift if [[ $opt = *=* ]]; then - name="${opt%=*}" + name="${opt%%=*}" arg="${opt#*=}" eqarg=1 else @@ -102,10 +102,10 @@ reqarg=$(mkl_meta_get "MKL_OPT_ARGS" "$(mkl_env_esc $name)") if [[ ! -z $reqarg ]]; then if [[ $eqarg == 0 && -z $arg ]]; then - arg=$1 + arg="$1" shift - if [[ -z $arg ]]; then + if [[ -z $arg && $reqarg != '\*' ]]; then mkl_err "Missing argument to option --$name $reqarg" exit 1 fi @@ -167,7 +167,7 @@ ;; *) - opt_$safeopt $arg || exit 1 + opt_$safeopt "$arg" || exit 1 mkl_var_append MKL_OPTS_SET "$safeopt" ;; esac diff -Nru kafkacat-1.5.0/configure.kafkacat kafkacat-1.6.0/configure.kafkacat --- kafkacat-1.5.0/configure.kafkacat 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/configure.kafkacat 1970-01-01 00:00:00.000000000 +0000 @@ -1,65 +0,0 @@ -#!/bin/bash -# - -mkl_require good_cflags -mkl_require gitversion as KAFKACAT_VERSION default 1.5.0 - - -function checks { - - # Check that librdkafka is available, and allow to link it statically. - mkl_meta_set "rdkafka" "desc" "librdkafka is available at http://github.com/edenhill/librdkafka. To quickly download all dependencies and build kafkacat try ./bootstrap.sh" - mkl_meta_set "rdkafka" "deb" "librdkafka-dev" - mkl_lib_check --static=-lrdkafka "rdkafka" "" fail CC "-lrdkafka" \ - "#include " - - # Make sure rdkafka is new enough. - mkl_meta_set "librdkafkaver" "name" "librdkafka metadata API" - mkl_meta_set "librdkafkaver" "desc" "librdkafka 0.8.4 or later is required for the Metadata API" - mkl_compile_check "librdkafkaver" "" fail CC "" \ -"#include -struct rd_kafka_metadata foo;" - - # Enable KafkaConsumer support if librdkafka is new enough - mkl_meta_set "librdkafka_ge_090" "name" "librdkafka KafkaConsumer support" - mkl_compile_check "librdkafka_ge_090" ENABLE_KAFKACONSUMER disable CC "" " - #include - #if RD_KAFKA_VERSION >= 0x00090000 - #else - #error \"rdkafka version < 0.9.0\" - #endif" - - - mkl_meta_set "yajl" "deb" "libyajl-dev" - # Check for JSON library (yajl) - if [[ $WITH_JSON == y ]] && \ - mkl_lib_check --static=-lyajl "yajl" HAVE_YAJL disable CC "-lyajl" \ - "#include -#if YAJL_MAJOR >= 2 -#else -#error \"Requires libyajl2\" -#endif -" - then - mkl_allvar_set "json" ENABLE_JSON y - fi - - - mkl_meta_set "avroc" "static" "libavro.a" - mkl_meta_set "libserdes" "deb" "libserdes-dev" - mkl_meta_set "libserdes" "static" "libserdes.a" - - # Check for Avro and Schema-Registry client libs - if [[ $WITH_AVRO == y ]] && - mkl_lib_check --libname=avro-c --static=-lavro "avroc" "" disable CC "-lavro" "#include " && - mkl_lib_check --static=-lserdes "serdes" HAVE_SERDES disable CC "-lserdes" \ - "#include - #include - #include "; then - mkl_allvar_set "avro" ENABLE_AVRO y - fi -} - - -mkl_toggle_option "kafkacat" WITH_JSON --enable-json "JSON support (requires libyajl2)" y -mkl_toggle_option "kafkacat" WITH_AVRO --enable-avro "Avro/Schema-Registry support (requires libserdes)" y diff -Nru kafkacat-1.5.0/configure.self kafkacat-1.6.0/configure.self --- kafkacat-1.5.0/configure.self 1970-01-01 00:00:00.000000000 +0000 +++ kafkacat-1.6.0/configure.self 2020-07-21 09:00:07.000000000 +0000 @@ -0,0 +1,65 @@ +#!/bin/bash +# + +mkl_require good_cflags +mkl_require gitversion as KAFKACAT_VERSION default 1.6.0 + + +function checks { + + # Check that librdkafka is available, and allow to link it statically. + mkl_meta_set "rdkafka" "desc" "librdkafka is available at http://github.com/edenhill/librdkafka. To quickly download all dependencies and build kafkacat try ./bootstrap.sh" + mkl_meta_set "rdkafka" "deb" "librdkafka-dev" + mkl_lib_check "rdkafka" "" fail CC "-lrdkafka" \ + "#include " + + # Make sure rdkafka is new enough. + mkl_meta_set "librdkafkaver" "name" "librdkafka metadata API" + mkl_meta_set "librdkafkaver" "desc" "librdkafka 0.8.4 or later is required for the Metadata API" + mkl_compile_check "librdkafkaver" "" fail CC "" \ +"#include +struct rd_kafka_metadata foo;" + + # Enable KafkaConsumer support if librdkafka is new enough + mkl_meta_set "librdkafka_ge_090" "name" "librdkafka KafkaConsumer support" + mkl_compile_check "librdkafka_ge_090" ENABLE_KAFKACONSUMER disable CC "" " + #include + #if RD_KAFKA_VERSION >= 0x00090000 + #else + #error \"rdkafka version < 0.9.0\" + #endif" + + + mkl_meta_set "yajl" "deb" "libyajl-dev" + # Check for JSON library (yajl) + if [[ $WITH_JSON == y ]] && \ + mkl_lib_check "yajl" HAVE_YAJL disable CC "-lyajl" \ + "#include +#if YAJL_MAJOR >= 2 +#else +#error \"Requires libyajl2\" +#endif +" + then + mkl_allvar_set "json" ENABLE_JSON y + fi + + + mkl_meta_set "avroc" "static" "libavro.a" + mkl_meta_set "libserdes" "deb" "libserdes-dev" + mkl_meta_set "libserdes" "static" "libserdes.a" + + # Check for Avro and Schema-Registry client libs + if [[ $WITH_AVRO == y ]] && + mkl_lib_check --libname=avro-c "avroc" "" disable CC "-lavro" "#include " && + mkl_lib_check "serdes" HAVE_SERDES disable CC "-lserdes" \ + "#include + #include + #include "; then + mkl_allvar_set "avro" ENABLE_AVRO y + fi +} + + +mkl_toggle_option "kafkacat" WITH_JSON --enable-json "JSON support (requires libyajl2)" y +mkl_toggle_option "kafkacat" WITH_AVRO --enable-avro "Avro/Schema-Registry support (requires libserdes)" y diff -Nru kafkacat-1.5.0/debian/changelog kafkacat-1.6.0/debian/changelog --- kafkacat-1.5.0/debian/changelog 2019-12-08 19:00:33.000000000 +0000 +++ kafkacat-1.6.0/debian/changelog 2020-07-21 09:05:35.000000000 +0000 @@ -1,3 +1,13 @@ +kafkacat (1.6.0-1) unstable; urgency=medium + + * New upstream release. + + Fix FTBFS due to ./configure script not accepting + --disable-option-checking. Closes: #963440 + * d/control: bump Standards-Version + * d/control: switch to debhelper 13 + + -- Vincent Bernat Tue, 21 Jul 2020 11:05:35 +0200 + kafkacat (1.5.0-1.1) unstable; urgency=medium * Non-maintainer upload. diff -Nru kafkacat-1.5.0/debian/compat kafkacat-1.6.0/debian/compat --- kafkacat-1.5.0/debian/compat 2019-12-08 19:00:33.000000000 +0000 +++ kafkacat-1.6.0/debian/compat 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -9 diff -Nru kafkacat-1.5.0/debian/control kafkacat-1.6.0/debian/control --- kafkacat-1.5.0/debian/control 2019-12-08 19:00:33.000000000 +0000 +++ kafkacat-1.6.0/debian/control 2020-07-21 09:05:35.000000000 +0000 @@ -3,13 +3,13 @@ Priority: optional Maintainer: Vincent Bernat Uploaders: Faidon Liambotis -Build-Depends: debhelper (>= 9), +Build-Depends: debhelper-compat (= 13), librdkafka-dev (>= 0.9.0), libavro-dev, pkg-config, zlib1g-dev, libyajl-dev (>= 2) -Standards-Version: 4.0.0 +Standards-Version: 4.5.0 Homepage: https://github.com/edenhill/kafkacat Vcs-Browser: https://github.com/edenhill/kafkacat/tree/debian Vcs-Git: https://github.com/edenhill/kafkacat.git -b debian diff -Nru kafkacat-1.5.0/debian/gbp.conf kafkacat-1.6.0/debian/gbp.conf --- kafkacat-1.5.0/debian/gbp.conf 2019-12-08 19:00:33.000000000 +0000 +++ kafkacat-1.6.0/debian/gbp.conf 2020-07-21 09:05:35.000000000 +0000 @@ -1,9 +1,7 @@ -[buildpackage] +[DEFAULT] upstream-tree = tag upstream-branch = master debian-branch = debian upstream-tag = %(version)s debian-tag = debian/%(version)s pristine-tar = False -tarball-dir = ../tarballs -export-dir = ../build-area diff -Nru kafkacat-1.5.0/json.c kafkacat-1.6.0/json.c --- kafkacat-1.5.0/json.c 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/json.c 2020-07-21 09:00:07.000000000 +0000 @@ -31,8 +31,8 @@ #include #define JS_STR(G, STR) do { \ - const char *_s = (STR); \ - yajl_gen_string(G, (const unsigned char *)_s, strlen(_s)); \ + const char *_s = (STR); \ + yajl_gen_string(G, (const unsigned char *)_s, strlen(_s)); \ } while (0) void fmt_msg_output_json (FILE *fp, const rd_kafka_message_t *rkmessage) { @@ -76,6 +76,13 @@ yajl_gen_integer(g, 0); #endif + JS_STR(g, "broker"); +#if RD_KAFKA_VERSION >= 0x010500ff + yajl_gen_integer(g, (int)rd_kafka_message_broker_id(rkmessage)); +#else + yajl_gen_integer(g, -1); +#endif + #if HAVE_HEADERS { diff -Nru kafkacat-1.5.0/kafkacat.c kafkacat-1.6.0/kafkacat.c --- kafkacat-1.5.0/kafkacat.c 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/kafkacat.c 2020-07-21 09:00:07.000000000 +0000 @@ -50,6 +50,10 @@ #include "kafkacat.h" +#if RD_KAFKA_VERSION >= 0x01040000 +#define ENABLE_TXNS 1 +#endif + struct conf conf = { .run = 1, @@ -59,6 +63,8 @@ .msg_size = 1024*1024, .null_str = "NULL", .fixed_key = NULL, + .metadata_timeout = 5, + .offset = RD_KAFKA_OFFSET_INVALID, }; static struct stats { @@ -84,7 +90,7 @@ * Fatal error: print error and exit */ void RD_NORETURN fatal0 (const char *func, int line, - const char *fmt, ...) { + const char *fmt, ...) { va_list ap; char buf[1024]; @@ -127,20 +133,27 @@ static void dr_msg_cb (rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, void *opaque) { static int say_once = 1; + int32_t broker_id = -1; if (rkmessage->err) { KC_INFO(1, "Delivery failed for message: %s\n", - rd_kafka_err2str(rkmessage->err)); + rd_kafka_err2str(rkmessage->err)); stats.tx_err_dr++; return; } - KC_INFO(3, "Message delivered to partition %"PRId32" (offset %"PRId64")\n", - rkmessage->partition, rkmessage->offset); +#if RD_KAFKA_VERSION >= 0x010500ff + broker_id = rd_kafka_message_broker_id(rkmessage); +#endif + + KC_INFO(3, + "Message delivered to partition %"PRId32" (offset %"PRId64") " + "on broker %"PRId32"\n", + rkmessage->partition, rkmessage->offset, broker_id); if (rkmessage->offset == 0 && say_once) { KC_INFO(3, "Enable message offset reporting " - "with '-X topic.produce.offset.report=true'\n"); + "with '-X topic.produce.offset.report=true'\n"); say_once = 0; } stats.tx_delivered++; @@ -165,7 +178,7 @@ if (!conf.run) KC_FATAL("Program terminated while " - "producing message of %zd bytes", len); + "producing message of %zd bytes", len); err = rd_kafka_producev( conf.rk, @@ -184,7 +197,7 @@ if (err != RD_KAFKA_RESP_ERR__QUEUE_FULL) KC_FATAL("Failed to produce message (%zd bytes): %s", - len, rd_kafka_err2str(err)); + len, rd_kafka_err2str(err)); stats.tx_err_q++; @@ -207,8 +220,8 @@ int fd; void *ptr; struct stat st; - ssize_t sz; - int msgflags = 0; + ssize_t sz; + int msgflags = 0; if ((fd = _COMPAT(open)(path, O_RDONLY)) == -1) { KC_INFO(1, "Failed to open %s: %s\n", path, strerror(errno)); @@ -234,43 +247,43 @@ _COMPAT(close)(fd); return -1; } - sz = st.st_size; - msgflags = RD_KAFKA_MSG_F_COPY; + sz = st.st_size; + msgflags = RD_KAFKA_MSG_F_COPY; #else - ptr = malloc(st.st_size); - if (!ptr) { - KC_INFO(1, "Failed to allocate message for %s: %s\n", - path, strerror(errno)); - _COMPAT(close)(fd); - return -1; - } - - sz = _read(fd, ptr, st.st_size); - if (sz < st.st_size) { - KC_INFO(1, "Read failed for %s (%zd/%zd): %s\n", - path, sz, (size_t)st.st_size, sz == -1 ? strerror(errno) : - "incomplete read"); - free(ptr); - close(fd); - return -1; - } - msgflags = RD_KAFKA_MSG_F_FREE; + ptr = malloc(st.st_size); + if (!ptr) { + KC_INFO(1, "Failed to allocate message for %s: %s\n", + path, strerror(errno)); + _COMPAT(close)(fd); + return -1; + } + + sz = _read(fd, ptr, st.st_size); + if (sz < st.st_size) { + KC_INFO(1, "Read failed for %s (%zd/%zd): %s\n", + path, sz, (size_t)st.st_size, sz == -1 ? strerror(errno) : + "incomplete read"); + free(ptr); + close(fd); + return -1; + } + msgflags = RD_KAFKA_MSG_F_FREE; #endif KC_INFO(4, "Producing file %s (%"PRIdMAX" bytes)\n", - path, (intmax_t)st.st_size); - produce(ptr, sz, NULL, 0, msgflags); + path, (intmax_t)st.st_size); + produce(ptr, sz, conf.fixed_key, conf.fixed_key_len, msgflags); - _COMPAT(close)(fd); + _COMPAT(close)(fd); - if (!(msgflags & RD_KAFKA_MSG_F_FREE)) { + if (!(msgflags & RD_KAFKA_MSG_F_FREE)) { #ifndef _MSC_VER - munmap(ptr, st.st_size); + munmap(ptr, st.st_size); #else - free(ptr); + free(ptr); #endif - } - return sz; + } + return sz; } @@ -283,6 +296,14 @@ size_t size = 0; ssize_t len; char errstr[512]; + char tmp[2]; + + size = sizeof(tmp); + if (rd_kafka_conf_get(conf.rk_conf, "transactional.id", + tmp, &size) == RD_KAFKA_CONF_OK && size > 1) { + KC_INFO(1, "Using transactional producer\n"); + conf.txn = 1; + } /* Assign per-message delivery report callback. */ rd_kafka_conf_set_dr_msg_cb(conf.rk_conf, dr_msg_cb); @@ -295,6 +316,23 @@ if (!conf.debug && conf.verbosity == 0) rd_kafka_set_log_level(conf.rk, 0); +#if ENABLE_TXNS + if (conf.txn) { + rd_kafka_error_t *error; + + error = rd_kafka_init_transactions(conf.rk, + conf.metadata_timeout*1000); + if (error) + KC_FATAL("init_transactions(): %s", + rd_kafka_error_string(error)); + + error = rd_kafka_begin_transaction(conf.rk); + if (error) + KC_FATAL("begin_transaction(): %s", + rd_kafka_error_string(error)); + } +#endif + /* Create topic */ if (!(conf.rkt = rd_kafka_topic_new(conf.rk, conf.topic, conf.rkt_conf))) @@ -318,7 +356,7 @@ conf.exitcode = 1; else if (good < pathcnt) KC_INFO(1, "Failed to produce from %i/%i files\n", - pathcnt - good, pathcnt); + pathcnt - good, pathcnt); } else { /* Read messages from input, delimited by conf.delim */ @@ -393,7 +431,7 @@ if (conf.flags & CONF_F_TEE && fwrite(sbuf, orig_len, 1, stdout) != 1) KC_FATAL("Tee write error for message of %zd bytes: %s", - orig_len, strerror(errno)); + orig_len, strerror(errno)); if (msgflags & RD_KAFKA_MSG_F_FREE) { /* rdkafka owns the allocated buffer @@ -410,9 +448,37 @@ if (conf.run) { if (!feof(fp)) KC_FATAL("Unable to read message: %s", - strerror(errno)); + strerror(errno)); + } + } + +#if ENABLE_TXNS + if (conf.txn) { + rd_kafka_error_t *error; + const char *what; + + if (conf.term_sig) { + KC_INFO(0, + "Aborting transaction due to " + "termination signal\n"); + what = "abort_transaction()"; + error = rd_kafka_abort_transaction( + conf.rk, conf.metadata_timeout * 1000); + } else { + KC_INFO(1, "Committing transaction\n"); + what = "commit_transaction()"; + error = rd_kafka_commit_transaction( + conf.rk, conf.metadata_timeout * 1000); + if (!error) + KC_INFO(1, + "Transaction successfully committed\n"); } + + if (error) + KC_FATAL("%s: %s", what, rd_kafka_error_string(error)); } +#endif + /* Wait for all messages to be transmitted */ conf.run = 1; @@ -461,11 +527,11 @@ } KC_INFO(1, "Reached end of topic %s [%"PRId32"] " - "at offset %"PRId64"%s\n", - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, - rkmessage->offset, - !conf.run ? ": exiting" : ""); + "at offset %"PRId64"%s\n", + rd_kafka_topic_name(rkmessage->rkt), + rkmessage->partition, + rkmessage->offset, + !conf.run ? ": exiting" : ""); } @@ -486,12 +552,12 @@ if (rkmessage->rkt) KC_FATAL("Topic %s [%"PRId32"] error: %s", - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, - rd_kafka_message_errstr(rkmessage)); + rd_kafka_topic_name(rkmessage->rkt), + rkmessage->partition, + rd_kafka_message_errstr(rkmessage)); else KC_FATAL("Consumer error: %s", - rd_kafka_message_errstr(rkmessage)); + rd_kafka_message_errstr(rkmessage)); } @@ -500,11 +566,11 @@ if (ts >= conf.stopts) { stop_partition(rkmessage); KC_INFO(1, "Reached stop timestamp for topic %s [%"PRId32"] " - "at offset %"PRId64"%s\n", - rd_kafka_topic_name(rkmessage->rkt), - rkmessage->partition, - rkmessage->offset, - !conf.run ? ": exiting" : ""); + "at offset %"PRId64"%s\n", + rd_kafka_topic_name(rkmessage->rkt), + rkmessage->partition, + rkmessage->offset, + !conf.run ? ": exiting" : ""); return; } } @@ -529,7 +595,7 @@ static void throttle_cb (rd_kafka_t *rk, const char *broker_name, int32_t broker_id, int throttle_time_ms, void *opaque){ KC_INFO(1, "Broker %s (%"PRId32") throttled request for %dms\n", - broker_name, broker_id, throttle_time_ms); + broker_name, broker_id, throttle_time_ms); } #endif @@ -552,30 +618,35 @@ void *opaque) { KC_INFO(1, "Group %s rebalanced (memberid %s): ", - conf.group, rd_kafka_memberid(rk)); + conf.group, rd_kafka_memberid(rk)); - switch (err) - { - case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: + switch (err) + { + case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: if (conf.verbosity >= 1) { fprintf(stderr, "assigned: "); print_partition_list(1, partitions); } - rd_kafka_assign(rk, partitions); - break; + if (conf.offset != RD_KAFKA_OFFSET_INVALID) { + int i; + for (i = 0 ; i < partitions->cnt ; i++) + partitions->elems[i].offset = conf.offset; + } + rd_kafka_assign(rk, partitions); + break; - case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: + case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: if (conf.verbosity >= 1) { fprintf(stderr, "revoked: "); print_partition_list(1, partitions); } - rd_kafka_assign(rk, NULL); - break; + rd_kafka_assign(rk, NULL); + break; - default: + default: KC_INFO(0, "failed: %s\n", rd_kafka_err2str(err)); - break; - } + break; + } } /** @@ -614,7 +685,9 @@ /* Subscribe */ if ((err = rd_kafka_subscribe(conf.rk, topiclist))) KC_FATAL("Failed to subscribe to %d topics: %s\n", - topiclist->cnt, rd_kafka_err2str(err)); + topiclist->cnt, rd_kafka_err2str(err)); + + KC_INFO(1, "Waiting for group rebalance\n"); rd_kafka_topic_partition_list_destroy(topiclist); @@ -651,7 +724,7 @@ int64_t *offsets; rd_kafka_resp_err_t err; rd_kafka_topic_partition_list_t *rktparlistp = - rd_kafka_topic_partition_list_new(1); + rd_kafka_topic_partition_list_new(1); for (i = 0 ; i < topic->partition_cnt ; i++) { int32_t partition = topic->partitions[i].id; @@ -662,13 +735,14 @@ continue; rd_kafka_topic_partition_list_add(rktparlistp, - rd_kafka_topic_name(conf.rkt), - partition)->offset = conf.startts; + rd_kafka_topic_name(conf.rkt), + partition)->offset = conf.startts; if (conf.partition != RD_KAFKA_PARTITION_UA) break; } - err = rd_kafka_offsets_for_times(conf.rk, rktparlistp, 10*1000); + err = rd_kafka_offsets_for_times(conf.rk, rktparlistp, + conf.metadata_timeout * 1000); if (err) KC_FATAL("offsets_for_times failed: %s", rd_kafka_err2str(err)); @@ -720,28 +794,29 @@ /* Query broker for topic + partition information. */ - if ((err = rd_kafka_metadata(conf.rk, 0, conf.rkt, &metadata, 5000))) + if ((err = rd_kafka_metadata(conf.rk, 0, conf.rkt, &metadata, + conf.metadata_timeout * 1000))) KC_FATAL("Failed to query metadata for topic %s: %s", - rd_kafka_topic_name(conf.rkt), rd_kafka_err2str(err)); + rd_kafka_topic_name(conf.rkt), rd_kafka_err2str(err)); /* Error handling */ if (metadata->topic_cnt == 0) KC_FATAL("No such topic in cluster: %s", - rd_kafka_topic_name(conf.rkt)); + rd_kafka_topic_name(conf.rkt)); if ((err = metadata->topics[0].err)) KC_FATAL("Topic %s error: %s", - rd_kafka_topic_name(conf.rkt), rd_kafka_err2str(err)); + rd_kafka_topic_name(conf.rkt), rd_kafka_err2str(err)); if (metadata->topics[0].partition_cnt == 0) KC_FATAL("Topic %s has no partitions", - rd_kafka_topic_name(conf.rkt)); + rd_kafka_topic_name(conf.rkt)); /* If Exit-at-EOF is enabled, set up array to track EOF * state for each partition. */ if (conf.exit_eof || conf.stopts) { part_stop = calloc(sizeof(*part_stop), - metadata->topics[0].partition_cnt); + metadata->topics[0].partition_cnt); if (conf.partition != RD_KAFKA_PARTITION_UA) part_stop_thres = 1; @@ -751,7 +826,7 @@ #if RD_KAFKA_VERSION >= 0x00090300 if (conf.startts) { - offsets = get_offsets(&metadata->topics[0]); + offsets = get_offsets(&metadata->topics[0]); } #endif @@ -770,7 +845,11 @@ /* Start consumer for this partition */ if (rd_kafka_consume_start_queue(conf.rkt, partition, - offsets ? offsets[i] : conf.offset, + offsets ? offsets[i] : + (conf.offset == + RD_KAFKA_OFFSET_INVALID ? + RD_KAFKA_OFFSET_BEGINNING : + conf.offset), rkqu) == -1) KC_FATAL("Failed to start consuming " "topic %s [%"PRId32"]: %s", @@ -785,10 +864,10 @@ if (conf.partition != RD_KAFKA_PARTITION_UA && i == metadata->topics[0].partition_cnt) KC_FATAL("Topic %s (with partitions 0..%i): " - "partition %i does not exist", - rd_kafka_topic_name(conf.rkt), - metadata->topics[0].partition_cnt-1, - conf.partition); + "partition %i does not exist", + rd_kafka_topic_name(conf.rkt), + metadata->topics[0].partition_cnt-1, + conf.partition); /* Read messages from Kafka, write to 'fp'. */ @@ -809,9 +888,9 @@ conf.partition != partition) continue; - /* Dont stop already stopped partitions */ - if (!part_stop || !part_stop[partition]) - rd_kafka_consume_stop(conf.rkt, partition); + /* Dont stop already stopped partitions */ + if (!part_stop || !part_stop[partition]) + rd_kafka_consume_stop(conf.rkt, partition); rd_kafka_consume_stop(conf.rkt, partition); } @@ -922,7 +1001,7 @@ /* Fetch metadata */ err = rd_kafka_metadata(conf.rk, conf.rkt ? 0 : 1, conf.rkt, - &metadata, 5000); + &metadata, conf.metadata_timeout * 1000); if (err != RD_KAFKA_RESP_ERR_NO_ERROR) KC_FATAL("Failed to acquire metadata: %s", rd_kafka_err2str(err)); @@ -989,6 +1068,9 @@ #if ENABLE_AVRO "Avro, " #endif +#if ENABLE_TXNS + "Transactions, " +#endif , rd_kafka_version_str(), features ); @@ -1013,6 +1095,13 @@ " -E Do not exit on non fatal error\n" " -K Key delimiter (same format as -D)\n" " -c Limit message count\n" + " -m Metadata (et.al.) request timeout.\n" + " This limits how long kafkacat will block\n" + " while waiting for initial metadata to be\n" + " retrieved from the Kafka cluster.\n" + " It also sets the timeout for the producer's\n" + " transaction commits, init, aborts, etc.\n" + " Default: 5 seconds.\n" " -F Read configuration properties from file,\n" " file format is \"property=value\".\n" " The KAFKACAT_CONFIG=path environment can " @@ -1057,6 +1146,12 @@ " With -l, only one file permitted.\n" " Otherwise, the entire file contents will\n" " be sent as one single message.\n" + " -X transactional.id=.. Enable transactions and send all\n" + " messages in a single transaction which\n" + " is committed when stdin is closed or the\n" + " input file(s) are fully read.\n" + " If kafkacat is terminated through Ctrl-C\n" + " (et.al) the transaction will be aborted.\n" "\n" "Consumer options:\n" " -o Offset to start consuming from:\n" @@ -1110,7 +1205,7 @@ " -O Print message offset using -K delimiter\n" " -c Exit after consuming this number " "of messages\n" - " -Z Print NULL values and keys as \"%s\"" + " -Z Print NULL values and keys as \"%s\" " "instead of empty.\n" " For JSON (-J) the nullstr is always null.\n" " -u Unbuffered output\n" @@ -1148,16 +1243,19 @@ " Example:\n" " -f 'Topic %%t [%%p] at offset %%o: key %%k: %%s\\n'\n" "\n" +#if ENABLE_JSON "JSON message envelope (on one line) when consuming with -J:\n" " { \"topic\": str, \"partition\": int, \"offset\": int,\n" " \"tstype\": \"create|logappend|unknown\", \"ts\": int, " "// timestamp in milliseconds since epoch\n" + " \"broker\": int,\n" " \"headers\": { \"\": str, .. }, // optional\n" " \"key\": str|json, \"payload\": str|json,\n" " \"key_error\": str, \"payload_error\": str } //optional\n" " (note: key_error and payload_error are only included if " "deserialization failed)\n" "\n" +#endif "Consumer mode (writes messages to stdout):\n" " kafkacat -b -t -p \n" " or:\n" @@ -1190,6 +1288,7 @@ */ static void term (int sig) { conf.run = 0; + conf.term_sig = sig; } @@ -1208,20 +1307,20 @@ rd_kafka_resp_err_t fatal_err; fatal_err = rd_kafka_fatal_error(rk, fatal_errstr, - sizeof(fatal_errstr)); + sizeof(fatal_errstr)); KC_INFO(0, "FATAL CLIENT ERROR: %s: %s: terminating\n", rd_kafka_err2str(fatal_err), fatal_errstr); conf.run = 0; } else #endif - if (err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN) { - KC_ERROR("%s: %s", rd_kafka_err2str(err), - reason ? reason : ""); - } else { - KC_INFO(1, "ERROR: %s: %s\n", rd_kafka_err2str(err), - reason ? reason : ""); - } + if (err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN) { + KC_ERROR("%s: %s", rd_kafka_err2str(err), + reason ? reason : ""); + } else { + KC_INFO(1, "ERROR: %s: %s\n", rd_kafka_err2str(err), + reason ? reason : ""); + } } @@ -1322,8 +1421,10 @@ if (!conf.srconf) conf.srconf = serdes_conf_new(NULL, 0, NULL); - if (!strcmp(name, "schema.registry.url")) + if (!strcmp(name, "schema.registry.url")) { + conf.flags |= CONF_F_SR_URL_SEEN; srlen = 0; + } serr = serdes_conf_set(conf.srconf, name+srlen, val, errstr, errstr_size); @@ -1571,7 +1672,7 @@ while ((opt = getopt(argc, argv, ":PCG:LQt:p:b:z:o:eED:K:k:H:Od:qvF:X:c:Tuf:ZlVh" - "s:r:J")) != -1) { + "s:r:Jm:")) != -1) { switch (opt) { case 'P': case 'C': @@ -1683,6 +1784,7 @@ if (!*optarg) KC_FATAL("-s url must not be empty"); conf.schema_registry_url = optarg; + conf.flags |= CONF_F_SR_URL_SEEN; #else KC_FATAL("This build of kafkacat lacks " "Avro/Schema-Registry support"); @@ -1711,6 +1813,9 @@ case 'c': conf.msg_cnt = strtoll(optarg, NULL, 10); break; + case 'm': + conf.metadata_timeout = strtoll(optarg, NULL, 10); + break; case 'Z': conf.flags |= CONF_F_NULL; conf.null_str_len = strlen(conf.null_str); @@ -1817,7 +1922,7 @@ else conf.mode = 'P'; KC_INFO(1, "Auto-selecting %s mode (use -P or -C to override)\n", - conf.mode == 'C' ? "Consumer":"Producer"); + conf.mode == 'C' ? "Consumer":"Producer"); } @@ -1882,7 +1987,7 @@ * Verify and initialize Avro/SR */ #if ENABLE_AVRO - if (!!conf.schema_registry_url != + if (!!(conf.flags & CONF_F_SR_URL_SEEN) != !!(conf.flags & (CONF_F_FMT_AVRO_VALUE|CONF_F_FMT_AVRO_KEY))) KC_FATAL("-r requires -s avro and vice-versa"); @@ -1939,8 +2044,8 @@ } else if (conf.mode == 'P') { conf.delim = parse_delim(delim); - if (conf.flags & CONF_F_KEY_DELIM) - conf.key_delim = parse_delim(key_delim); + if (conf.flags & CONF_F_KEY_DELIM) + conf.key_delim = parse_delim(key_delim); } /* Automatically enable API version requests if needed and @@ -1958,10 +2063,10 @@ int main (int argc, char **argv) { #ifdef SIGIO - char tmp[16]; + char tmp[16]; #endif FILE *in = stdin; - struct timeval tv; + struct timeval tv; rd_kafka_topic_partition_list_t *rktparlist = NULL; /* Certain Docker images don't have kafkacat as the entry point, @@ -1981,9 +2086,9 @@ signal(SIGPIPE, term); #endif - /* Seed rng for random partitioner, jitter, etc. */ - rd_gettimeofday(&tv, NULL); - srand(tv.tv_usec); + /* Seed rng for random partitioner, jitter, etc. */ + rd_gettimeofday(&tv, NULL); + srand(tv.tv_usec); /* Create config containers */ conf.rk_conf = rd_kafka_conf_new(); @@ -2029,6 +2134,9 @@ #if ENABLE_KAFKACONSUMER case 'G': + if (conf.stopts || conf.startts) + KC_FATAL("-o ..@ timestamps can't be used " + "with -G mode\n"); kafkaconsumer_run(stdout, &argv[optind], argc-optind); break; #endif diff -Nru kafkacat-1.5.0/kafkacat.h kafkacat-1.6.0/kafkacat.h --- kafkacat-1.5.0/kafkacat.h 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/kafkacat.h 2020-07-21 09:00:07.000000000 +0000 @@ -94,13 +94,14 @@ #define CONF_F_OFFSET 0x4 /* Print offsets */ #define CONF_F_TEE 0x8 /* Tee output when producing */ #define CONF_F_NULL 0x10 /* -Z: Send empty messages as NULL */ -#define CONF_F_LINE 0x20 /* Read files in line mode when producing */ +#define CONF_F_LINE 0x20 /* Read files in line mode when producing */ #define CONF_F_APIVERREQ 0x40 /* Enable api.version.request=true */ #define CONF_F_APIVERREQ_USER 0x80 /* User set api.version.request */ #define CONF_F_NO_CONF_SEARCH 0x100 /* Disable default config file search */ #define CONF_F_BROKERS_SEEN 0x200 /* Brokers have been configured */ #define CONF_F_FMT_AVRO_KEY 0x400 /* Convert key from Avro to JSON */ #define CONF_F_FMT_AVRO_VALUE 0x800 /* Convert value from Avro to JSON */ +#define CONF_F_SR_URL_SEEN 0x1000 /* schema.registry.url/-r seen */ int delim; int key_delim; @@ -130,8 +131,10 @@ #endif int exit_eof; int64_t msg_cnt; + int metadata_timeout; char *null_str; int null_str_len; + int txn; rd_kafka_conf_t *rk_conf; rd_kafka_topic_conf_t *rkt_conf; @@ -141,6 +144,8 @@ char *debug; + int term_sig; /**< Termination signal */ + #if ENABLE_AVRO serdes_conf_t *srconf; char *schema_registry_url; @@ -151,19 +156,19 @@ void RD_NORETURN fatal0 (const char *func, int line, - const char *fmt, ...); + const char *fmt, ...); void error0 (int erroronexit, const char *func, int line, - const char *fmt, ...); + const char *fmt, ...); #define KC_FATAL(.../*fmt*/) fatal0(__FUNCTION__, __LINE__, __VA_ARGS__) #define KC_ERROR(.../*fmt*/) error0(conf.exitonerror, __FUNCTION__, __LINE__, __VA_ARGS__) /* Info printout */ -#define KC_INFO(VERBLVL,.../*fmt*/) do { \ - if (conf.verbosity >= (VERBLVL)) \ - fprintf(stderr, "%% " __VA_ARGS__); \ +#define KC_INFO(VERBLVL,.../*fmt*/) do { \ + if (conf.verbosity >= (VERBLVL)) \ + fprintf(stderr, "%% " __VA_ARGS__); \ } while (0) diff -Nru kafkacat-1.5.0/mklove/Makefile.base kafkacat-1.6.0/mklove/Makefile.base --- kafkacat-1.5.0/mklove/Makefile.base 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/mklove/Makefile.base 2020-07-21 09:00:07.000000000 +0000 @@ -25,13 +25,25 @@ -include $(TOPDIR)/Makefile.config endif +# Use C compiler as default linker. +# C++ libraries will need to override this with CXX after +# including Makefile.base +CC_LD?=$(CC) + _UNAME_S := $(shell uname -s) ifeq ($(_UNAME_S),Darwin) LIBFILENAME=$(LIBNAME).$(LIBVER)$(SOLIB_EXT) LIBFILENAMELINK=$(LIBNAME)$(SOLIB_EXT) + LDD_PRINT="otool -L" else LIBFILENAME=$(LIBNAME)$(SOLIB_EXT).$(LIBVER) LIBFILENAMELINK=$(LIBNAME)$(SOLIB_EXT) + LDD_PRINT="ldd" +endif + +# DESTDIR must be an absolute path +ifneq ($(DESTDIR),) +DESTDIR:=$(abspath $(DESTDIR)) endif INSTALL?= install @@ -65,6 +77,12 @@ man7dir?= $(mandir)/man7 man8dir?= $(mandir)/man8 +# An application Makefile should set DISABLE_LDS=y prior to +# including Makefile.base if it does not wish to have a linker-script. +ifeq ($(WITH_LDS)-$(DISABLE_LDS),y-) +# linker-script file +LIBNAME_LDS?=$(LIBNAME).lds +endif # Checks that mklove is set up and ready for building mklove-check: @@ -80,18 +98,53 @@ $(CXX) -MD -MP $(CPPFLAGS) $(CXXFLAGS) -c $< -o $@ -lib: $(LIBFILENAME) $(LIBNAME).a $(LIBFILENAMELINK) lib-gen-pkg-config +lib: $(LIBFILENAME) $(LIBNAME).a $(LIBNAME)-static.a $(LIBFILENAMELINK) lib-gen-pkg-config -$(LIBNAME).lds: #overridable +# Linker-script (if WITH_LDS=y): overridable by application Makefile +$(LIBNAME_LDS): -$(LIBFILENAME): $(OBJS) $(LIBNAME).lds +$(LIBFILENAME): $(OBJS) $(LIBNAME_LDS) @printf "$(MKL_YELLOW)Creating shared library $@$(MKL_CLR_RESET)\n" - $(CC) $(LDFLAGS) $(LIB_LDFLAGS) $(OBJS) -o $@ $(LIBS) + $(CC_LD) $(LDFLAGS) $(LIB_LDFLAGS) $(OBJS) -o $@ $(LIBS) $(LIBNAME).a: $(OBJS) @printf "$(MKL_YELLOW)Creating static library $@$(MKL_CLR_RESET)\n" $(AR) rcs$(ARFLAGS) $@ $(OBJS) +ifeq ($(MKL_NO_SELFCONTAINED_STATIC_LIB),y) +$(LIBNAME)-static.a: +else ifneq ($(WITH_STATIC_LINKING),y) +$(LIBNAME)-static.a: +else +$(LIBNAME)-static.a: $(LIBNAME).a +ifneq ($(MKL_STATIC_LIBS),) + @printf "$(MKL_YELLOW)Creating self-contained static library $@$(MKL_CLR_RESET)\n" +ifeq ($(HAS_LIBTOOL_STATIC),y) + $(LIBTOOL) -static -o $@ - $(LIBNAME).a $(MKL_STATIC_LIBS) +else + (_tmp=$$(mktemp arstaticXXXXXX) ; \ + echo "CREATE $@" > $$_tmp ; \ + for _f in $(LIBNAME).a $(MKL_STATIC_LIBS) ; do \ + echo "ADDLIB $$_f" >> $$_tmp ; \ + done ; \ + echo "SAVE" >> $$_tmp ; \ + echo "END" >> $$_tmp ; \ + cat $$_tmp ; \ + ar -M < $$_tmp || exit 1 ; \ + rm $$_tmp) +endif + cp $@ $(LIBNAME)-static-dbg.a + strip -S $@ + ranlib $@ +ifneq ($(MKL_DYNAMIC_LIBS),) + @printf "$(MKL_RED)WARNING:$(MKL_YELLOW) $@: The following libraries were not available as static libraries and need to be linked dynamically: $(MKL_DYNAMIC_LIBS)$(MKL_CLR_RESET)\n" +endif +else + @printf "$(MKL_RED)WARNING:$(MKL_YELLOW) $@: Not creating self-contained static library $@: no static libraries available/enabled$(MKL_CLR_RESET)\n" +endif +endif + + $(LIBFILENAMELINK): $(LIBFILENAME) @printf "$(MKL_YELLOW)Creating $@ symlink$(MKL_CLR_RESET)\n" rm -f "$@" && ln -s "$^" "$@" @@ -123,7 +176,7 @@ Description: $(MKL_APP_DESC_ONELINE) (static) Version: $(MKL_APP_VERSION) Cflags: -I$${includedir} -Libs: -L$${libdir} $${libdir}/$(LIBNAME).a $(LIBS) +Libs: -L$${libdir} $${libdir}/$(LIBNAME)-static.a $(MKL_DYNAMIC_LIBS) endef export _PKG_CONFIG_STATIC_DEF @@ -132,7 +185,7 @@ @printf "$(MKL_YELLOW)Generating pkg-config file $@$(MKL_CLR_RESET)\n" @echo "$$_PKG_CONFIG_DEF" > $@ -$(LIBNAME0)-static.pc: $(TOPDIR)/Makefile.config +$(LIBNAME0)-static.pc: $(TOPDIR)/Makefile.config $(LIBNAME)-static.a @printf "$(MKL_YELLOW)Generating pkg-config file $@$(MKL_CLR_RESET)\n" @echo "$$_PKG_CONFIG_STATIC_DEF" > $@ @@ -148,7 +201,7 @@ $(BIN): $(OBJS) @printf "$(MKL_YELLOW)Creating program $@$(MKL_CLR_RESET)\n" - $(CC) $(CPPFLAGS) $(LDFLAGS) $(OBJS) -o $@ $(LIBS) + $(CC_LD) $(CPPFLAGS) $(LDFLAGS) $(OBJS) -o $@ $(LIBS) file-check: @@ -165,22 +218,37 @@ done ; \ $$RET +copyright-check: + @(_exit=0 ; \ + for f in $$(git ls-tree -r --name-only HEAD | \ + egrep '\.(c|h|cpp|sh|py|pl)$$' ) ; do \ + if [ -n "$(MKL_COPYRIGHT_SKIP)" ] && echo "$$f" | egrep -q "$(MKL_COPYRIGHT_SKIP)" ; then \ + continue ; \ + fi ; \ + if ! head -40 $$f | grep -qi copyright $$f ; then \ + echo error: Copyright missing in $$f ; \ + _exit=1 ; \ + fi; \ + done ; \ + exit $$_exit) + lib-install: @printf "$(MKL_YELLOW)Install $(LIBNAME) to $$DESTDIR$(prefix)$(MKL_CLR_RESET)\n" - $(INSTALL) -d $$DESTDIR$(includedir)/$(PKGNAME) ; \ - $(INSTALL) -d $$DESTDIR$(libdir) ; \ - $(INSTALL) $(HDRS) $$DESTDIR$(includedir)/$(PKGNAME) ; \ - $(INSTALL) $(LIBNAME).a $$DESTDIR$(libdir) ; \ - $(INSTALL) $(LIBFILENAME) $$DESTDIR$(libdir) ; \ + $(INSTALL) -d $$DESTDIR$(includedir)/$(PKGNAME) + $(INSTALL) -d $$DESTDIR$(libdir) + $(INSTALL) $(HDRS) $$DESTDIR$(includedir)/$(PKGNAME) + $(INSTALL) $(LIBNAME).a $$DESTDIR$(libdir) + [ ! -f $(LIBNAME)-static.a ] || $(INSTALL) $(LIBNAME)-static.a $$DESTDIR$(libdir) + $(INSTALL) $(LIBFILENAME) $$DESTDIR$(libdir) [ -f "$(LIBNAME0).pc" ] && ( \ - $(INSTALL) -d $$DESTDIR$(pkgconfigdir) ; \ + $(INSTALL) -d $$DESTDIR$(pkgconfigdir) && \ $(INSTALL) -m 0644 $(LIBNAME0).pc $$DESTDIR$(pkgconfigdir) \ - ) ; \ + ) [ -f "$(LIBNAME0)-static.pc" ] && ( \ - $(INSTALL) -d $$DESTDIR$(pkgconfigdir) ; \ + $(INSTALL) -d $$DESTDIR$(pkgconfigdir) && \ $(INSTALL) -m 0644 $(LIBNAME0)-static.pc $$DESTDIR$(pkgconfigdir) \ - ) ; \ + ) (cd $$DESTDIR$(libdir) && ln -sf $(LIBFILENAME) $(LIBFILENAMELINK)) lib-uninstall: @@ -188,6 +256,7 @@ for hdr in $(HDRS) ; do \ rm -f $$DESTDIR$(includedir)/$(PKGNAME)/$$hdr ; done rm -f $$DESTDIR$(libdir)/$(LIBNAME).a + rm -f $$DESTDIR$(libdir)/$(LIBNAME)-static.a rm -f $$DESTDIR$(libdir)/$(LIBFILENAME) rm -f $$DESTDIR$(libdir)/$(LIBFILENAMELINK) rmdir $$DESTDIR$(includedir)/$(PKGNAME) || true @@ -205,13 +274,25 @@ rm -f $$DESTDIR$(bindir)/$(BIN) rmdir $$DESTDIR$(bindir) || true +doc-install: $(DOC_FILES) + @printf "$(MKL_YELLOW)Installing documentation to $$DESTDIR$(prefix)$(MKL_CLR_RESET)\n" + $(INSTALL) -d $$DESTDIR$(docdir) + $(INSTALL) $(DOC_FILES) $$DESTDIR$(docdir) + +doc-uninstall: + @printf "$(MKL_YELLOW)Uninstall documentation from $$DESTDIR$(prefix)$(MKL_CLR_RESET)\n" + for _f in $(DOC_FILES) ; do rm -f $$DESTDIR$(docdir)/$$_f ; done + rmdir $$DESTDIR$(docdir) || true generic-clean: rm -f $(OBJS) $(DEPS) lib-clean: generic-clean lib-clean-pkg-config rm -f $(LIBNAME)*.a $(LIBFILENAME) $(LIBFILENAMELINK) \ - $(LIBNAME).lds + $(LIBNAME_LDS) bin-clean: generic-clean rm -f $(BIN) + +deps-clean: + rm -rf "$(MKLOVE_DIR)/deps" diff -Nru kafkacat-1.5.0/mklove/modules/configure.base kafkacat-1.6.0/mklove/modules/configure.base --- kafkacat-1.5.0/mklove/modules/configure.base 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/mklove/modules/configure.base 2020-07-21 09:00:07.000000000 +0000 @@ -5,7 +5,7 @@ # MKL_MODULES="base" -MKL_CACHEVARS="" +MKL_CACHEVARS="CFLAGS LDFLAGS PKG_CONFIG_PATH" MKL_MKVARS="" MKL_DEFINES="" MKL_CHECKS="" @@ -13,9 +13,12 @@ MKL_IDNEXT=1 -MKL_OUTMK=_mklout.mk -MKL_OUTH=_mklout.h -MKL_OUTDBG=config.log +# Default mklove directory to PWD/mklove +[[ -z "$MKLOVE_DIR" ]] && MKLOVE_DIR="$PWD/mklove" + +MKL_OUTMK="$PWD/_mklout.mk" +MKL_OUTH="$PWD/_mklout.h" +MKL_OUTDBG="$PWD/config.log" MKL_GENERATORS="base:mkl_generate_late_vars" MKL_CLEANERS="" @@ -33,6 +36,8 @@ MKL_NO_DOWNLOAD=0 +MKL_INSTALL_DEPS=n +MKL_SOURCE_DEPS_ONLY=n if [[ -z "$MKL_REPO_URL" ]]; then MKL_REPO_URL="http://github.com/edenhill/mklove/raw/master" @@ -40,10 +45,6 @@ -# Default mklove directory to PWD/mklove -[[ -z "$MKLOVE_DIR" ]] && MKLOVE_DIR=mklove - - ########################################################################### # # Variable types: @@ -63,7 +64,7 @@ } # Sets a runtime variable (only used during configure) -# If cache=1 these variables are cached to config.cache. +# If "cache" is provided these variables are cached to config.cache. # Arguments: # variable name # variable value @@ -192,9 +193,10 @@ # config name # variable name # value +# [ separator (" ") ] function mkl_mkvar_prepend { if [[ ! -z $2 ]]; then - mkl_env_prepend "$2" "$3" + mkl_env_prepend "$2" "$3" "$4" mkl_in_list "$MKL_MKVARS" "$2"|| mkl_env_append MKL_MKVARS $2 fi } @@ -205,9 +207,10 @@ # config name # variable name # value +# [ separator (" ") ] function mkl_mkvar_append { if [[ ! -z $2 ]]; then - mkl_env_append "$2" "$3" + mkl_env_append "$2" "$3" "$4" mkl_in_list "$MKL_MKVARS" "$2"|| mkl_env_append MKL_MKVARS $2 fi } @@ -218,9 +221,10 @@ # config name # variable name # value +# [ separator (" ") ] function mkl_mkvar_prepend { if [[ ! -z $2 ]]; then - mkl_env_prepend "$2" "$3" + mkl_env_prepend "$2" "$3" "$4" mkl_in_list "$MKL_MKVARS" "$2"|| mkl_env_append MKL_MKVARS $2 fi } @@ -304,6 +308,304 @@ } +########################################################################### +# +# Dependency installation, et.al. +# +# +########################################################################### + +# Returns the local dependency directory. +function mkl_depdir { + local dir="$MKLOVE_DIR/deps" + [[ -d $dir ]] || mkdir -p "$dir" + if ! grep -q ^deps$ "$MKLOVE_DIR/.gitignore" 2>/dev/null ; then + echo "deps" >> "$MKLOVE_DIR/.gitignore" + fi + + echo "$dir" +} + +# Returns the package's installation directory / DESTDIR. +function mkl_dep_destdir { + echo "$(mkl_depdir)/dest/$1" +} + +# Returns the package's source directory. +function mkl_dep_srcdir { + echo "$(mkl_depdir)/src/$1" +} + + +# Get the static library file name(s) for a package. +function mkl_lib_static_fnames { + local name=$1 + mkl_meta_get $name "static" "" +} + + +# Returns true if previous ./configure ran a dep install for this package. +function mkl_dep_install_cached { + local name=$1 + + if [[ -n $(mkl_var_get "MKL_STATUS_${1}_INSTALL") ]] || + [[ -n $(mkl_var_get "MKL_STATUS_${1}_INSTALL_SRC") ]]; then + return 0 # true + else + return 1 # false + fi +} + +# Install an external dependency using the platform's native +# package manager. +# Should only be called from mkl_dep_install +# +# Param 1: config name +function mkl_dep_install_pkg { + local name=$1 + local iname="${name}_INSTALL" + local retcode=1 # default to fail + local method="none" + local pkgs + local cmd + + mkl_dbg "Attempting native install of dependency $name on $MKL_DISTRO with effective user $EUID" + + + # Try the platform specific installer first. + case ${MKL_DISTRO}-${EUID} in + debian-0|ubuntu-0) + method=apt + pkgs=$(mkl_meta_get $name deb) + cmd="apt install -y $pkgs" + ;; + + centos-0|rhel-0|redhat-0|fedora-0) + method=yum + pkgs=$(mkl_meta_get $name rpm) + cmd="yum install -y $pkgs" + ;; + + alpine-0) + method=apk + pkgs=$(mkl_meta_get $name apk) + cmd="apk add $pkgs" + ;; + + osx-*) + method=brew + pkgs=$(mkl_meta_get $name brew) + cmd="brew install $pkgs" + ;; + + *) + mkl_dbg "$name: No native installer set for $name on $MKL_DISTRO (euid $EUID)" + return 1 + ;; + esac + + if [[ -z $pkgs ]]; then + mkl_dbg "$name: No packages to install ($method)" + return 1 + fi + + mkl_check_begin --verb "installing dependencies ($cmd)" $iname "" no-cache "$name" + $cmd >>$MKL_OUTDBG 2>&1 + retcode=$? + + if [[ $retcode -eq 0 ]]; then + mkl_dbg "Native install of $name (using $method, $cmd) succeeded" + mkl_check_done "$iname" "" cont "using $method" + mkl_meta_set $name installed_with "$method" + elif [[ $method != "none" ]]; then + mkl_dbg "Native install of $name (using $method, $cmd) failed: retcode $retcode" + mkl_check_failed "$iname" "" cont "using $method" + fi + + return $retcode +} + + +# Returns 0 (yes) if this dependency has a source builder, else 1 (no) +function mkl_dep_has_builder { + local name=$1 + local func="${name}_install_source" + mkl_func_exists $func +} + + +# Returns 0 (yes) if this dependency has a package installer, else 1 (no) +function mkl_dep_has_installer { + local name=$1 + if mkl_dep_has_builder "$name" || \ + [[ -n $(mkl_meta_get $name deb) ]] || \ + [[ -n $(mkl_meta_get $name rpm) ]] || \ + [[ -n $(mkl_meta_get $name brew) ]] || \ + [[ -n $(mkl_meta_get $name apk) ]]; then + return 0 + else + return 1 + fi +} + + +# Install an external dependency from source. +# +# The resulting libraries must be installed in $ddir/usr/lib (or lib64), +# and include files in $ddir/usr/include. +# +# Any dependency installed from source will be linked statically +# regardless of --enable-static, if the build produced static libraries. + +# +# Param 1: config name +function mkl_dep_install_source { + local name=$1 + local iname="${name}_INSTALL_SRC" + local retcode= + + local func="${name}_install_source" + + if ! mkl_dep_has_builder $name ; then + mkl_dbg "No source builder for $name ($func) available" + return 1 + fi + + mkl_check_begin --verb "building dependency" $iname "" no-cache "$name" + + # Create install directory / DESTDIR + local ddir=$(mkl_dep_destdir $name) + [[ -d $ddir ]] || mkdir -p "$ddir" + + # Create and go to source directory + local sdir=$(mkl_dep_srcdir $name) + [[ -d $sdir ]] || mkdir -p "$sdir" + mkl_pushd "$sdir" + + local ilog="${sdir}/_mkl_install.log" + + # Build and install + mkl_dbg "Building $name from source in $sdir (func $func)" + + $func $name "$ddir" >$ilog 2>&1 + retcode=$? + + mkl_popd # $sdir + + if [[ $retcode -eq 0 ]]; then + mkl_dbg "Source install of $name succeeded" + mkl_check_done "$iname" "" cont "ok" "from source" + mkl_meta_set $name installed_with "source" + else + mkl_dbg "Source install of $name failed" + mkl_check_failed "$iname" "" disable "source installer failed (see $ilog)" + mkl_err "$name source build failed, see $ilog for details. Last 50 lines:" + tail -50 "$ilog" + fi + + return $retcode +} + + +# Tries to resolve/find full paths to static libraries for a module, +# using the provided scan dir path. +# Any found libraries are set as STATIC_LIB_.. defines. +# +# Param 1: config name +# Param 2: scandir +# +# Returns 0 if libraries were found, else 1. +function mkl_resolve_static_libs { + local name="$1" + local scandir="$2" + local stlibfnames=$(mkl_lib_static_fnames $name) + local stlibvar="STATIC_LIB_${name}" + + if [[ -z $stlibfnames || -n "${!stlibvar}" ]]; then + mkl_dbg "$name: not resolving static libraries (stlibfnames=$stlibfnames, $stlibvar=${!stlibvar})" + mkl_allvar_set "$name" "WITH_STATIC_LIB_$name" y + return 1 + fi + + local fname= + local stlibs="" + mkl_dbg "$name: resolving static libraries from $stlibfnames in $scandir" + for fname in $stlibfnames ; do + local stlib=$(find "${scandir}" -name "$fname" 2>/dev/null | head -1) + if [[ -n $stlib ]]; then + stlibs="${stlibs} $stlib" + fi + done + + # Trim leading whitespaces + stlibs=${stlibs# } + + if [[ -n $stlibs ]]; then + mkl_dbg "$name: $stlibvar: found static libs: $stlibs" + mkl_var_set $stlibvar "$stlibs" "cache" + mkl_allvar_set "$name" "WITH_STATIC_LIB_$name" y + return 0 + else + mkl_dbg "$name: did not find any static libraries for $stlibfnames in ${scandir}" + return 1 + fi +} + + +# Install an external dependecy +# +# Param 1: config name (e.g zstd) +function mkl_dep_install { + local name=$1 + local retcode= + + local ddir=$(mkl_dep_destdir $name) + + if [[ $MKL_SOURCE_DEPS_ONLY != y ]] || ! mkl_dep_has_builder $name ; then + # + # Try native package manager first, or if no source builder + # is available for this dependency. + # + mkl_dep_install_pkg $name + retcode=$? + + if [[ $retcode -eq 0 ]]; then + return $retcode + fi + fi + + # + # Try source installer. + # + mkl_dep_install_source $name + retcode=$? + + if [[ $retcode -ne 0 ]]; then + if [[ $MKL_SOURCE_DEPS_ONLY == y ]]; then + # Require dependencies, regardless of original action, + # if --source-deps-only is specified, to ensure + # that we do indeed link with the desired library. + mkl_fail "$name" "" fail "Failed to install dependency $name" + fi + return $retcode + fi + + local ddir=$(mkl_dep_destdir $name) + + # Find the static library(s), if any. + if ! mkl_resolve_static_libs "$name" "${ddir}/usr"; then + # No static libraries found, set up dynamic linker path + mkl_mkvar_prepend LDFLAGS LDFLAGS "-L${ddir}/usr/lib64 -L${ddir}/usr/lib" + mkl_mkvar_prepend PKG_CONFIG_PATH PKG_CONFIG_PATH "${ddir}/usr/lib/pkgconfig" ":" + fi + + # Append the package's install path to compiler and linker flags. + mkl_dbg "$name: Adding install-deps paths ($ddir) to compiler and linker flags" + mkl_mkvar_prepend CFLAGS CFLAGS "-I${ddir}/usr/include" + + return $retcode +} + ########################################################################### @@ -342,13 +644,34 @@ " # Dig up some metadata to assist the user case $MKL_DISTRO in - Debian|Ubuntu|*) + debian|ubuntu) local debs=$(mkl_meta_get $conf "deb") - pkg_cmd="sudo apt-get install" + pkg_cmd="sudo apt install -y" if [[ ${#debs} > 0 ]]; then install_pkgs="$install_pkgs $debs" fi ;; + centos|rhel|redhat|fedora) + local rpms=$(mkl_meta_get $conf "rpm") + pkg_cmd="sudo yum install -y" + if [[ ${#rpms} > 0 ]]; then + install_pkgs="$install_pkgs $rpms" + fi + ;; + alpine) + local apks=$(mkl_meta_get $conf "apk") + pkg_cmd="apk add " + if [[ ${#apks} > 0 ]]; then + install_pkgs="$install_pkgs $apks" + fi + ;; + osx) + local pkgs=$(mkl_meta_get $conf "brew") + pkg_cmd="brew install" + if [[ ${#pkgs} > 0 ]]; then + install_pkgs="$install_pkgs $pkgs" + fi + ;; esac done @@ -414,14 +737,13 @@ [[ $MKL_FAILFATAL ]] && action="fail" mkl_check_done "$1" "$2" "$action" "failed" - mkl_dbg "Check $1 ($2, action $action (originally $3)) failed: $4" case $action in fail) # Check failed fatally, fail everything eventually - mkl_fail "$1" "$2" "$3" "$4$extra" + mkl_fail "$1" "$2" "$3" "$4" return 1 ;; @@ -476,6 +798,19 @@ # Generate output files. # Must be called following a succesful configure run. function mkl_generate { + + # Generate MKL_STATIC_LIBS and MKL_DYNAMIC_LIBS from LIBS + local arg= + for arg in $LIBS ; do + if [[ $arg == -l* ]]; then + mkl_mkvar_append "" MKL_DYNAMIC_LIBS $arg + elif [[ $arg == *.a ]]; then + mkl_mkvar_append "" MKL_STATIC_LIBS $arg + else + mkl_dbg "Ignoring arg $arg from LIBS while building STATIC and DYNAMIC lists" + fi + done + local mf= for mf in $MKL_GENERATORS ; do MKL_MODULE=${mf%:*} @@ -483,6 +818,17 @@ $func || exit 1 done + # Generate a built-in options define based on WITH_..=y + local with_y= + for n in $MKL_MKVARS ; do + if [[ $n == WITH_* ]] && [[ ${!n} == y ]]; then + with_y="$with_y ${n#WITH_}" + fi + done + with_y="${with_y# }" + + mkl_allvar_set "BUILT_WITH" "BUILT_WITH" "$with_y" + mkl_write_mk "# Automatically generated by $0 $*" mkl_write_mk "# Config variables" mkl_write_mk "#" @@ -566,7 +912,7 @@ local n= for n in $MKL_MKVARS ; do # Skip the boring booleans - if [[ $n == WITH_* || $n == WITHOUT_* || $n == HAVE_* || $n == def_* ]]; then + if [[ $n == ENABLE_* || $n == WITH_* || $n == WITHOUT_* || $n == HAVE_* || $n == def_* ]]; then continue fi printf " %-24s %s\n" "$n" "${!n}" @@ -608,7 +954,7 @@ if [[ ! -z $MKL_DEBUG ]]; then echo -e "${MKL_BLUE}DBG:$$: $*${MKL_CLR_RESET}" 1>&2 fi - echo "DBG: $*" >> $MKL_OUTDBG + echo "DBG $$: $*" >> $MKL_OUTDBG } # Error print (with color) @@ -748,6 +1094,14 @@ } +# Silent versions of pushd and popd +function mkl_pushd { + pushd "$1" >/dev/null +} + +function mkl_popd { + popd >/dev/null +} ########################################################################### @@ -781,7 +1135,8 @@ IFS="$IFS=" while read -r n v ; do [[ -z $n || $n = \#* || -z $v ]] && continue - mkl_var_set $n $v cache + # Don't let cache overwrite variables + [[ -n ${n+r} ]] || mkl_var_set $n $v cache done < config.cache IFS=$ORIG_IFS } @@ -869,7 +1224,7 @@ fi local name=$(mkl_meta_get $1 name "$4") - [[ -z $name ]] && name="x:$1" + [[ -z $name ]] && name="$1" echo -n "$verb $name..." if [[ $3 != "no-cache" ]]; then @@ -886,6 +1241,26 @@ return 1 } + +# Calls the manual_checks function for the given module. +# Use this for modules that provide check hooks that require +# certain call ordering, such as dependent library checks. +# +# Param 1: module name +function mkl_check { + local modname=$1 + + local func="${modname}_manual_checks" + if ! mkl_func_exists "$func" ; then + mkl_fail "Check function for module $modname not found: missing mkl_require $modname ?" + return 1 + fi + + $func + return $? +} + + # Print that a check is done # Arguments: # config name @@ -898,6 +1273,8 @@ local cname=${1//-/_} mkl_var_set "MKL_STATUS_$cname" "$4" cache + mkl_dbg "Setting $1 ($cname) status to $4 (action $3)" + local extra="" if [[ $4 = "failed" ]]; then local clr=$MKL_YELLOW @@ -922,6 +1299,7 @@ # Perform configure check by compiling source snippet # Arguments: +# [--sub] (run checker as a sub-check, not doing begin/fail/ok) # [--ldflags="..." ] (appended after "compiler arguments" below) # config name # define name @@ -930,12 +1308,22 @@ # compiler arguments (optional "", example: "-lzookeeper") # source snippet function mkl_compile_check { + + local sub=0 + if [[ $1 == --sub ]]; then + sub=1 + shift + fi + local ldf= if [[ $1 == --ldflags=* ]]; then - ldf=${1#*=} - shift + ldf=${1#*=} + shift + fi + + if [[ $sub -eq 0 ]]; then + mkl_check_begin "$1" "$2" "$3" "$1 (by compile)" && return $? fi - mkl_check_begin "$1" "$2" "$3" "$1 (by compile)" && return $? local cflags= @@ -956,14 +1344,14 @@ " >> $srcfile local cmd="${!4} $cflags $(mkl_mkvar_get CPPFLAGS) -Wall -Werror $srcfile -o ${srcfile}.o $ldf $(mkl_mkvar_get LDFLAGS) $5"; - mkl_dbg "Compile check $1 ($2): $cmd" + mkl_dbg "Compile check $1 ($2) (sub=$sub): $cmd" local output output=$($cmd 2>&1) if [[ $? != 0 ]] ; then mkl_dbg "compile check for $1 ($2) failed: $cmd: $output" - mkl_check_failed "$1" "$2" "$3" "compile check failed: + [[ $sub -eq 0 ]] && mkl_check_failed "$1" "$2" "$3" "compile check failed: CC: $4 flags: $5 $cmd: @@ -971,7 +1359,7 @@ source: $6" local ret=1 else - mkl_check_done "$1" "$2" "$3" "ok" + [[ $sub -eq 0 ]] && mkl_check_done "$1" "$2" "$3" "ok" local ret=0 fi @@ -983,6 +1371,31 @@ } +# Low-level: Try to link with a library. +# Arguments: +# linker flags (e.g. "-lpthreads") +function mkl_link_check0 { + local libs=$1 + local srcfile=$(mktemp _mktmpXXXXXX) + echo "#include +int main () { FILE *fp = stderr; return fp ? 0 : 0; }" > ${srcfile}.c + + local cmd="${CC} $(mkl_mkvar_get CFLAGS) $(mkl_mkvar_get LDFLAGS) ${srcfile}.c -o ${srcfile}_out $libs"; + mkl_dbg "Link check for $1: $cmd" + + local output + output=$($cmd 2>&1) + local retcode=$? + + if [[ $retcode -ne 0 ]] ; then + mkl_dbg "Link check for $1 failed: $output" + fi + + rm -f $srcfile* + return $retcode +} + + # Try to link with a library. # Arguments: # config name @@ -992,54 +1405,96 @@ function mkl_link_check { mkl_check_begin "$1" "$2" "$3" "$1 (by linking)" && return $? - local srcfile=$(mktemp _mktmpXXXXXX) - echo "int main () { return 0; }" > $srcfile - - local cmd="${CC} $(mkl_mkvar_get LDFLAGS) -c $srcfile -o ${srcfile}.o $4"; - mkl_dbg "Link check $1 ($2): $cmd" - - local output - output=$($cmd 2>&1) - - if [[ $? != 0 ]] ; then + if mkl_link_check0 "$4" ; then + mkl_check_done "$1" "$2" "$3" "ok" "$4" + return 0 + else mkl_dbg "link check for $1 ($2) failed: $output" mkl_check_failed "$1" "$2" "$3" "compile check failed: $output" - local ret=1 - else - mkl_check_done "$1" "$2" "$3" "ok" "$4" - local ret=0 + return 1 fi - - rm -f $srcfile* - return $ret } # Tries to figure out if we can use a static library or not. +# +# WARNING: This function must not emit any stdout output other than the +# updated list of libs. Do not use any stdout-printing checker. +# # Arguments: -# library name (e.g. -lrdkafka) -# compiler flags (optional "", e.g: "-lyajl") +# config name (e.g., zstd) +# compiler flags (optional "", e.g: "-lzstd") # Returns/outputs: # New list of compiler flags function mkl_lib_check_static { - local libname=$1 + local configname=$1 local libs=$2 - local arfile_var=STATIC_LIB_${libname#-l} + local arfile_var=STATIC_LIB_${configname} + local stfnames=$(mkl_lib_static_fnames $configname) - mkl_dbg "Check $libname for static library (libs $libs, arfile variable $arfile_var=${!arfile_var})" + mkl_dbg "$configname: Check for static library (libs $libs, arfile variable $arfile_var=${!arfile_var}, static filenames $stfnames)" + + # If STATIC_LIB_ specifies .a file(s) we use that instead. + if [[ -n ${!arfile_var} ]]; then + libs="${!arfile_var}" + + elif [[ $WITH_STATIC_LINKING != y ]]; then + # Static linking not enabled + echo "" + return + + elif [[ $HAS_LDFLAGS_STATIC == y ]] && [[ -n $stfnames ]]; then + local libname + local stlibs= + for libname in $stfnames; do + # Convert the static filename to a linker flag: + # libzstd.a -> -lzstd + libname=${libname#lib} + libname="-l${libname%.a}" + stlibs="${stlibs}${libname} " + done + libs="${LDFLAGS_STATIC} $stlibs ${LDFLAGS_DYNAMIC}" + mkl_dbg "$configname: after replacing libs: $libs" + + elif [[ $libs == *-L* ]]; then + # Try to resolve full static paths using any -Lpaths in $libs + local lpath + for lpath in $libs; do + [[ $lpath == -L* ]] || continue + + lpath="${lpath#-L}" + [[ -d $lpath ]] || continue + + if mkl_resolve_static_libs "$configname" "$lpath"; then + break + fi + done + + libs="${!arfile_var}" + mkl_dbg "$configname: after -L resolve, libs is $libs" - # If STATIC_LIB_ specifies an existing .a file we - # use that instead. - if [[ -f ${!arfile_var} ]]; then - libs=$(echo $libs | sed -e "s|$libname|${!arfile_var}|g") - elif [[ $HAS_LDFLAGS_STATIC == y ]]; then - libs=$(echo $libs | sed -e "s|$libname|${LDFLAGS_STATIC} $libname ${LDFLAGS_DYNAMIC}|g") else - mkl_dbg "$libname: Neither $arfile_var specified or static linker flags supported: static linking probably won't work" + mkl_dbg "$configname: Neither $arfile_var=/path/to/libname.a specified nor static linker flags supported: static linking probably won't work" + libs="" + fi + + if [[ -z $libs ]]; then + echo "" + return fi + # Attempt to link a small program with these static libraries + mkl_dbg "$configname: verifying that linking \"$libs\" works" + if ! mkl_link_check0 "$libs" ; then + mkl_dbg "$configname: Could not use static libray flags: $libs" + echo "" + return + fi + + mkl_allvar_set "$configname" "${configname}_STATIC" "y" + echo $libs } @@ -1050,50 +1505,40 @@ # If STATIC_LIB_ is set to the path of an .a file # it will be used instead of -l. # +# _STATIC will be automatically defined (for both Makefile.config +# and config.h) if the library is to be linked statically, or was installed +# with a source dependency installer. +# # Arguments: -# [--static=] (allows static linking (--enable-static) for the -# library provided, e.g.: --static=-lrdkafka "librdkafka"... -# If not set it will default to the compiler flags.) -# [--no-static] (don't attempt to link this library statically) +# [--override-action=] (internal use, overrides action argument) +# [--no-static] (do not attempt to link the library statically) # [--libname=] (library name if different from config name, such as # when the libname includes a dash) -# [--no-LIBS] (don't add this library to LIBS, useful for -# plugin libraries that rely on the parent process -# to have done all necessary linking) # config name (library name (for pkg-config)) # define name # action (fail|disable|cont) # compiler (CC|CXX) # compiler flags (optional "", e.g: "-lyajl") # source snippet -function mkl_lib_check { +function mkl_lib_check0 { - local staticopt= - local no_staticopt= + local override_action= + local nostaticopt= local libnameopt= local libname= - local no_LIBSopt= while [[ $1 == --* ]]; do - case $1 in - --static=*) - staticopt=$1 - ;; - --no-static) - no_staticopt=$1 - ;; - --libname=*) - libnameopt=$1 - libname="${libnameopt#*=}" - ;; - --no-LIBS) - no_LIBSopt=$1 - ;; - *) - mkl_err "Invalid option to mkl_lib_check: $1" - return 1 - ;; - esac + if [[ $1 == --override-action=* ]]; then + override_action=${1#*=} + elif [[ $1 == --no-static ]]; then + nostaticopt=$1 + elif [[ $1 == --libname* ]]; then + libnameopt=$1 + libname="${libnameopt#*=}" + else + mkl_err "mkl_lib_check: invalid option $1" + exit 1 + fi shift done @@ -1101,34 +1546,37 @@ libname=$1 fi + local action=$3 + if [[ -n $override_action ]]; then + action=$override_action + fi + # pkg-config result (0=ok) local pkg_conf_failed=1 if [[ $WITH_PKGCONFIG == "y" ]]; then # Let pkg-config populate CFLAGS, et.al. - mkl_pkg_config_check $no_staticopt $staticopt $libnameopt $no_LIBSopt "$1" "" cont - pkg_conf_failed=$? + # Return on success. + mkl_pkg_config_check $nostaticopt $libnameopt "$1" "$2" cont "$4" "$6" && return $? fi - local libs="" - if [[ $pkg_conf_failed ]]; then - libs="$5" - if [[ -z $no_staticopt && $WITH_STATIC_LINKING == y ]]; then - local use_staticopt="$libs" - if [[ ! -z $staticopt ]]; then - use_staticopt="${staticopt#*=}" - fi - libs=$(mkl_lib_check_static "$use_staticopt" "$libs") + local libs="$5" + + if [[ -z $nostaticopt ]]; then + local stlibs=$(mkl_lib_check_static $1 "$libs") + if [[ -n $stlibs ]]; then + libs=$stlibs fi fi - if ! mkl_compile_check "$1" "$2" "$3" "$4" "$libs" "$6"; then + if ! mkl_compile_check "$1" "$2" "$action" "$4" "$libs" "$6"; then return 1 fi - if [[ $pkg_conf_failed == 1 && -z "$no_LIBSopt" ]]; then + if [[ -n $libs ]]; then # Add libraries in reverse order to make sure inter-dependencies # are resolved in the correct order. # E.g., check for crypto and then ssl should result in -lssl -lcrypto + mkl_dbg "$1: from lib_check: LIBS: prepend $libs" mkl_mkvar_prepend "$1" LIBS "$libs" fi @@ -1136,94 +1584,149 @@ } +# Wrapper for mkl_lib_check0 which attempts dependency installation +# if --install-deps is specified. +# +# See mkl_lib_check0 for arguments and details. +function mkl_lib_check { + + local arg= + local name= + + # Find config name parameter (first non-option (--...)) + for arg in $* ; do + if [[ $arg == --* ]]; then + continue + fi + name=$arg + break + done + + if [[ $MKL_INSTALL_DEPS != y ]] || ! mkl_dep_has_installer "$name" ; then + mkl_lib_check0 "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" + return $? + fi + + + # Automatic dependency installation mode: + # First pass is lib check with cont, + # if it fails, attempt dependency installation, + # and then make second with caller's fail-action. + + local retcode= + + # With --source-deps-only we want to make sure the dependency + # being used is in-fact from the dependency builder (if supported), + # rather than a system installed alternative, so skip the pre-check and + # go directly to dependency installation/build below. + if [[ $MKL_SOURCE_DEPS_ONLY != y ]] || ! mkl_dep_has_builder $name ; then + mkl_lib_check0 --override-action=cont "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" + retcode=$? + if [[ $retcode -eq 0 ]]; then + # Successful on first pass + return $retcode + fi + else + mkl_dbg "$name: skipping dependency pre-check in favour of --source-deps-only" + fi + + # Install dependency + if ! mkl_dep_install "$name" ; then + return 1 + fi + + # Second pass: check again, this time fail hard + mkl_lib_check0 --override-action=fail "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" + return $? +} + + + # Check for library with pkg-config # Automatically sets CFLAGS and LIBS from pkg-config information. # Arguments: -# [--static=] (allows static linking (--enable-static) for the -# library provided, e.g.: --static=-lrdkafka "librdkafka"..) -# [--no-static] (don't attempt to link this library statically) +# [--no-static] (do not attempt to link the library statically) # [--libname=] (library name if different from config name, such as # when the libname includes a dash) -# [--no-LIBS] (don't add this library to LIBS, useful for -# plugin libraries that rely on the parent process -# to have done all necessary linking) # config name # define name -# action (fail|disable|ignore) +# action (fail|disable|ignore|cont) +# compiler (CC|CXX) +# source snippet function mkl_pkg_config_check { - local staticopt= - local no_staticopt= - local libnameopt= - local libname= - local no_LIBSopt= - - - while [[ $1 == --* ]]; do - case $1 in - --static=*) - staticopt=$1 - ;; - --no-static) - no_staticopt=$1 - ;; - --libname*) - libnameopt=$1 - libname="${libnameopt#*=}" - ;; - --no-LIBS) - no_LIBSopt=$1 - ;; - *) - mkl_err "Invalid option to mkl_pkg_config_check: $1" - return 1 - ;; - esac + local nostaticopt= + if [[ $1 == --no-static ]]; then + nostaticopt=$1 shift - done - - if [[ -z $libname ]]; then - libname=$1 fi + local libname=$1 + if [[ $1 == --libname* ]]; then + libname="${libnameopt#*=}" + shift + fi local cname="${1}_PKGCONFIG" mkl_check_begin "$cname" "$2" "no-cache" "$1 (by pkg-config)" && return $? local cflags= local cmd="${PKG_CONFIG} --short-errors --cflags $libname" - mkl_dbg "pkg-config check $libname ($2): $cmd" + mkl_dbg "pkg-config check $libname for CFLAGS ($2): $cmd" cflags=$($cmd 2>&1) if [[ $? != 0 ]]; then mkl_dbg "'$cmd' failed: $cflags" - mkl_check_failed "$cname" "$2" "$3" "'$cmd' failed: + # Clear define name ($2): caller may have additional checks + mkl_check_failed "$cname" "" "$3" "'$cmd' failed: $cflags" return 1 fi + if [[ $(mkl_meta_get $1 installed_with) == "source" && \ + $WITH_STATIC_LINKING == y && \ + $MKL_SOURCE_DEPS_ONLY == y ]]; then + # If attempting static linking and we're using source-only + # dependencies, then there is no need for pkg-config since + # the source installer will have set the required flags. + return 1 + fi + local libs= - libs=$(${PKG_CONFIG} --short-errors --libs $libname 2>&1) + cmd="${PKG_CONFIG} --short-errors --libs $libname" + mkl_dbg "pkg-config check $libname for LIBS ($2): $cmd" + libs=$($cmd 2>&1) if [[ $? != 0 ]]; then mkl_dbg "${PKG_CONFIG} --libs $libname failed: $libs" - mkl_check_failed "$cname" "$2" "$3" "pkg-config --libs failed" + # Clear define name ($2): caller may have additional checks + mkl_check_failed "$cname" "" "$3" "pkg-config --libs failed" return 1 fi - mkl_mkvar_append $1 "CFLAGS" "$cflags" + mkl_dbg "$1: from pkg-config: CFLAGS '$CFLAGS', LIBS '$LIBS'" + + local snippet="$5" + if [[ -n $snippet ]]; then + mkl_dbg "$1: performing compile check using pkg-config info" - if [[ -z $no_staticopt && $WITH_STATIC_LINKING == y ]]; then - local use_staticopt="$libs" - if [[ ! -z $staticopt ]]; then - use_staticopt="${staticopt#*=}" + if ! mkl_compile_check --sub "$1" "$2" "no-cache" "$4" "$cflags $libs" "$snippet"; then + mkl_check_failed "$cname" "" "$3" "compile check failed" + return 1 fi - libs=$(mkl_lib_check_static "$use_staticopt" "$libs") fi - if [[ -z $no_LIBSopt ]]; then - mkl_mkvar_prepend "$1" LIBS "$libs" + mkl_mkvar_append $1 "CFLAGS" "$cflags" + + if [[ -z $nostaticopt ]]; then + local stlibs=$(mkl_lib_check_static $1 "$libs") + if [[ -n $stlibs ]]; then + libs=$stlibs + fi fi + mkl_dbg "$1: from pkg-config: LIBS: prepend $libs" + mkl_mkvar_prepend "$1" LIBS "$libs" + mkl_check_done "$1" "$2" "$3" "ok" return 0 @@ -1552,6 +2055,8 @@ # Rename module's special functions so we can call them separetely later. mkl_func_rename "options" "${modname}_options" + mkl_func_rename "install_source" "${modname}_install_source" + mkl_func_rename "manual_checks" "${modname}_manual_checks" mkl_func_push MKL_CHECKS "$modname" "checks" mkl_func_push MKL_GENERATORS "$modname" "generate" mkl_func_push MKL_CLEANERS "$modname" "clean" @@ -1625,7 +2130,7 @@ MKL_USAGE="Usage: ./configure [OPTIONS...] mklove configure script - mklove, not autoconf - Copyright (c) 2014-2015 Magnus Edenhill - https://github.com/edenhill/mklove + Copyright (c) 2014-2019 Magnus Edenhill - https://github.com/edenhill/mklove " function mkl_usage { @@ -1668,11 +2173,13 @@ # Arguments: # option group ("Standard", "Cross-Compilation", etc..) # variable name -# option ("--foo=feh") +# option ("--foo", "--foo=*", "--foo=args_required") # help # default (optional) # assignvalue (optional, default:"y") # function block (optional) +# +# If option takes the form --foo=* then arguments are optional. function mkl_option { local optgroup=$1 local varname=$2 @@ -1704,6 +2211,10 @@ if [[ $3 == *=* ]]; then optname="${optname%=*}" optval="${3#*=}" + if [[ $optval == '*' ]]; then + # Avoid globbing of --foo=* optional arguments + optval='\*' + fi fi safeopt=$(mkl_env_esc $optname) @@ -1780,7 +2291,7 @@ # Arguments: # option group ("Standard", ..) # variable name (WITH_FOO) -# option (--enable-foo) +# option (--enable-foo, --enable-foo=*, or --enable-foo=req) # help ("foo.." ("Enable" and "Disable" will be prepended)) # default (y or n) diff -Nru kafkacat-1.5.0/mklove/modules/configure.builtin kafkacat-1.6.0/mklove/modules/configure.builtin --- kafkacat-1.5.0/mklove/modules/configure.builtin 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/mklove/modules/configure.builtin 2020-07-21 09:00:07.000000000 +0000 @@ -22,6 +22,8 @@ "Modifiable arch-independent data" "\$prefix/com" mkl_option "Standard" localstatedir "--localstatedir=PATH" \ "Modifiable local state data" "\$prefix/var" +mkl_option "Standard" runstatedir "--runstatedir=PATH" \ + "Modifiable per-process data" "\$prefix/var/run" mkl_option "Standard" libdir "--libdir=PATH" "Libraries" "\$exec_prefix/lib" mkl_option "Standard" includedir "--includedir=PATH" "C/C++ header files" \ "\$prefix/include" @@ -41,12 +43,18 @@ mkl_option "Configure tool" "" "--help" "Show configure usage" -mkl_toggle_option "Compatibility" "mk:MKL_MAINT_MODE" "--enable-maintainer-mode" "Maintainer mode (no-op)" +# These autoconf compatibility options are ignored by mklove +mkl_toggle_option "Compatibility" "mk:COMPAT_MAINT_MODE" "--enable-maintainer-mode" "Maintainer mode (no-op)" +mkl_option "Compatibility" "mk:PROGRAM_PREFIX" "--program-prefix=PFX" "Program prefix (no-op)" +mkl_option "Compatibility" "mk:COMPAT_DISABLE_DEP_TRACK" "--disable-dependency-tracking" "Disable dependency tracking (no-op)" +mkl_option "Compatibility" "mk:COMPAT_DISABLE_SILENT_RULES" "--disable-silent-rules" "Verbose build output (no-op)" +mkl_option "Compatibility" "mk:COMPAT_SILENT" "--silent" "Less verbose build output (no-op)" +mkl_toggle_option "Compatibility" "mk:COMPAT_ENABLE_SHARED" "--enable-shared" "Build shared library (no-op)" +mkl_toggle_option "Compatibility" "mk:COMPAT_DISABLE_OPT_CHECK" '--enable-option-checking=*' "Disable configure option checking (no-op)" -mkl_option "Configure tool" "mk:PROGRAM_PREFIX" "--program-prefix=PFX" "Program prefix" -mkl_option "Compatibility" "mk:DISABL_DEP_TRACK" "--disable-dependency-tracking" "Disable dependency tracking (no-op)" -mkl_option "Compatibility" "mk:DISABL_SILENT_RULES" "--disable-silent-rules" "Verbose build output (no-op)" +mkl_option "Dependency" env:MKL_INSTALL_DEPS "--install-deps" "Attempt to install missing dependencies" +mkl_option "Dependency" env:MKL_SOURCE_DEPS_ONLY "--source-deps-only" "Only perform source builds of dependencies, not using any package managers" function checks { diff -Nru kafkacat-1.5.0/mklove/modules/configure.cc kafkacat-1.6.0/mklove/modules/configure.cc --- kafkacat-1.5.0/mklove/modules/configure.cc 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/mklove/modules/configure.cc 2020-07-21 09:00:07.000000000 +0000 @@ -49,7 +49,7 @@ mkl_mkvar_append CPPFLAGS CPPFLAGS "-m$MBITS" mkl_mkvar_append LDFLAGS LDFLAGS "-m$MBITS" fi - if [[ -z "$ARFLAGS" && $MBITS == 64 && $MKL_DISTRO == "SunOS" ]]; then + if [[ -z "$ARFLAGS" && $MBITS == 64 && $MKL_DISTRO == "sunos" ]]; then # Turn on 64-bit archives on SunOS mkl_mkvar_append ARFLAGS ARFLAGS "S" fi @@ -57,7 +57,7 @@ # Provide prefix and checks for various other build tools. local t= - for t in LD:ld NM:nm OBJDUMP:objdump STRIP:strip ; do + for t in LD:ld NM:nm OBJDUMP:objdump STRIP:strip LIBTOOL:libtool ; do local tenv=${t%:*} t=${t#*:} local tval="${!tenv}" @@ -109,7 +109,7 @@ # install if [ -z "$INSTALL" ]; then - if [[ $MKL_DISTRO == "SunOS" ]]; then + if [[ $MKL_DISTRO == "sunos" ]]; then mkl_meta_set ginstall name "GNU install" if mkl_command_check ginstall "" ignore "ginstall --version"; then INSTALL=ginstall @@ -154,6 +154,8 @@ # OSX linker can't enable/disable static linking so we'll # need to find the .a through STATIC_LIB_libname env var mkl_mkvar_set staticlinking HAS_LDFLAGS_STATIC n + # libtool -static supported + mkl_mkvar_set staticlinking HAS_LIBTOOL_STATIC y fi fi } diff -Nru kafkacat-1.5.0/mklove/modules/configure.host kafkacat-1.6.0/mklove/modules/configure.host --- kafkacat-1.5.0/mklove/modules/configure.host 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/mklove/modules/configure.host 2020-07-21 09:00:07.000000000 +0000 @@ -19,10 +19,10 @@ mkl_option "Cross-compilation" "mk:BUILD" "--build=BUILD" "Configure for building on BUILD (no-op)" mkl_option "Cross-compilation" "mk:TARGET" "--target=TARGET" "Configure for building cross-toolkits for platform TARGET (no-op)" -function checks { - # Try to figure out what OS/distro we are running on. - mkl_check_begin "distro" "" "no-cache" "OS or distribution" +# Resolve the OS/distro at import time, rather than as a check, +# so that MKL_DISTRO is available to other modules at import time. +function resolve_distro { solib_ext=.so # Try lsb_release @@ -35,6 +35,17 @@ Linux) sys=Linux solib_ext=.so + + if [[ -f /etc/os-release ]]; then + eval $(grep ^ID= /etc/os-release) + if [[ -n $ID ]]; then + sys="$ID" + fi + elif [[ -f /etc/centos-release ]]; then + sys=centos + elif [[ -f /etc/alpine-release ]]; then + sys=alpine + fi ;; Darwin) sys=osx @@ -51,12 +62,23 @@ esac fi - if [[ -z $sys ]]; then + # Convert to lower case + sys=$(echo $sys | tr '[:upper:]' '[:lower:]') + mkl_mkvar_set "distro" "MKL_DISTRO" "$sys" + mkl_allvar_set "distro" "SOLIB_EXT" "$solib_ext" +} + +resolve_distro + + +function checks { + # Try to figure out what OS/distro we are running on. + mkl_check_begin "distro" "" "no-cache" "OS or distribution" + + if [[ -z $MKL_DISTRO ]]; then mkl_check_failed "distro" "" "ignore" "" else - mkl_check_done "distro" "" "ignore" "ok" "$sys" - mkl_mkvar_set "distro" "MKL_DISTRO" "$sys" - mkl_allvar_set "distro" "SOLIB_EXT" "$solib_ext" + mkl_check_done "distro" "" "ignore" "ok" "$MKL_DISTRO" fi } diff -Nru kafkacat-1.5.0/rdendian.h kafkacat-1.6.0/rdendian.h --- kafkacat-1.5.0/rdendian.h 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/rdendian.h 2020-07-21 09:00:07.000000000 +0000 @@ -41,34 +41,34 @@ */ #ifdef __FreeBSD__ - #include +#include #elif defined __GLIBC__ - #include - #ifndef be64toh - /* Support older glibc (<2.9) which lack be64toh */ - #include - #if __BYTE_ORDER == __BIG_ENDIAN - #define be16toh(x) (x) - #define be32toh(x) (x) - #define be64toh(x) (x) - #define le64toh(x) __bswap_64 (x) - #define le32toh(x) __bswap_32 (x) - #else - #define be16toh(x) __bswap_16 (x) - #define be32toh(x) __bswap_32 (x) - #define be64toh(x) __bswap_64 (x) - #define le64toh(x) (x) - #define le32toh(x) (x) - #endif - #endif +#include +#ifndef be64toh +/* Support older glibc (<2.9) which lack be64toh */ +#include +#if __BYTE_ORDER == __BIG_ENDIAN +#define be16toh(x) (x) +#define be32toh(x) (x) +#define be64toh(x) (x) +#define le64toh(x) __bswap_64 (x) +#define le32toh(x) __bswap_32 (x) +#else +#define be16toh(x) __bswap_16 (x) +#define be32toh(x) __bswap_32 (x) +#define be64toh(x) __bswap_64 (x) +#define le64toh(x) (x) +#define le32toh(x) (x) +#endif +#endif #elif defined __CYGWIN__ - #include +#include #elif defined __BSD__ - #include +#include #elif defined __sun - #include - #include +#include +#include #define __LITTLE_ENDIAN 1234 #define __BIG_ENDIAN 4321 #ifdef _BIG_ENDIAN @@ -92,8 +92,8 @@ #endif /* __sun */ #elif defined __APPLE__ - #include - #include +#include +#include #if __DARWIN_BYTE_ORDER == __DARWIN_BIG_ENDIAN #define be64toh(x) (x) #define be32toh(x) (x) @@ -129,17 +129,17 @@ (((x) & 0xff00) << 8) | \ (((x) & 0xff0000) >> 8) | \ (((x) & 0xff000000) >> 24)) -#define le64toh(x) \ - ((((x) & 0x00000000000000ffL) << 56) | \ - (((x) & 0x000000000000ff00L) << 40) | \ - (((x) & 0x0000000000ff0000L) << 24) | \ - (((x) & 0x00000000ff000000L) << 8) | \ - (((x) & 0x000000ff00000000L) >> 8) | \ - (((x) & 0x0000ff0000000000L) >> 24) | \ - (((x) & 0x00ff000000000000L) >> 40) | \ +#define le64toh(x) \ + ((((x) & 0x00000000000000ffL) << 56) | \ + (((x) & 0x000000000000ff00L) << 40) | \ + (((x) & 0x0000000000ff0000L) << 24) | \ + (((x) & 0x00000000ff000000L) << 8) | \ + (((x) & 0x000000ff00000000L) >> 8) | \ + (((x) & 0x0000ff0000000000L) >> 24) | \ + (((x) & 0x00ff000000000000L) >> 40) | \ (((x) & 0xff00000000000000L) >> 56)) #else - #include +#include #endif diff -Nru kafkacat-1.5.0/rdport.h kafkacat-1.6.0/rdport.h --- kafkacat-1.5.0/rdport.h 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/rdport.h 2020-07-21 09:00:07.000000000 +0000 @@ -1,3 +1,31 @@ +/* + * librdkafka - Apache Kafka C library + * + * Copyright (c) 2012-2019 Magnus Edenhill + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #pragma once /** @@ -21,26 +49,26 @@ typedef SSIZE_T ssize_t; ssize_t getdelim (char **bufptr, size_t *n, - int delim, FILE *fp); + int delim, FILE *fp); /** -* @brief gettimeofday() for win32 -*/ + * @brief gettimeofday() for win32 + */ static RD_UNUSED int rd_gettimeofday (struct timeval *tv, struct timezone *tz) { - SYSTEMTIME st; - FILETIME ft; - ULARGE_INTEGER d; - - GetSystemTime(&st); - SystemTimeToFileTime(&st, &ft); - d.HighPart = ft.dwHighDateTime; - d.LowPart = ft.dwLowDateTime; - tv->tv_sec = (long)((d.QuadPart - 116444736000000000llu) / 10000000L); - tv->tv_usec = (long)(st.wMilliseconds * 1000); + SYSTEMTIME st; + FILETIME ft; + ULARGE_INTEGER d; + + GetSystemTime(&st); + SystemTimeToFileTime(&st, &ft); + d.HighPart = ft.dwHighDateTime; + d.LowPart = ft.dwLowDateTime; + tv->tv_sec = (long)((d.QuadPart - 116444736000000000llu) / 10000000L); + tv->tv_usec = (long)(st.wMilliseconds * 1000); - return 0; + return 0; } @@ -53,4 +81,4 @@ #define _COMPAT(FUNC) FUNC #define rd_gettimeofday(tv,tz) gettimeofday(tv,tz) -#endif \ No newline at end of file +#endif diff -Nru kafkacat-1.5.0/README.md kafkacat-1.6.0/README.md --- kafkacat-1.5.0/README.md 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/README.md 2020-07-21 09:00:07.000000000 +0000 @@ -2,7 +2,7 @@ # kafkacat -Copyright (c) 2014-2019 Magnus Edenhill +Copyright (c) 2014-2020 Magnus Edenhill [https://github.com/edenhill/kafkacat](https://github.com/edenhill/kafkacat) @@ -37,7 +37,7 @@ ```bash # List brokers and topics in cluster -$ docker run -it --network=host edenhill/kafkacat:1.5.0 -b YOUR_BROKER -L +$ docker run -it --network=host edenhill/kafkacat:1.6.0 -b YOUR_BROKER -L ``` See [Examples](#examples) for usage options, and [Running in Docker](#running-in-docker) for more information on how to properly run docker-based clients with Kafka. @@ -51,12 +51,23 @@ apt-get install kafkacat ```` +On recent openSUSE systems: + +``` +zypper addrepo https://download.opensuse.org/repositories/network:utilities/openSUSE_Factory/network:utilities.repo +zypper refresh +zypper install kafkacat +``` +(see [this page](https://software.opensuse.org/download/package?package=kafkacat&project=network%3Autilities) for instructions to install with openSUSE LEAP) + And on Mac OS X with homebrew installed: ```` brew install kafkacat ```` +See [this blog](https://rmoff.net/2020/04/20/how-to-install-kafkacat-on-fedora/) for how to install kafkacat on recent Fedora systems. + Otherwise follow directions below. @@ -121,6 +132,12 @@ $ kafkacat -P -b mybroker -t filedrop -p 0 myfile1.bin /etc/motd thirdfile.tgz + +Produce messages transactionally (one single transaction for all messages): + + $ kafkacat -B -b mybroker -t mytopic -X transactional.id=myproducerapp + + Read the last 2000 messages from 'syslog' topic, then exit $ kafkacat -C -b mybroker -t syslog -p 0 -o -2000 -e @@ -185,6 +202,11 @@ $ kafkacat -b mybroker -X enable.idempotence=true -P -t mytopic .... +Connect to cluster using SSL and SASL PLAIN authentication: + + $ kafkacat -b mybroker -X security.protocol=SASL_SSL -X sasl.mechanism=PLAIN -X sasl.username=myapikey -X sasl.password=myapisecret ... + + Metadata listing: ``` @@ -232,7 +254,7 @@ ## Running in Docker -The latest kafkacat docker image is `edenhill/kafkacat:1.5.0`, there's +The latest kafkacat docker image is `edenhill/kafkacat:1.6.0`, there's also [Confluent's kafkacat docker images on Docker Hub](https://hub.docker.com/r/confluentinc/cp-kafkacat/). If you are connecting to Kafka brokers also running on Docker you should specify the network name as part of the `docker run` command using the `--network` parameter. For more details of networking with Kafka and Docker [see this post](https://rmoff.net/2018/08/02/kafka-listeners-explained/). diff -Nru kafkacat-1.5.0/tools.c kafkacat-1.6.0/tools.c --- kafkacat-1.5.0/tools.c 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/tools.c 2020-07-21 09:00:07.000000000 +0000 @@ -61,7 +61,8 @@ errstr, sizeof(errstr)))) KC_FATAL("Failed to create producer: %s", errstr); - err = rd_kafka_offsets_for_times(conf.rk, offsets, 10*1000); + err = rd_kafka_offsets_for_times(conf.rk, offsets, + conf.metadata_timeout * 1000); #else err = RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED; #endif diff -Nru kafkacat-1.5.0/.travis.yml kafkacat-1.6.0/.travis.yml --- kafkacat-1.5.0/.travis.yml 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/.travis.yml 2020-07-21 09:00:07.000000000 +0000 @@ -12,3 +12,4 @@ script: - ./bootstrap.sh + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then otool -L kafkacat ; else ldd kafkacat ; fi diff -Nru kafkacat-1.5.0/win32/kafkacat.vcxproj kafkacat-1.6.0/win32/kafkacat.vcxproj --- kafkacat-1.5.0/win32/kafkacat.vcxproj 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/win32/kafkacat.vcxproj 2020-07-21 09:00:07.000000000 +0000 @@ -1,6 +1,5 @@ - Debug @@ -130,12 +129,12 @@ - + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + diff -Nru kafkacat-1.5.0/win32/packages.config kafkacat-1.6.0/win32/packages.config --- kafkacat-1.5.0/win32/packages.config 2019-09-12 08:30:57.000000000 +0000 +++ kafkacat-1.6.0/win32/packages.config 2020-07-21 09:00:07.000000000 +0000 @@ -1,4 +1,4 @@ - +