diff -Nru librdkafka-0.11.3/.appveyor.yml librdkafka-0.11.6/.appveyor.yml
--- librdkafka-0.11.3/.appveyor.yml 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/.appveyor.yml 2018-10-10 06:54:38.000000000 +0000
@@ -1,4 +1,4 @@
-version: 0.11.1-R-post{build}
+version: 0.11.4-R-pre{build}
pull_requests:
do_not_increment_build_number: true
image: Visual Studio 2013
@@ -8,7 +8,7 @@
- platform: x64
- platform: win32
install:
-- ps: "$OpenSSLVersion = \"1_0_2m\"\n$OpenSSLExe = \"OpenSSL-$OpenSSLVersion.exe\"\n\nRemove-Item C:\\OpenSSL-Win32 -recurse\nRemove-Item C:\\OpenSSL-Win64 -recurse\n\nWrite-Host \"Installing OpenSSL v1.0 32-bit ...\" -ForegroundColor Cyan\nWrite-Host \"Downloading...\"\n$exePath = \"$($env:USERPROFILE)\\Win32OpenSSL-1_0_2m.exe\"\n(New-Object Net.WebClient).DownloadFile('https://slproweb.com/download/Win32OpenSSL-1_0_2m.exe', $exePath)\nWrite-Host \"Installing...\"\ncmd /c start /wait $exePath /silent /verysilent /sp- /suppressmsgboxes /DIR=C:\\OpenSSL-Win32\nWrite-Host \"Installed\" -ForegroundColor Green\n\nWrite-Host \"Installing OpenSSL v1.0 64-bit ...\" -ForegroundColor Cyan\nWrite-Host \"Downloading...\"\n$exePath = \"$($env:USERPROFILE)\\Win64OpenSSL-1_0_2m.exe\"\n(New-Object Net.WebClient).DownloadFile('https://slproweb.com/download/Win64OpenSSL-1_0_2m.exe', $exePath)\nWrite-Host \"Installing...\"\ncmd /c start /wait $exePath /silent /verysilent /sp- /suppressmsgboxes /DIR=C:\\OpenSSL-Win64\nWrite-Host \"Installed\" -ForegroundColor Green\n\nif (!(Test-Path(\"C:\\OpenSSL-Win32\"))) {\n echo \"Downloading https://slproweb.com/download/Win32$OpenSSLExe\"\n Start-FileDownload 'https://slproweb.com/download/Win32$OpenSSLExe'\n Start-Process \"Win32$OpenSSLExe\" -ArgumentList \"/silent /verysilent /sp- /suppressmsgboxes\" -Wait\n} else {\n echo \"OpenSSL-Win32 already exists: not downloading\"\n}\n\nif (!(Test-Path(\"C:\\OpenSSL-Win64\"))) {\n echo \"Downloading https://slproweb.com/download/Win64$OpenSSLExe\"\n Start-FileDownload 'https://slproweb.com/download/Win64$OpenSSLExe' \n Start-Process \"Win64$OpenSSLExe\" -ArgumentList \"/silent /verysilent /sp- /suppressmsgboxes\" -Wait\n} else {\n echo \"OpenSSL-Win64 already exists: not downloading\"\n}\n\n\n\n# Download the CoApp tools.\n$msiPath = \"$($env:USERPROFILE)\\CoApp.Tools.Powershell.msi\"\n(New-Object Net.WebClient).DownloadFile('http://coapp.org/files/CoApp.Tools.Powershell.msi', $msiPath)\n\n# Install the CoApp tools from the downloaded .msi.\nStart-Process -FilePath msiexec -ArgumentList /i, $msiPath, /quiet -Wait\n\n# Make the tools available for later PS scripts to use.\n$env:PSModulePath = $env:PSModulePath + ';C:\\Program Files (x86)\\Outercurve Foundation\\Modules'\nImport-Module CoApp\n\n# Install NuGet\n#Install-PackageProvider NuGet -MinimumVersion '2.8.5.201' -Force\n#Import-PackageProvider NuGet -MinimumVersion '2.8.5.201' -Force\n\n# Install CoApp for creating nuget packages\n#$msiPath = \"$($env:USERPROFILE)\\CoApp.Tools.Powershell.msi\"\n#(New-Object #Net.WebClient).DownloadFile('http://downloads.coapp.org/files/CoApp.Tools.Powershell.msi', $msiPath)\n#cmd /c start /wait msiexec /i \"$msiPath\" /quiet\n\n# Install CoApp module\n#Install-Module CoApp -Force"
+- ps: "$OpenSSLVersion = \"1_0_2p\"\n$OpenSSLExe = \"OpenSSL-$OpenSSLVersion.exe\"\n\nRemove-Item C:\\OpenSSL-Win32 -recurse\nRemove-Item C:\\OpenSSL-Win64 -recurse\n\nWrite-Host \"Installing OpenSSL v1.0 32-bit ...\" -ForegroundColor Cyan\nWrite-Host \"Downloading...\"\n$exePath = \"$($env:USERPROFILE)\\Win32OpenSSL-1_0_2p.exe\"\n(New-Object Net.WebClient).DownloadFile('https://slproweb.com/download/Win32OpenSSL-1_0_2p.exe', $exePath)\nWrite-Host \"Installing...\"\ncmd /c start /wait $exePath /silent /verysilent /sp- /suppressmsgboxes /DIR=C:\\OpenSSL-Win32\nWrite-Host \"Installed\" -ForegroundColor Green\n\nWrite-Host \"Installing OpenSSL v1.0 64-bit ...\" -ForegroundColor Cyan\nWrite-Host \"Downloading...\"\n$exePath = \"$($env:USERPROFILE)\\Win64OpenSSL-1_0_2p.exe\"\n(New-Object Net.WebClient).DownloadFile('https://slproweb.com/download/Win64OpenSSL-1_0_2p.exe', $exePath)\nWrite-Host \"Installing...\"\ncmd /c start /wait $exePath /silent /verysilent /sp- /suppressmsgboxes /DIR=C:\\OpenSSL-Win64\nWrite-Host \"Installed\" -ForegroundColor Green\n\nif (!(Test-Path(\"C:\\OpenSSL-Win32\"))) {\n echo \"Downloading https://slproweb.com/download/Win32$OpenSSLExe\"\n Start-FileDownload 'https://slproweb.com/download/Win32$OpenSSLExe'\n Start-Process \"Win32$OpenSSLExe\" -ArgumentList \"/silent /verysilent /sp- /suppressmsgboxes\" -Wait\n} else {\n echo \"OpenSSL-Win32 already exists: not downloading\"\n}\n\nif (!(Test-Path(\"C:\\OpenSSL-Win64\"))) {\n echo \"Downloading https://slproweb.com/download/Win64$OpenSSLExe\"\n Start-FileDownload 'https://slproweb.com/download/Win64$OpenSSLExe' \n Start-Process \"Win64$OpenSSLExe\" -ArgumentList \"/silent /verysilent /sp- /suppressmsgboxes\" -Wait\n} else {\n echo \"OpenSSL-Win64 already exists: not downloading\"\n}\n\n\n\n# Download the CoApp tools.\n$msiPath = \"$($env:USERPROFILE)\\CoApp.Tools.Powershell.msi\"\n(New-Object Net.WebClient).DownloadFile('http://coapp.org/files/CoApp.Tools.Powershell.msi', $msiPath)\n\n# Install the CoApp tools from the downloaded .msi.\nStart-Process -FilePath msiexec -ArgumentList /i, $msiPath, /quiet -Wait\n\n# Make the tools available for later PS scripts to use.\n$env:PSModulePath = $env:PSModulePath + ';C:\\Program Files (x86)\\Outercurve Foundation\\Modules'\nImport-Module CoApp\n\n# Install NuGet\n#Install-PackageProvider NuGet -MinimumVersion '2.8.5.201' -Force\n#Import-PackageProvider NuGet -MinimumVersion '2.8.5.201' -Force\n\n# Install CoApp for creating nuget packages\n#$msiPath = \"$($env:USERPROFILE)\\CoApp.Tools.Powershell.msi\"\n#(New-Object #Net.WebClient).DownloadFile('http://downloads.coapp.org/files/CoApp.Tools.Powershell.msi', $msiPath)\n#cmd /c start /wait msiexec /i \"$msiPath\" /quiet\n\n# Install CoApp module\n#Install-Module CoApp -Force"
cache:
- c:\OpenSSL-Win32
- c:\OpenSSL-Win64
@@ -26,7 +26,7 @@
parallel: true
verbosity: normal
test_script:
-- cmd: if exist DISABLED\win32\outdir\v140 ( win32\outdir\v140\%PLATFORM%\%CONFIGURATION%\tests.exe -l -p1 ) else ( win32\outdir\v120\%PLATFORM%\%CONFIGURATION%\tests.exe -l -p1 )
+- cmd: if exist DISABLED\win32\outdir\v140 ( win32\outdir\v140\%PLATFORM%\%CONFIGURATION%\tests.exe -l -p1 ) else ( cd tests && ..\win32\outdir\v120\%PLATFORM%\%CONFIGURATION%\tests.exe -l -p1 && cd ..)
artifacts:
- path: test_report*.json
name: Test report
@@ -52,7 +52,13 @@
$autopkgFile = "win32/librdkafka.autopkg"
+ pwd
+
+ ls $autopkgFile
+
+
+
# Get the ".autopkg.template" file, replace "@version" with the Appveyor version number, then save to the ".autopkg" file.
cat ($autopkgFile + ".template") | % { $_ -replace "@version", $env:appveyor_build_version } > $autopkgFile
@@ -69,9 +75,9 @@
deploy:
- provider: S3
access_key_id:
- secure: BDJ8FdNEzVKO7MTZX50dWIvrBEPOl9oHSRQ9S/s3uu0=
+ secure: t+Xo4x1mYVbqzvUDlnuMgFGp8LjQJNOfsDUAMxBsVH4=
secret_access_key:
- secure: GBmNSeDpUa7hqqTJBT+PLMgsZCxQMBau+6Vnitzqw15UlpuaZRzid1s5egIacZO6
+ secure: SNziQPPJs4poCHM7dk6OxufUYcGQhMWiNPx6Y1y6DYuWGjPc3K0APGeousLHsbLv
region: us-west-1
bucket: librdkafka-ci-packages
folder: librdkafka/p-librdkafka__bld-appveyor__plat-windows__arch-$(platform)__bldtype-$(configuration)__tag-$(APPVEYOR_REPO_TAG_NAME)__sha-$(APPVEYOR_REPO_COMMIT)__bid-$(APPVEYOR_BUILD_ID)
diff -Nru librdkafka-0.11.3/CMakeLists.txt librdkafka-0.11.6/CMakeLists.txt
--- librdkafka-0.11.3/CMakeLists.txt 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/CMakeLists.txt 2018-10-10 06:54:38.000000000 +0000
@@ -94,7 +94,9 @@
option(RDKAFKA_BUILD_EXAMPLES "Build examples" ON)
option(RDKAFKA_BUILD_TESTS "Build tests" ON)
-
+if(WIN32)
+ option(WITHOUT_WIN32_CONFIG "Avoid including win32_config.h on cmake builds" ON)
+endif(WIN32)
# In:
# * TRYCOMPILE_SRC_DIR
diff -Nru librdkafka-0.11.3/CONFIGURATION.md librdkafka-0.11.6/CONFIGURATION.md
--- librdkafka-0.11.3/CONFIGURATION.md 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/CONFIGURATION.md 2018-10-10 06:54:38.000000000 +0000
@@ -9,24 +9,24 @@
bootstrap.servers | * | | | Alias for `metadata.broker.list`
message.max.bytes | * | 1000 .. 1000000000 | 1000000 | Maximum Kafka protocol request message size.
*Type: integer*
message.copy.max.bytes | * | 0 .. 1000000000 | 65535 | Maximum size for message to be copied to buffer. Messages larger than this will be passed by reference (zero-copy) at the expense of larger iovecs.
*Type: integer*
-receive.message.max.bytes | * | 1000 .. 1000000000 | 100000000 | Maximum Kafka protocol response message size. This is a safety precaution to avoid memory exhaustion in case of protocol hickups. The value should be at least fetch.message.max.bytes * number of partitions consumed from + messaging overhead (e.g. 200000 bytes).
*Type: integer*
+receive.message.max.bytes | * | 1000 .. 2147483647 | 100000000 | Maximum Kafka protocol response message size. This serves as a safety precaution to avoid memory exhaustion in case of protocol hickups. This value is automatically adjusted upwards to be at least `fetch.max.bytes` + 512 to allow for protocol overhead.
*Type: integer*
max.in.flight.requests.per.connection | * | 1 .. 1000000 | 1000000 | Maximum number of in-flight requests per broker connection. This is a generic property applied to all broker communication, however it is primarily relevant to produce requests. In particular, note that other mechanisms limit the number of outstanding consumer fetch request per broker to one.
*Type: integer*
max.in.flight | * | | | Alias for `max.in.flight.requests.per.connection`
metadata.request.timeout.ms | * | 10 .. 900000 | 60000 | Non-topic request timeout in milliseconds. This is for metadata requests, etc.
*Type: integer*
topic.metadata.refresh.interval.ms | * | -1 .. 3600000 | 300000 | Topic metadata refresh interval in milliseconds. The metadata is automatically refreshed on error and connect. Use -1 to disable the intervalled refresh.
*Type: integer*
-metadata.max.age.ms | * | 1 .. 86400000 | -1 | Metadata cache max age. Defaults to metadata.refresh.interval.ms * 3
*Type: integer*
+metadata.max.age.ms | * | 1 .. 86400000 | -1 | Metadata cache max age. Defaults to topic.metadata.refresh.interval.ms * 3
*Type: integer*
topic.metadata.refresh.fast.interval.ms | * | 1 .. 60000 | 250 | When a topic loses its leader a new metadata request will be enqueued with this initial interval, exponentially increasing until the topic metadata has been refreshed. This is used to recover quickly from transitioning leader brokers.
*Type: integer*
topic.metadata.refresh.fast.cnt | * | 0 .. 1000 | 10 | *Deprecated: No longer used.*
*Type: integer*
topic.metadata.refresh.sparse | * | true, false | true | Sparse metadata requests (consumes less network bandwidth)
*Type: boolean*
topic.blacklist | * | | | Topic blacklist, a comma-separated list of regular expressions for matching topic names that should be ignored in broker metadata information as if the topics did not exist.
*Type: pattern list*
-debug | * | generic, broker, topic, metadata, queue, msg, protocol, cgrp, security, fetch, feature, interceptor, plugin, all | | A comma-separated list of debug contexts to enable. Debugging the Producer: broker,topic,msg. Consumer: cgrp,topic,fetch
*Type: CSV flags*
-socket.timeout.ms | * | 10 .. 300000 | 60000 | Timeout for network requests.
*Type: integer*
+debug | * | generic, broker, topic, metadata, feature, queue, msg, protocol, cgrp, security, fetch, interceptor, plugin, consumer, admin, all | | A comma-separated list of debug contexts to enable. Detailed Producer debugging: broker,topic,msg. Consumer: consumer,cgrp,topic,fetch
*Type: CSV flags*
+socket.timeout.ms | * | 10 .. 300000 | 60000 | Default timeout for network requests. Producer: ProduceRequests will use the lesser value of `socket.timeout.ms` and remaining `message.timeout.ms` for the first message in the batch. Consumer: FetchRequests will use `fetch.wait.max.ms` + `socket.timeout.ms`. Admin: Admin requests will use `socket.timeout.ms` or explicitly set `rd_kafka_AdminOptions_set_operation_timeout()` value.
*Type: integer*
socket.blocking.max.ms | * | 1 .. 60000 | 1000 | Maximum time a broker socket operation may block. A lower value improves responsiveness at the expense of slightly higher CPU usage. **Deprecated**
*Type: integer*
socket.send.buffer.bytes | * | 0 .. 100000000 | 0 | Broker socket send buffer size. System default is used if 0.
*Type: integer*
socket.receive.buffer.bytes | * | 0 .. 100000000 | 0 | Broker socket receive buffer size. System default is used if 0.
*Type: integer*
socket.keepalive.enable | * | true, false | false | Enable TCP keep-alives (SO_KEEPALIVE) on broker sockets
*Type: boolean*
-socket.nagle.disable | * | true, false | false | Disable the Nagle algorithm (TCP_NODELAY).
*Type: boolean*
-socket.max.fails | * | 0 .. 1000000 | 3 | Disconnect from broker when this number of send failures (e.g., timed out requests) is reached. Disable with 0. NOTE: The connection is automatically re-established.
*Type: integer*
+socket.nagle.disable | * | true, false | false | Disable the Nagle algorithm (TCP_NODELAY) on broker sockets.
*Type: boolean*
+socket.max.fails | * | 0 .. 1000000 | 1 | Disconnect from broker when this number of send failures (e.g., timed out requests) is reached. Disable with 0. WARNING: It is highly recommended to leave this setting at its default value of 1 to avoid the client and broker to become desynchronized in case of request timeouts. NOTE: The connection is automatically re-established.
*Type: integer*
broker.address.ttl | * | 0 .. 86400000 | 1000 | How long to cache the broker address resolving results (milliseconds).
*Type: integer*
broker.address.family | * | any, v4, v6 | any | Allowed broker IP address families: any, v4, v6
*Type: enum value*
reconnect.backoff.jitter.ms | * | 0 .. 3600000 | 500 | Throttle broker reconnection attempts by this value +-50%.
*Type: integer*
@@ -40,6 +40,7 @@
log.queue | * | true, false | false | Disable spontaneous log_cb from internal librdkafka threads, instead enqueue log messages on queue set with `rd_kafka_set_log_queue()` and serve log callbacks or events through the standard poll APIs. **NOTE**: Log messages will linger in a temporary queue until the log queue has been set.
*Type: boolean*
log.thread.name | * | true, false | true | Print internal thread name in log messages (useful for debugging librdkafka internals)
*Type: boolean*
log.connection.close | * | true, false | true | Log broker disconnects. It might be useful to turn this off when interacting with 0.9 brokers with an aggressive `connection.max.idle.ms` value.
*Type: boolean*
+background_event_cb | * | | | Background queue event callback (set with rd_kafka_conf_set_background_event_cb())
*Type: pointer*
socket_cb | * | | | Socket creation callback to provide race-free CLOEXEC
*Type: pointer*
connect_cb | * | | | Socket connect callback
*Type: pointer*
closesocket_cb | * | | | Socket close callback
*Type: pointer*
@@ -50,15 +51,20 @@
api.version.request | * | true, false | true | Request broker's supported API versions to adjust functionality to available protocol features. If set to false, or the ApiVersionRequest fails, the fallback version `broker.version.fallback` will be used. **NOTE**: Depends on broker version >=0.10.0. If the request is not supported by (an older) broker the `broker.version.fallback` fallback is used.
*Type: boolean*
api.version.request.timeout.ms | * | 1 .. 300000 | 10000 | Timeout for broker API version requests.
*Type: integer*
api.version.fallback.ms | * | 0 .. 604800000 | 1200000 | Dictates how long the `broker.version.fallback` fallback is used in the case the ApiVersionRequest fails. **NOTE**: The ApiVersionRequest is only issued when a new connection to the broker is made (such as after an upgrade).
*Type: integer*
-broker.version.fallback | * | | 0.9.0 | Older broker versions (<0.10.0) provides no way for a client to query for supported protocol features (ApiVersionRequest, see `api.version.request`) making it impossible for the client to know what features it may use. As a workaround a user may set this property to the expected broker version and the client will automatically adjust its feature set accordingly if the ApiVersionRequest fails (or is disabled). The fallback broker version will be used for `api.version.fallback.ms`. Valid values are: 0.9.0, 0.8.2, 0.8.1, 0.8.0. Any other value, such as 0.10.2.1, enables ApiVersionRequests.
*Type: string*
+broker.version.fallback | * | | 0.9.0 | Older broker versions (before 0.10.0) provide no way for a client to query for supported protocol features (ApiVersionRequest, see `api.version.request`) making it impossible for the client to know what features it may use. As a workaround a user may set this property to the expected broker version and the client will automatically adjust its feature set accordingly if the ApiVersionRequest fails (or is disabled). The fallback broker version will be used for `api.version.fallback.ms`. Valid values are: 0.9.0, 0.8.2, 0.8.1, 0.8.0. Any other value, such as 0.10.2.1, enables ApiVersionRequests.
*Type: string*
security.protocol | * | plaintext, ssl, sasl_plaintext, sasl_ssl | plaintext | Protocol used to communicate with brokers.
*Type: enum value*
ssl.cipher.suites | * | | | A cipher suite is a named combination of authentication, encryption, MAC and key exchange algorithm used to negotiate the security settings for a network connection using TLS or SSL network protocol. See manual page for `ciphers(1)` and `SSL_CTX_set_cipher_list(3).
*Type: string*
+ssl.curves.list | * | | | The supported-curves extension in the TLS ClientHello message specifies the curves (standard/named, or 'explicit' GF(2^k) or GF(p)) the client is willing to have the server use. See manual page for `SSL_CTX_set1_curves_list(3)`. OpenSSL >= 1.0.2 required.
*Type: string*
+ssl.sigalgs.list | * | | | The client uses the TLS ClientHello signature_algorithms extension to indicate to the server which signature/hash algorithm pairs may be used in digital signatures. See manual page for `SSL_CTX_set1_sigalgs_list(3)`. OpenSSL >= 1.0.2 required.
*Type: string*
ssl.key.location | * | | | Path to client's private key (PEM) used for authentication.
*Type: string*
ssl.key.password | * | | | Private key passphrase
*Type: string*
ssl.certificate.location | * | | | Path to client's public key (PEM) used for authentication.
*Type: string*
ssl.ca.location | * | | | File or directory path to CA certificate(s) for verifying the broker's key.
*Type: string*
ssl.crl.location | * | | | Path to CRL for verifying broker's certificate validity.
*Type: string*
+ssl.keystore.location | * | | | Path to client's keystore (PKCS#12) used for authentication.
*Type: string*
+ssl.keystore.password | * | | | Client's keystore (PKCS#12) password.
*Type: string*
sasl.mechanisms | * | | GSSAPI | SASL mechanism to use for authentication. Supported: GSSAPI, PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. **NOTE**: Despite the name only one mechanism must be configured.
*Type: string*
+sasl.mechanism | * | | | Alias for `sasl.mechanisms`
sasl.kerberos.service.name | * | | kafka | Kerberos principal name that Kafka runs as, not including /hostname@REALM
*Type: string*
sasl.kerberos.principal | * | | kafkaclient | This client's Kerberos principal name. (Not supported on Windows, will use the logon user's principal).
*Type: string*
sasl.kerberos.kinit.cmd | * | | kinit -S "%{sasl.kerberos.service.name}/%{broker.name}" -k -t "%{sasl.kerberos.keytab}" %{sasl.kerberos.principal} | Full kerberos kinit command string, %{config.prop.name} is replaced by corresponding config object value, %{broker.name} returns the broker's hostname.
*Type: string*
@@ -66,7 +72,7 @@
sasl.kerberos.min.time.before.relogin | * | 1 .. 86400000 | 60000 | Minimum time in milliseconds between key refresh attempts.
*Type: integer*
sasl.username | * | | | SASL username for use with the PLAIN and SASL-SCRAM-.. mechanisms
*Type: string*
sasl.password | * | | | SASL password for use with the PLAIN and SASL-SCRAM-.. mechanism
*Type: string*
-plugin.library.paths | * | | | List of plugin libaries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically.
*Type: string*
+plugin.library.paths | * | | | List of plugin libraries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically.
*Type: string*
interceptors | * | | | Interceptors added through rd_kafka_conf_interceptor_add_..() and any configuration handled by interceptors.
*Type: *
group.id | * | | | Client group id string. All clients sharing the same group.id belong to the same group.
*Type: string*
partition.assignment.strategy | * | | range,roundrobin | Name of partition assignment strategy to use when elected group leader assigns partitions to group members.
*Type: string*
@@ -74,7 +80,7 @@
heartbeat.interval.ms | * | 1 .. 3600000 | 1000 | Group session keepalive heartbeat interval.
*Type: integer*
group.protocol.type | * | | consumer | Group protocol type
*Type: string*
coordinator.query.interval.ms | * | 1 .. 3600000 | 600000 | How often to query for the current client group coordinator. If the currently assigned coordinator is down the configured query interval will be divided by ten to more quickly recover in case of coordinator reassignment.
*Type: integer*
-enable.auto.commit | C | true, false | true | Automatically and periodically commit offsets in the background.
*Type: boolean*
+enable.auto.commit | C | true, false | true | Automatically and periodically commit offsets in the background. Note: setting this to false does not prevent the consumer from fetching previously committed start offsets. To circumvent this behaviour set specific start offsets per partition in the call to assign().
*Type: boolean*
auto.commit.interval.ms | C | 0 .. 86400000 | 5000 | The frequency in milliseconds that the consumer offsets are committed (written) to offset storage. (0 = disable). This setting is used by the high-level consumer.
*Type: integer*
enable.auto.offset.store | C | true, false | true | Automatically store offset of last message provided to application.
*Type: boolean*
queued.min.messages | C | 1 .. 10000000 | 100000 | Minimum number of messages per topic+partition librdkafka tries to maintain in the local consumer queue.
*Type: integer*
@@ -82,6 +88,7 @@
fetch.wait.max.ms | C | 0 .. 300000 | 100 | Maximum time the broker may wait to fill the response with fetch.min.bytes.
*Type: integer*
fetch.message.max.bytes | C | 1 .. 1000000000 | 1048576 | Initial maximum number of bytes per topic+partition to request when fetching messages from the broker. If the client encounters a message larger than this value it will gradually try to increase it until the entire message can be fetched.
*Type: integer*
max.partition.fetch.bytes | C | | | Alias for `fetch.message.max.bytes`
+fetch.max.bytes | C | 0 .. 2147483135 | 52428800 | Maximum amount of data the broker shall return for a Fetch request. Messages are fetched in batches by the consumer and if the first message batch in the first non-empty partition of the Fetch request is larger than this value, then the message batch will still be returned to ensure the consumer can make progress. The maximum message batch size accepted by the broker is defined via `message.max.bytes` (broker config) or `max.message.bytes` (broker topic config). `fetch.max.bytes` is automatically adjusted upwards to be at least `message.max.bytes` (consumer config).
*Type: integer*
fetch.min.bytes | C | 1 .. 100000000 | 1 | Minimum number of bytes the broker responds with. If fetch.wait.max.ms expires the accumulated data will be sent to the client regardless of this setting.
*Type: integer*
fetch.error.backoff.ms | C | 0 .. 300000 | 500 | How long to postpone the next fetch request for a topic+partition in case of a fetch error.
*Type: integer*
offset.store.method | C | none, file, broker | broker | Offset commit store method: 'file' - local file store (offset.store.path, et.al), 'broker' - broker commit store (requires Apache Kafka 0.8.2 or later on the broker).
*Type: enum value*
@@ -96,8 +103,10 @@
linger.ms | P | | | Alias for `queue.buffering.max.ms`
message.send.max.retries | P | 0 .. 10000000 | 2 | How many times to retry sending a failing MessageSet. **Note:** retrying may cause reordering.
*Type: integer*
retries | P | | | Alias for `message.send.max.retries`
-retry.backoff.ms | P | 1 .. 300000 | 100 | The backoff time in milliseconds before retrying a message send.
*Type: integer*
-compression.codec | P | none, gzip, snappy, lz4 | none | compression codec to use for compressing message sets. This is the default value for all topics, may be overriden by the topic configuration property `compression.codec`.
*Type: enum value*
+retry.backoff.ms | P | 1 .. 300000 | 100 | The backoff time in milliseconds before retrying a protocol request.
*Type: integer*
+queue.buffering.backpressure.threshold | P | 1 .. 1000000 | 1 | The threshold of outstanding not yet transmitted broker requests needed to backpressure the producer's message accumulator. If the number of not yet transmitted requests equals or exceeds this number, produce request creation that would have otherwise been triggered (for example, in accordance with linger.ms) will be delayed. A lower number yields larger and more effective batches. A higher value can improve latency when using compression on slow machines.
*Type: integer*
+compression.codec | P | none, gzip, snappy, lz4 | none | compression codec to use for compressing message sets. This is the default value for all topics, may be overridden by the topic configuration property `compression.codec`.
*Type: enum value*
+compression.type | P | | | Alias for `compression.codec`
batch.num.messages | P | 1 .. 1000000 | 10000 | Maximum number of messages batched in one MessageSet. The total MessageSet size is also limited by message.max.bytes.
*Type: integer*
delivery.report.only.error | P | true, false | false | Only provide delivery reports for failed messages.
*Type: boolean*
dr_cb | P | | | Delivery report callback (set with rd_kafka_conf_set_dr_cb())
*Type: pointer*
@@ -108,17 +117,22 @@
Property | C/P | Range | Default | Description
-----------------------------------------|-----|-----------------|--------------:|--------------------------
-request.required.acks | P | -1 .. 1000 | 1 | This field indicates how many acknowledgements the leader broker must receive from ISR brokers before responding to the request: *0*=Broker does not send any response/ack to client, *1*=Only the leader broker will need to ack the message, *-1* or *all*=broker will block until message is committed by all in sync replicas (ISRs) or broker's `in.sync.replicas` setting before sending response.
*Type: integer*
+request.required.acks | P | -1 .. 1000 | 1 | This field indicates how many acknowledgements the leader broker must receive from ISR brokers before responding to the request: *0*=Broker does not send any response/ack to client, *1*=Only the leader broker will need to ack the message, *-1* or *all*=broker will block until message is committed by all in sync replicas (ISRs) or broker's `min.insync.replicas` setting before sending response.
*Type: integer*
acks | P | | | Alias for `request.required.acks`
request.timeout.ms | P | 1 .. 900000 | 5000 | The ack timeout of the producer request in milliseconds. This value is only enforced by the broker and relies on `request.required.acks` being != 0.
*Type: integer*
-message.timeout.ms | P | 0 .. 900000 | 300000 | Local message timeout. This value is only enforced locally and limits the time a produced message waits for successful delivery. A time of 0 is infinite.
*Type: integer*
+message.timeout.ms | P | 0 .. 900000 | 300000 | Local message timeout. This value is only enforced locally and limits the time a produced message waits for successful delivery. A time of 0 is infinite. This is the maximum time librdkafka may use to deliver a message (including retries). Delivery error occurs when either the retry count or the message timeout are exceeded.
*Type: integer*
+queuing.strategy | P | fifo, lifo | fifo | Producer queuing strategy. FIFO preserves produce ordering, while LIFO prioritizes new messages. WARNING: `lifo` is experimental and subject to change or removal.
*Type: enum value*
produce.offset.report | P | true, false | false | Report offset of produced message back to application. The application must be use the `dr_msg_cb` to retrieve the offset from `rd_kafka_message_t.offset`.
*Type: boolean*
-partitioner_cb | P | | | Partitioner callback (set with rd_kafka_topic_conf_set_partitioner_cb())
*Type: pointer*
+partitioner | P | | consistent_random | Partitioner: `random` - random distribution, `consistent` - CRC32 hash of key (Empty and NULL keys are mapped to single partition), `consistent_random` - CRC32 hash of key (Empty and NULL keys are randomly partitioned), `murmur2` - Java Producer compatible Murmur2 hash of key (NULL keys are mapped to single partition), `murmur2_random` - Java Producer compatible Murmur2 hash of key (NULL keys are randomly partitioned. This is functionally equivalent to the default partitioner in the Java Producer.).
*Type: string*
+partitioner_cb | P | | | Custom partitioner callback (set with rd_kafka_topic_conf_set_partitioner_cb())
*Type: pointer*
+msg_order_cmp | P | | | Message queue ordering comparator (set with rd_kafka_topic_conf_set_msg_order_cmp()). Also see `queuing.strategy`.
*Type: pointer*
opaque | * | | | Application opaque (set with rd_kafka_topic_conf_set_opaque())
*Type: pointer*
-compression.codec | P | none, gzip, snappy, lz4, inherit | inherit | Compression codec to use for compressing message sets.
*Type: enum value*
-auto.commit.enable | C | true, false | true | If true, periodically commit offset of the last message handed to the application. This committed offset will be used when the process restarts to pick up where it left off. If false, the application will have to call `rd_kafka_offset_store()` to store an offset (optional). **NOTE:** This property should only be used with the simple legacy consumer, when using the high-level KafkaConsumer the global `enable.auto.commit` property must be used instead. **NOTE:** There is currently no zookeeper integration, offsets will be written to broker or local file according to offset.store.method.
*Type: boolean*
+compression.codec | P | none, gzip, snappy, lz4, inherit | inherit | Compression codec to use for compressing message sets. inherit = inherit global compression.codec configuration.
*Type: enum value*
+compression.type | P | | | Alias for `compression.codec`
+compression.level | P | -1 .. 12 | -1 | Compression level parameter for algorithm selected by configuration property `compression.codec`. Higher values will result in better compression at the cost of more CPU usage. Usable range is algorithm-dependent: [0-9] for gzip; [0-12] for lz4; only 0 for snappy; -1 = codec-dependent default compression level.
*Type: integer*
+auto.commit.enable | C | true, false | true | [**LEGACY PROPERTY:** This property is used by the simple legacy consumer only. When using the high-level KafkaConsumer, the global `enable.auto.commit` property must be used instead]. If true, periodically commit offset of the last message handed to the application. This committed offset will be used when the process restarts to pick up where it left off. If false, the application will have to call `rd_kafka_offset_store()` to store an offset (optional). **NOTE:** There is currently no zookeeper integration, offsets will be written to broker or local file according to offset.store.method.
*Type: boolean*
enable.auto.commit | C | | | Alias for `auto.commit.enable`
-auto.commit.interval.ms | C | 10 .. 86400000 | 60000 | The frequency in milliseconds that the consumer offsets are committed (written) to offset storage. This setting is used by the low-level legacy consumer.
*Type: integer*
+auto.commit.interval.ms | C | 10 .. 86400000 | 60000 | [**LEGACY PROPERTY:** This setting is used by the simple legacy consumer only. When using the high-level KafkaConsumer, the global `auto.commit.interval.ms` property must be used instead]. The frequency in milliseconds that the consumer offsets are committed (written) to offset storage.
*Type: integer*
auto.offset.reset | C | smallest, earliest, beginning, largest, latest, end, error | largest | Action to take when there is no initial offset in offset store or the desired offset is out of range: 'smallest','earliest' - automatically reset the offset to the smallest offset, 'largest','latest' - automatically reset the offset to the largest offset, 'error' - trigger an error which is retrieved by consuming messages and checking 'message->err'.
*Type: enum value*
offset.store.path | C | | . | Path to local file for storing offsets. If the path is a directory a filename will be automatically generated in that directory based on the topic and partition.
*Type: string*
offset.store.sync.interval.ms | C | -1 .. 86400000 | -1 | fsync() interval for the offset file, in milliseconds. Use -1 to disable syncing, and 0 for immediate sync after each write.
*Type: integer*
diff -Nru librdkafka-0.11.3/configure.librdkafka librdkafka-0.11.6/configure.librdkafka
--- librdkafka-0.11.3/configure.librdkafka 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/configure.librdkafka 2018-10-10 06:54:38.000000000 +0000
@@ -38,20 +38,71 @@
function checks {
- # required libs
+ # -lrt is needed on linux for clock_gettime: link it if it exists.
+ mkl_lib_check "librt" "" cont CC "-lrt"
+
+
+ # Use internal tinycthread if C11 threads not avaialable
+ mkl_lib_check "c11threads" WITH_C11THREADS disable CC "" \
+ "
+#include
+
+
+static int start_func (void *arg) {
+ int iarg = *(int *)arg;
+ return iarg;
+}
+
+void foo (void) {
+ thrd_t thr;
+ int arg = 1;
+ if (thrd_create(&thr, start_func, (void *)&arg) != thrd_success) {
+ ;
+ }
+}
+"
+
+ # pthreads required (even if C11 threads available) for rwlocks
mkl_lib_check "libpthread" "" fail CC "-lpthread" \
"#include "
+ # Check if dlopen() is available
+ mkl_lib_check "libdl" "WITH_LIBDL" disable CC "-ldl" \
+"
+#include
+#include
+void foo (void) {
+ void *h = dlopen(\"__bad_lib\", 0);
+ void *p = dlsym(h, \"sym\");
+ if (p)
+ p = NULL;
+ dlclose(h);
+}"
+
+ if [[ $WITH_LIBDL == "y" ]]; then
+ mkl_allvar_set WITH_PLUGINS WITH_PLUGINS y
+ fi
+
# optional libs
mkl_lib_check "zlib" "WITH_ZLIB" disable CC "-lz" \
"#include "
mkl_lib_check "libcrypto" "" disable CC "-lcrypto"
+ if mkl_lib_check "libm" "" disable CC "-lm" \
+ "#include "; then
+ mkl_allvar_set WITH_HDRHISTOGRAM WITH_HDRHISTOGRAM y
+ fi
+
if [[ "$ENABLE_LZ4_EXT" == "y" ]]; then
mkl_lib_check --static=-llz4 "liblz4" "WITH_LZ4_EXT" disable CC "-llz4" \
"#include "
fi
+ # rapidjson (>=1.1.0) is used in tests to verify statistics data, not used
+ # by librdkafka itself.
+ mkl_compile_check "rapidjson" "WITH_RAPIDJSON" disable CXX "" \
+ "#include "
+
# Snappy support is built-in
mkl_allvar_set WITH_SNAPPY WITH_SNAPPY y
@@ -125,9 +176,6 @@
}"
- # -lrt is needed on linux for clock_gettime: link it if it exists.
- mkl_lib_check "librt" "" cont CC "-lrt"
-
# Older g++ (<=4.1?) gives invalid warnings for the C++ code.
mkl_mkvar_append CXXFLAGS CXXFLAGS "-Wno-non-virtual-dtor"
@@ -157,22 +205,16 @@
return buf;
}"
- # Check if dlopen() is available
- mkl_lib_check "libdl" "WITH_LIBDL" disable CC "-ldl" \
-"
-#include
-#include
-void foo (void) {
- void *h = dlopen(\"__bad_lib\", 0);
- void *p = dlsym(h, \"sym\");
- if (p)
- p = NULL;
- dlclose(h);
-}"
- if [[ $WITH_LIBDL == "y" ]]; then
- mkl_allvar_set WITH_PLUGINS WITH_PLUGINS y
- fi
+ # See if GNU's pthread_setname_np() is available, and in what form.
+ mkl_compile_check "pthread_setname_gnu" "HAVE_PTHREAD_SETNAME_GNU" disable CC "-D_GNU_SOURCE -lpthread" \
+'
+#include
+
+void foo (void) {
+ pthread_setname_np(pthread_self(), "abc");
+}
+'
# Figure out what tool to use for dumping public symbols.
# We rely on configure.cc setting up $NM if it exists.
diff -Nru librdkafka-0.11.3/debian/changelog librdkafka-0.11.6/debian/changelog
--- librdkafka-0.11.3/debian/changelog 2018-02-05 23:22:34.000000000 +0000
+++ librdkafka-0.11.6/debian/changelog 2019-01-31 14:24:28.000000000 +0000
@@ -1,8 +1,44 @@
-librdkafka (0.11.3-1build1) bionic; urgency=high
+librdkafka (0.11.6-1.1~cloud0) bionic-stein; urgency=medium
- * No change rebuild against openssl1.1.
+ * New update for the Ubuntu Cloud Archive.
- -- Dimitri John Ledkov Mon, 05 Feb 2018 23:22:34 +0000
+ -- Openstack Ubuntu Testing Bot Thu, 31 Jan 2019 14:24:28 +0000
+
+librdkafka (0.11.6-1.1) unstable; urgency=medium
+
+ * Non-maintainer upload.
+ * Apply patch to fix hurd-i386 build (Closes: #900716).
+
+ -- Samuel Thibault Sun, 04 Nov 2018 18:41:48 +0100
+
+librdkafka (0.11.6-1) unstable; urgency=medium
+
+ * New upstream version
+ * Update librdkafka1.symbols for 0.11.6
+ * Bump Standards-Version, no changes needed
+ * Drop references to the supported protocol versions
+
+ -- Christos Trochalakis Wed, 24 Oct 2018 12:01:10 +0300
+
+librdkafka (0.11.5-1) unstable; urgency=medium
+
+ * New upstream release
+ * Update librdkafka1.symbols for 0.11.5
+ * Bump Standards-Version to 4.1.5, no changes needed
+
+ -- Christos Trochalakis Thu, 19 Jul 2018 17:06:55 +0300
+
+librdkafka (0.11.4-1) unstable; urgency=medium
+
+ * Team upload
+ * New upstream release
+ * Bump Standards-Version, no changes needed
+ * Update librdkafka1.symbols for 0.11.4
+ * d/copyright:
+ + Switch the copyright-format URL to https
+ + Move license paragraph to the end
+
+ -- Mpampis Kostas Tue, 10 Apr 2018 13:19:03 +0300
librdkafka (0.11.3-1) unstable; urgency=medium
diff -Nru librdkafka-0.11.3/debian/control librdkafka-0.11.6/debian/control
--- librdkafka-0.11.3/debian/control 2018-02-05 23:22:34.000000000 +0000
+++ librdkafka-0.11.6/debian/control 2018-10-24 09:01:10.000000000 +0000
@@ -1,7 +1,6 @@
Source: librdkafka
Priority: optional
-Maintainer: Ubuntu Developers
-XSBC-Original-Maintainer: Faidon Liambotis
+Maintainer: Faidon Liambotis
Uploaders: Christos Trochalakis
Build-Depends: debhelper (>= 10),
liblz4-dev (>= 0.0~r123),
@@ -9,11 +8,11 @@
libssl-dev,
python,
zlib1g-dev
-Standards-Version: 4.1.2
+Standards-Version: 4.2.1
Section: libs
Homepage: https://github.com/edenhill/librdkafka
-Vcs-Git: https://anonscm.debian.org/cgit/pkg-kafka/librdkafka.git -b debian/sid
-Vcs-Browser: https://anonscm.debian.org/cgit/pkg-kafka/librdkafka.git
+Vcs-Git: https://salsa.debian.org/kafka-team/librdkafka.git
+Vcs-Browser: https://salsa.debian.org/kafka-team/librdkafka
Package: librdkafka1
Architecture: any
@@ -24,7 +23,7 @@
containing both Producer and Consumer support. It was designed with message
delivery reliability and high performance in mind, current figures exceed
800000 msgs/second for the producer and 3 million msgs/second for the
- consumer. It implements both the 0.8 and 0.9 versions of the protocol.
+ consumer.
.
More information about Apache Kafka can be found at http://kafka.apache.org/
.
@@ -39,7 +38,7 @@
containing both Producer and Consumer support. It was designed with message
delivery reliability and high performance in mind, current figures exceed
800000 msgs/second for the producer and 3 million msgs/second for the
- consumer. It implements both the 0.8 and 0.9 versions of the protocol.
+ consumer.
.
More information about Apache Kafka can be found at http://kafka.apache.org/
.
@@ -57,7 +56,7 @@
containing both Producer and Consumer support. It was designed with message
delivery reliability and high performance in mind, current figures exceed
800000 msgs/second for the producer and 3 million msgs/second for the
- consumer. It implements both the 0.8 and 0.9 versions of the protocol.
+ consumer.
.
More information about Apache Kafka can be found at http://kafka.apache.org/
.
diff -Nru librdkafka-0.11.3/debian/copyright librdkafka-0.11.6/debian/copyright
--- librdkafka-0.11.3/debian/copyright 2017-12-18 10:29:39.000000000 +0000
+++ librdkafka-0.11.6/debian/copyright 2018-10-24 09:01:10.000000000 +0000
@@ -1,29 +1,7 @@
-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: librdkafka
Source: https://github.com/edenhill/librdkafka
-License: BSD-2-clause
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- .
- 1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- .
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE.
-
Files: *
Copyright: 2012-2015, Magnus Edenhill
License: BSD-2-clause
@@ -97,3 +75,25 @@
Files: debian/*
Copyright: 2013 Faidon Liambotis
License: BSD-2-clause
+
+License: BSD-2-clause
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ .
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
diff -Nru librdkafka-0.11.3/debian/gbp.conf librdkafka-0.11.6/debian/gbp.conf
--- librdkafka-0.11.3/debian/gbp.conf 2017-12-18 10:29:39.000000000 +0000
+++ librdkafka-0.11.6/debian/gbp.conf 2018-10-24 09:01:10.000000000 +0000
@@ -1,6 +1,6 @@
[DEFAULT]
upstream-tree=tag
-upstream-branch=0.11.x
+upstream-branch=master
debian-branch=debian/sid
upstream-tag=v%(version%~%-)s
debian-tag=debian/%(version)s
diff -Nru librdkafka-0.11.3/debian/librdkafka1.symbols librdkafka-0.11.6/debian/librdkafka1.symbols
--- librdkafka-0.11.3/debian/librdkafka1.symbols 2017-12-18 10:29:39.000000000 +0000
+++ librdkafka-0.11.6/debian/librdkafka1.symbols 2018-10-24 09:01:10.000000000 +0000
@@ -1,5 +1,53 @@
librdkafka.so.1 librdkafka1 #MINVER#
* Build-Depends-Package: librdkafka-dev
+ rd_kafka_AdminOptions_destroy@Base 0.11.5
+ rd_kafka_AdminOptions_new@Base 0.11.5
+ rd_kafka_AdminOptions_set_broker@Base 0.11.5
+ rd_kafka_AdminOptions_set_opaque@Base 0.11.5
+ rd_kafka_AdminOptions_set_operation_timeout@Base 0.11.5
+ rd_kafka_AdminOptions_set_request_timeout@Base 0.11.5
+ rd_kafka_AdminOptions_set_validate_only@Base 0.11.5
+ rd_kafka_AlterConfigs@Base 0.11.5
+ rd_kafka_AlterConfigs_result_resources@Base 0.11.5
+ rd_kafka_ConfigEntry_is_default@Base 0.11.5
+ rd_kafka_ConfigEntry_is_read_only@Base 0.11.5
+ rd_kafka_ConfigEntry_is_sensitive@Base 0.11.5
+ rd_kafka_ConfigEntry_is_synonym@Base 0.11.5
+ rd_kafka_ConfigEntry_name@Base 0.11.5
+ rd_kafka_ConfigEntry_source@Base 0.11.5
+ rd_kafka_ConfigEntry_synonyms@Base 0.11.5
+ rd_kafka_ConfigEntry_value@Base 0.11.5
+ rd_kafka_ConfigResource_configs@Base 0.11.5
+ rd_kafka_ConfigResource_destroy@Base 0.11.5
+ rd_kafka_ConfigResource_destroy_array@Base 0.11.5
+ rd_kafka_ConfigResource_error@Base 0.11.5
+ rd_kafka_ConfigResource_error_string@Base 0.11.5
+ rd_kafka_ConfigResource_name@Base 0.11.5
+ rd_kafka_ConfigResource_new@Base 0.11.5
+ rd_kafka_ConfigResource_set_config@Base 0.11.5
+ rd_kafka_ConfigResource_type@Base 0.11.5
+ rd_kafka_ConfigSource_name@Base 0.11.5
+ rd_kafka_CreatePartitions@Base 0.11.5
+ rd_kafka_CreatePartitions_result_topics@Base 0.11.5
+ rd_kafka_CreateTopics@Base 0.11.5
+ rd_kafka_CreateTopics_result_topics@Base 0.11.5
+ rd_kafka_DeleteTopic_destroy@Base 0.11.5
+ rd_kafka_DeleteTopic_destroy_array@Base 0.11.5
+ rd_kafka_DeleteTopic_new@Base 0.11.5
+ rd_kafka_DeleteTopics@Base 0.11.5
+ rd_kafka_DeleteTopics_result_topics@Base 0.11.5
+ rd_kafka_DescribeConfigs@Base 0.11.5
+ rd_kafka_DescribeConfigs_result_resources@Base 0.11.5
+ rd_kafka_NewPartitions_destroy@Base 0.11.5
+ rd_kafka_NewPartitions_destroy_array@Base 0.11.5
+ rd_kafka_NewPartitions_new@Base 0.11.5
+ rd_kafka_NewPartitions_set_replica_assignment@Base 0.11.5
+ rd_kafka_NewTopic_destroy@Base 0.11.5
+ rd_kafka_NewTopic_destroy_array@Base 0.11.5
+ rd_kafka_NewTopic_new@Base 0.11.5
+ rd_kafka_NewTopic_set_config@Base 0.11.5
+ rd_kafka_NewTopic_set_replica_assignment@Base 0.11.5
+ rd_kafka_ResourceType_name@Base 0.11.5
rd_kafka_assign@Base 0.9.0
rd_kafka_assignment@Base 0.9.0
rd_kafka_brokers_add@Base 0.8.0
@@ -21,6 +69,7 @@
rd_kafka_conf_new@Base 0.8.0
rd_kafka_conf_properties_show@Base 0.8.0
rd_kafka_conf_set@Base 0.8.0
+ rd_kafka_conf_set_background_event_cb@Base 0.11.5
rd_kafka_conf_set_closesocket_cb@Base 0.9.3
rd_kafka_conf_set_connect_cb@Base 0.9.3
rd_kafka_conf_set_consume_cb@Base 0.9.0
@@ -48,12 +97,20 @@
rd_kafka_consume_stop@Base 0.8.0
rd_kafka_consumer_close@Base 0.9.0
rd_kafka_consumer_poll@Base 0.9.0
+ rd_kafka_controllerid@Base 0.11.5
+ rd_kafka_default_topic_conf_dup@Base 0.11.4
rd_kafka_destroy@Base 0.8.0
+ rd_kafka_destroy_flags@Base 0.11.6
rd_kafka_dump@Base 0.8.0
rd_kafka_err2name@Base 0.9.1
rd_kafka_err2str@Base 0.8.0
rd_kafka_errno2err@Base 0.8.3
rd_kafka_errno@Base 0.9.1
+ rd_kafka_event_AlterConfigs_result@Base 0.11.5
+ rd_kafka_event_CreatePartitions_result@Base 0.11.5
+ rd_kafka_event_CreateTopics_result@Base 0.11.5
+ rd_kafka_event_DeleteTopics_result@Base 0.11.5
+ rd_kafka_event_DescribeConfigs_result@Base 0.11.5
rd_kafka_event_destroy@Base 0.9.2
rd_kafka_event_error@Base 0.9.2
rd_kafka_event_error_string@Base 0.9.2
@@ -72,10 +129,20 @@
rd_kafka_get_err_descs@Base 0.9.1
rd_kafka_get_watermark_offsets@Base 0.9.1
rd_kafka_group_list_destroy@Base 0.9.1
+ rd_kafka_header_add@Base 0.11.4
+ rd_kafka_header_cnt@Base 0.11.4
+ rd_kafka_header_get@Base 0.11.4
+ rd_kafka_header_get_all@Base 0.11.4
+ rd_kafka_header_get_last@Base 0.11.4
+ rd_kafka_header_remove@Base 0.11.4
+ rd_kafka_headers_copy@Base 0.11.4
+ rd_kafka_headers_destroy@Base 0.11.4
+ rd_kafka_headers_new@Base 0.11.4
rd_kafka_interceptor_add_on_acknowledgement@Base 0.11.0
rd_kafka_interceptor_add_on_commit@Base 0.11.0
rd_kafka_interceptor_add_on_consume@Base 0.11.0
rd_kafka_interceptor_add_on_destroy@Base 0.11.0
+ rd_kafka_interceptor_add_on_request_sent@Base 0.11.4
rd_kafka_interceptor_add_on_send@Base 0.11.0
rd_kafka_last_error@Base 0.9.1
rd_kafka_list_groups@Base 0.9.1
@@ -84,12 +151,17 @@
rd_kafka_mem_free@Base 0.9.1
rd_kafka_memberid@Base 0.9.0
rd_kafka_message_destroy@Base 0.8.0
+ rd_kafka_message_detach_headers@Base 0.11.4
+ rd_kafka_message_headers@Base 0.11.4
rd_kafka_message_latency@Base 0.11.0
+ rd_kafka_message_set_headers@Base 0.11.4
rd_kafka_message_timestamp@Base 0.9.1
rd_kafka_metadata@Base 0.8.4
rd_kafka_metadata_destroy@Base 0.8.4
rd_kafka_msg_partitioner_consistent@Base 0.9.0
rd_kafka_msg_partitioner_consistent_random@Base 0.9.1
+ rd_kafka_msg_partitioner_murmur2@Base 0.11.4
+ rd_kafka_msg_partitioner_murmur2_random@Base 0.11.4
rd_kafka_msg_partitioner_random@Base 0.8.0
rd_kafka_name@Base 0.8.0
rd_kafka_new@Base 0.8.0
@@ -106,8 +178,10 @@
rd_kafka_produce_batch@Base 0.8.4
rd_kafka_producev@Base 0.9.4
rd_kafka_query_watermark_offsets@Base 0.9.1
+ rd_kafka_queue_cb_event_enable@Base 0.11.5
rd_kafka_queue_destroy@Base 0.8.4
rd_kafka_queue_forward@Base 0.9.2
+ rd_kafka_queue_get_background@Base 0.11.5
rd_kafka_queue_get_consumer@Base 0.9.2
rd_kafka_queue_get_main@Base 0.9.2
rd_kafka_queue_get_partition@Base 0.9.4
@@ -130,6 +204,7 @@
rd_kafka_topic_conf_get@Base 0.9.0
rd_kafka_topic_conf_new@Base 0.8.0
rd_kafka_topic_conf_set@Base 0.8.0
+ rd_kafka_topic_conf_set_msg_order_cmp@Base 0.11.4
rd_kafka_topic_conf_set_opaque@Base 0.8.0
rd_kafka_topic_conf_set_partitioner_cb@Base 0.8.0
rd_kafka_topic_destroy@Base 0.8.0
@@ -148,6 +223,9 @@
rd_kafka_topic_partition_list_new@Base 0.9.0
rd_kafka_topic_partition_list_set_offset@Base 0.9.1
rd_kafka_topic_partition_list_sort@Base 0.9.4
+ rd_kafka_topic_result_error@Base 0.11.5
+ rd_kafka_topic_result_error_string@Base 0.11.5
+ rd_kafka_topic_result_name@Base 0.11.5
rd_kafka_type@Base 0.11.0
rd_kafka_unittest@Base 0.11.0
rd_kafka_unsubscribe@Base 0.9.0
diff -Nru librdkafka-0.11.3/debian/patches/hurd.diff librdkafka-0.11.6/debian/patches/hurd.diff
--- librdkafka-0.11.3/debian/patches/hurd.diff 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/debian/patches/hurd.diff 2018-11-04 17:41:07.000000000 +0000
@@ -0,0 +1,28 @@
+Index: librdkafka-0.11.4/src/rd.h
+===================================================================
+--- librdkafka-0.11.4.orig/src/rd.h
++++ librdkafka-0.11.4/src/rd.h
+@@ -158,8 +158,8 @@ static RD_INLINE RD_UNUSED char *rd_strn
+ #ifdef __APPLE__
+ /* Some versions of MacOSX dont have IOV_MAX */
+ #define IOV_MAX 1024
+-#elif defined(_MSC_VER)
+-/* There is no IOV_MAX on MSVC but it is used internally in librdkafka */
++#elif defined(_MSC_VER) || defined(__GNU__)
++/* There is no IOV_MAX on MSVC or GNU but it is used internally in librdkafka */
+ #define IOV_MAX 1024
+ #else
+ #error "IOV_MAX not defined"
+Index: librdkafka-0.11.4/src/snappy_compat.h
+===================================================================
+--- librdkafka-0.11.4.orig/src/snappy_compat.h
++++ librdkafka-0.11.4/src/snappy_compat.h
+@@ -5,7 +5,7 @@
+
+ #ifdef __FreeBSD__
+ # include
+-#elif defined(__APPLE_CC_) || defined(__MACH__) /* MacOS/X support */
++#elif defined(__APPLE_CC_) || (defined(__MACH__) && defined(__APPLE__)) /* MacOS/X support */
+ # include
+
+ #if __DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN
diff -Nru librdkafka-0.11.3/debian/patches/series librdkafka-0.11.6/debian/patches/series
--- librdkafka-0.11.3/debian/patches/series 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/debian/patches/series 2018-11-04 17:41:18.000000000 +0000
@@ -0,0 +1 @@
+hurd.diff
diff -Nru librdkafka-0.11.3/dev-conf.sh librdkafka-0.11.6/dev-conf.sh
--- librdkafka-0.11.3/dev-conf.sh 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/dev-conf.sh 2018-10-10 06:54:38.000000000 +0000
@@ -4,12 +4,47 @@
set -e
./configure --clean
+
+# enable pedantic
#export CFLAGS='-std=c99 -pedantic -Wshadow'
#export CXXFLAGS='-std=c++98 -pedantic'
-FSAN="-fsanitize=address"
-export CPPFLAGS="$CPPFLAGS $FSAN"
-export LDFLAGS="$LDFLAGS $FSAN"
-./configure --enable-devel --enable-werror
-#--disable-optimization
-# --enable-sharedptr-debug #--enable-refcnt-debug
+# enable FSAN address, thread, ..
+#FSAN="-fsanitize=address"
+#FSAN="-fsanitize=thread"
+#FSAN="-fsanitize=undefined -fsanitize-undefined-trap-on-error -fno-omit-frame-pointer"
+
+if [[ ! -z $FSAN ]]; then
+ export CPPFLAGS="$CPPFLAGS $FSAN"
+ export LDFLAGS="$LDFLAGS $FSAN"
+fi
+
+OPTS=""
+
+# enable devel asserts
+OPTS="$OPTS --enable-devel"
+
+# disable optimizations
+OPTS="$OPTS --disable-optimization"
+
+# gprof
+#OPTS="$OPTS --enable-profiling --disable-optimization"
+
+# disable lz4
+#OPTS="$OPTS --disable-lz4"
+
+# disable cyrus-sasl
+#OPTS="$OPTS --disable-sasl"
+
+# enable sharedptr debugging
+#OPTS="$OPTS --enable-sharedptr-debug"
+
+#enable refcnt debugging
+#OPTS="$OPTS --enable-refcnt-debug"
+
+echo "Devel configuration options: $OPTS"
+./configure $OPTS
+
+make clean
+make -j
+(cd tests ; make -j build)
diff -Nru librdkafka-0.11.3/Doxyfile librdkafka-0.11.6/Doxyfile
--- librdkafka-0.11.3/Doxyfile 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/Doxyfile 2018-10-10 06:54:38.000000000 +0000
@@ -759,7 +759,7 @@
# spaces.
# Note: If this tag is empty the current directory is searched.
-INPUT = mainpage.doxy INTRODUCTION.md CONFIGURATION.md src/rdkafka.h src-cpp/rdkafkacpp.h
+INPUT = mainpage.doxy INTRODUCTION.md CONFIGURATION.md STATISTICS.md src/rdkafka.h src-cpp/rdkafkacpp.h
# This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
diff -Nru librdkafka-0.11.3/examples/CMakeLists.txt librdkafka-0.11.6/examples/CMakeLists.txt
--- librdkafka-0.11.3/examples/CMakeLists.txt 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/examples/CMakeLists.txt 2018-10-10 06:54:38.000000000 +0000
@@ -1,20 +1,27 @@
-add_executable(rdkafka_example rdkafka_example.c)
-target_link_libraries(rdkafka_example PUBLIC rdkafka)
+if(WIN32)
+ set(win32_sources ../win32/wingetopt.c ../win32/wingetopt.h)
+endif(WIN32)
-add_executable(rdkafka_simple_producer rdkafka_simple_producer.c)
+add_executable(rdkafka_simple_producer rdkafka_simple_producer.c ${win32_sources})
target_link_libraries(rdkafka_simple_producer PUBLIC rdkafka)
-add_executable(rdkafka_consumer_example rdkafka_consumer_example.c)
-target_link_libraries(rdkafka_consumer_example PUBLIC rdkafka)
-
-add_executable(rdkafka_performance rdkafka_performance.c)
+add_executable(rdkafka_performance rdkafka_performance.c ${win32_sources})
target_link_libraries(rdkafka_performance PUBLIC rdkafka)
-add_executable(rdkafka_example_cpp rdkafka_example.cpp)
+add_executable(rdkafka_example_cpp rdkafka_example.cpp ${win32_sources})
target_link_libraries(rdkafka_example_cpp PUBLIC rdkafka++)
-add_executable(kafkatest_verifiable_client kafkatest_verifiable_client.cpp)
-target_link_libraries(kafkatest_verifiable_client PUBLIC rdkafka++)
-
-add_executable(rdkafka_consumer_example_cpp rdkafka_consumer_example.cpp)
+add_executable(rdkafka_consumer_example_cpp rdkafka_consumer_example.cpp ${win32_sources})
target_link_libraries(rdkafka_consumer_example_cpp PUBLIC rdkafka++)
+
+# The targets below has Unix include dirs and do not compile on Windows.
+if(NOT WIN32)
+ add_executable(rdkafka_example rdkafka_example.c)
+ target_link_libraries(rdkafka_example PUBLIC rdkafka)
+
+ add_executable(rdkafka_consumer_example rdkafka_consumer_example.c)
+ target_link_libraries(rdkafka_consumer_example PUBLIC rdkafka)
+
+ add_executable(kafkatest_verifiable_client kafkatest_verifiable_client.cpp)
+ target_link_libraries(kafkatest_verifiable_client PUBLIC rdkafka++)
+endif(NOT WIN32)
diff -Nru librdkafka-0.11.3/examples/.gitignore librdkafka-0.11.6/examples/.gitignore
--- librdkafka-0.11.3/examples/.gitignore 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/examples/.gitignore 2018-10-10 06:54:38.000000000 +0000
@@ -5,3 +5,4 @@
rdkafka_consumer_example_cpp
kafkatest_verifiable_client
rdkafka_simple_producer
+rdkafka_consume_batch
diff -Nru librdkafka-0.11.3/examples/kafkatest_verifiable_client.cpp librdkafka-0.11.6/examples/kafkatest_verifiable_client.cpp
--- librdkafka-0.11.3/examples/kafkatest_verifiable_client.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/examples/kafkatest_verifiable_client.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -567,13 +567,45 @@
-
-static void read_conf_file (const std::string &conf_file) {
+/**
+ * @brief Read (Java client) configuration file
+ */
+static void read_conf_file (RdKafka::Conf *conf, const std::string &conf_file) {
std::ifstream inf(conf_file.c_str());
+ if (!inf) {
+ std::cerr << now() << ": " << conf_file << ": could not open file" << std::endl;
+ exit(1);
+ }
+
+ std::cerr << now() << ": " << conf_file << ": read config file" << std::endl;
+
std::string line;
+ int linenr = 0;
+
while (std::getline(inf, line)) {
- std::cerr << now() << ": conf_file: " << conf_file << ": " << line << std::endl;
+ linenr++;
+
+ // Ignore comments and empty lines
+ if (line[0] == '#' || line.length() == 0)
+ continue;
+
+ // Match on key=value..
+ size_t d = line.find("=");
+ if (d == 0 || d == std::string::npos) {
+ std::cerr << now() << ": " << conf_file << ":" << linenr << ": " << line << ": ignoring invalid line (expect key=value): " << ::std::endl;
+ continue;
+ }
+
+ std::string key = line.substr(0, d);
+ std::string val = line.substr(d+1);
+
+ std::string errstr;
+ if (conf->set(key, val, errstr)) {
+ std::cerr << now() << ": " << conf_file << ":" << linenr << ": " << key << "=" << val << ": " << errstr << ": ignoring error" << std::endl;
+ } else {
+ std::cerr << now() << ": " << conf_file << ":" << linenr << ": " << key << "=" << val << ": applied to configuration" << std::endl;
+ }
}
inf.close();
@@ -586,12 +618,11 @@
std::string brokers = "localhost";
std::string errstr;
std::vector topics;
- std::string conf_file;
std::string mode = "P";
int throughput = 0;
int32_t partition = RdKafka::Topic::PARTITION_UA;
- bool do_conf_dump = false;
MyHashPartitionerCb hash_partitioner;
+ int64_t create_time = -1;
std::cerr << now() << ": librdkafka version " << RdKafka::version_str() <<
" (" << RdKafka::version() << ")" << std::endl;
@@ -600,10 +631,15 @@
* Create configuration objects
*/
RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
- RdKafka::Conf *tconf = RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC);
+
+ /* Java VerifiableProducer defaults to acks=all */
+ if (conf->set("acks", "all", errstr)) {
+ std::cerr << now() << ": " << errstr << std::endl;
+ exit(1);
+ }
/* Avoid slow shutdown on error */
- if (tconf->set("message.timeout.ms", "60000", errstr)) {
+ if (conf->set("message.timeout.ms", "60000", errstr)) {
std::cerr << now() << ": " << errstr << std::endl;
exit(1);
}
@@ -617,7 +653,7 @@
conf->set("log.thread.name", "true", errstr);
/* correct producer offsets */
- tconf->set("produce.offset.report", "true", errstr);
+ conf->set("produce.offset.report", "true", errstr);
/* auto commit is explicitly enabled with --enable-autocommit */
conf->set("enable.auto.commit", "false", errstr);
@@ -650,13 +686,13 @@
throughput = atoi(val);
else if (!strcmp(name, "--producer.config") ||
!strcmp(name, "--consumer.config"))
- read_conf_file(val);
+ read_conf_file(conf, val);
else if (!strcmp(name, "--group-id"))
conf->set("group.id", val, errstr);
else if (!strcmp(name, "--session-timeout"))
conf->set("session.timeout.ms", val, errstr);
else if (!strcmp(name, "--reset-policy")) {
- if (tconf->set("auto.offset.reset", val, errstr)) {
+ if (conf->set("auto.offset.reset", val, errstr)) {
std::cerr << now() << ": " << errstr << std::endl;
exit(1);
}
@@ -686,6 +722,13 @@
}
} else if (!strcmp(name, "--value-prefix")) {
value_prefix = std::string(val) + ".";
+ } else if (!strcmp(name, "--acks")) {
+ if (conf->set("acks", val, errstr)) {
+ std::cerr << now() << ": " << errstr << std::endl;
+ exit(1);
+ }
+ } else if (!strcmp(name, "--message-create-time")) {
+ create_time = (int64_t)atoi(val);
} else if (!strcmp(name, "--debug")) {
conf->set("debug", val, errstr);
} else if (!strcmp(name, "-X")) {
@@ -742,31 +785,6 @@
ExampleEventCb ex_event_cb;
conf->set("event_cb", &ex_event_cb, errstr);
- if (do_conf_dump) {
- int pass;
-
- for (pass = 0 ; pass < 2 ; pass++) {
- std::list *dump;
- if (pass == 0) {
- dump = conf->dump();
- std::cerr << now() << ": # Global config" << std::endl;
- } else {
- dump = tconf->dump();
- std::cerr << now() << ": # Topic config" << std::endl;
- }
-
- for (std::list::iterator it = dump->begin();
- it != dump->end(); ) {
- std::cerr << *it << " = ";
- it++;
- std::cerr << *it << std::endl;
- it++;
- }
- std::cerr << std::endl;
- }
- exit(0);
- }
-
signal(SIGINT, sigterm);
signal(SIGTERM, sigterm);
signal(SIGALRM, sigwatchdog);
@@ -797,7 +815,7 @@
* Create topic handle.
*/
RdKafka::Topic *topic = RdKafka::Topic::create(producer, topics[0],
- tconf, errstr);
+ NULL, errstr);
if (!topic) {
std::cerr << now() << ": Failed to create topic: " << errstr << std::endl;
exit(1);
@@ -815,11 +833,22 @@
std::ostringstream msg;
msg << value_prefix << i;
while (true) {
- RdKafka::ErrorCode resp =
- producer->produce(topic, partition,
- RdKafka::Producer::RK_MSG_COPY /* Copy payload */,
- const_cast(msg.str().c_str()),
- msg.str().size(), NULL, NULL);
+ RdKafka::ErrorCode resp;
+ if (create_time == -1) {
+ resp = producer->produce(topic, partition,
+ RdKafka::Producer::RK_MSG_COPY /* Copy payload */,
+ const_cast(msg.str().c_str()),
+ msg.str().size(), NULL, NULL);
+ } else {
+ resp = producer->produce(topics[0], partition,
+ RdKafka::Producer::RK_MSG_COPY /* Copy payload */,
+ const_cast(msg.str().c_str()),
+ msg.str().size(),
+ NULL, 0,
+ create_time,
+ NULL);
+ }
+
if (resp == RdKafka::ERR__QUEUE_FULL) {
producer->poll(100);
continue;
@@ -859,10 +888,7 @@
* Consumer mode
*/
- tconf->set("auto.offset.reset", "smallest", errstr);
-
- /* Set default topic config */
- conf->set("default_topic_conf", tconf, errstr);
+ conf->set("auto.offset.reset", "smallest", errstr);
ExampleRebalanceCb ex_rebalance_cb;
conf->set("rebalance_cb", &ex_rebalance_cb, errstr);
diff -Nru librdkafka-0.11.3/examples/Makefile librdkafka-0.11.6/examples/Makefile
--- librdkafka-0.11.3/examples/Makefile 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/examples/Makefile 2018-10-10 06:54:38.000000000 +0000
@@ -72,6 +72,10 @@
$(CXX) $(CPPFLAGS) $(CXXFLAGS) rdkafka_consumer_example.cpp -o $@ $(LDFLAGS) \
../src-cpp/librdkafka++.a ../src/librdkafka.a $(LIBS) -lstdc++
+rdkafka_consume_batch: ../src-cpp/librdkafka++.a ../src/librdkafka.a rdkafka_consume_batch.cpp
+ $(CXX) $(CPPFLAGS) $(CXXFLAGS) rdkafka_consume_batch.cpp -o $@ $(LDFLAGS) \
+ ../src-cpp/librdkafka++.a ../src/librdkafka.a $(LIBS) -lstdc++
+
rdkafka_zookeeper_example: ../src/librdkafka.a rdkafka_zookeeper_example.c
$(CC) $(CPPFLAGS) $(CFLAGS) -I/usr/include/zookeeper rdkafka_zookeeper_example.c -o $@ $(LDFLAGS) \
../src/librdkafka.a $(LIBS) -lzookeeper_mt -ljansson
diff -Nru librdkafka-0.11.3/examples/rdkafka_consume_batch.cpp librdkafka-0.11.6/examples/rdkafka_consume_batch.cpp
--- librdkafka-0.11.3/examples/rdkafka_consume_batch.cpp 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/examples/rdkafka_consume_batch.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,260 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * Apache Kafka consumer & producer example programs
+ * using the Kafka driver from librdkafka
+ * (https://github.com/edenhill/librdkafka)
+ *
+ * This example shows how to read batches of messages.
+ * Note that messages are fetched from the broker in batches regardless
+ * of how the application polls messages from librdkafka, this example
+ * merely shows how to accumulate a set of messages in the application.
+ */
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+#ifndef _MSC_VER
+#include
+#endif
+
+#ifdef _MSC_VER
+#include "../win32/wingetopt.h"
+#include
+#elif _AIX
+#include
+#else
+#include
+#include
+#endif
+
+/*
+ * Typically include path in a real application would be
+ * #include
+ */
+#include "rdkafkacpp.h"
+
+
+
+static bool run = true;
+
+static void sigterm (int sig) {
+ run = false;
+}
+
+
+
+/**
+ * @returns the current wall-clock time in milliseconds
+ */
+static int64_t now () {
+#ifndef _MSC_VER
+ struct timeval tv;
+ gettimeofday(&tv, NULL);
+ return ((int64_t)tv.tv_sec * 1000) + (tv.tv_usec / 1000);
+#else
+#error "now() not implemented for Windows, please submit a PR"
+#endif
+}
+
+
+
+/**
+ * @brief Accumulate a batch of \p batch_size messages, but wait
+ * no longer than \p batch_tmout milliseconds.
+ */
+static std::vector
+consume_batch (RdKafka::KafkaConsumer *consumer, size_t batch_size, int batch_tmout) {
+
+ std::vector msgs;
+ msgs.reserve(batch_size);
+
+ int64_t end = now() + batch_tmout;
+ int remaining_timeout = batch_tmout;
+
+ while (msgs.size() < batch_size) {
+ RdKafka::Message *msg = consumer->consume(remaining_timeout);
+
+ switch (msg->err()) {
+ case RdKafka::ERR__TIMED_OUT:
+ delete msg;
+ return msgs;
+
+ case RdKafka::ERR_NO_ERROR:
+ msgs.push_back(msg);
+ break;
+
+ default:
+ std::cerr << "%% Consumer error: " << msg->errstr() << std::endl;
+ run = false;
+ delete msg;
+ return msgs;
+ }
+
+ remaining_timeout = end - now();
+ if (remaining_timeout < 0)
+ break;
+ }
+
+ return msgs;
+}
+
+
+int main (int argc, char **argv) {
+ std::string errstr;
+ std::string topic_str;
+ std::vector topics;
+ int batch_size = 100;
+ int batch_tmout = 1000;
+
+ /* Create configuration objects */
+ RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
+
+ if (conf->set("enable.partition.eof", "false", errstr) != RdKafka::Conf::CONF_OK) {
+ std::cerr << errstr << std::endl;
+ exit(1);
+ }
+
+ /* Read command line arguments */
+ int opt;
+ while ((opt = getopt(argc, argv, "g:B:T::b:X:")) != -1) {
+ switch (opt) {
+ case 'g':
+ if (conf->set("group.id", optarg, errstr) != RdKafka::Conf::CONF_OK) {
+ std::cerr << errstr << std::endl;
+ exit(1);
+ }
+ break;
+
+ case 'B':
+ batch_size = atoi(optarg);
+ break;
+
+ case 'T':
+ batch_tmout = atoi(optarg);
+ break;
+
+ case 'b':
+ if (conf->set("bootstrap.servers", optarg, errstr) != RdKafka::Conf::CONF_OK) {
+ std::cerr << errstr << std::endl;
+ exit(1);
+ }
+ break;
+
+ case 'X':
+ {
+ char *name, *val;
+
+ name = optarg;
+ if (!(val = strchr(name, '='))) {
+ std::cerr << "%% Expected -X property=value, not " <<
+ name << std::endl;
+ exit(1);
+ }
+
+ *val = '\0';
+ val++;
+
+ if (conf->set(name, val, errstr) != RdKafka::Conf::CONF_OK) {
+ std::cerr << errstr << std::endl;
+ exit(1);
+ }
+ }
+ break;
+
+ default:
+ goto usage;
+ }
+ }
+
+ /* Topics to consume */
+ for (; optind < argc ; optind++)
+ topics.push_back(std::string(argv[optind]));
+
+ if (topics.empty() || optind != argc) {
+ usage:
+ fprintf(stderr,
+ "Usage: %s -g -B [options] topic1 topic2..\n"
+ "\n"
+ "librdkafka version %s (0x%08x)\n"
+ "\n"
+ " Options:\n"
+ " -g Consumer group id\n"
+ " -B How many messages to batch (default: 100).\n"
+ " -T How long to wait for batch-size to accumulate in milliseconds. (default 1000 ms)\n"
+ " -b Broker address (localhost:9092)\n"
+ " -X Set arbitrary librdkafka configuration property\n"
+ "\n",
+ argv[0],
+ RdKafka::version_str().c_str(), RdKafka::version());
+ exit(1);
+ }
+
+
+ signal(SIGINT, sigterm);
+ signal(SIGTERM, sigterm);
+
+ /* Create consumer */
+ RdKafka::KafkaConsumer *consumer = RdKafka::KafkaConsumer::create(conf, errstr);
+ if (!consumer) {
+ std::cerr << "Failed to create consumer: " << errstr << std::endl;
+ exit(1);
+ }
+
+ delete conf;
+
+ /* Subscribe to topics */
+ RdKafka::ErrorCode err = consumer->subscribe(topics);
+ if (err) {
+ std::cerr << "Failed to subscribe to " << topics.size() << " topics: "
+ << RdKafka::err2str(err) << std::endl;
+ exit(1);
+ }
+
+ /* Consume messages in batches of \p batch_size */
+ while (run) {
+ auto msgs = consume_batch(consumer, batch_size, batch_tmout);
+ std::cout << "Accumulated " << msgs.size() << " messages:" << std::endl;
+
+ for (auto &msg : msgs) {
+ std::cout << " Message in " << msg->topic_name() << " [" << msg->partition() << "] at offset " << msg->offset() << std::endl;
+ delete msg;
+ }
+ }
+
+ /* Close and destroy consumer */
+ consumer->close();
+ delete consumer;
+
+ return 0;
+}
diff -Nru librdkafka-0.11.3/examples/rdkafka_consumer_example.cpp librdkafka-0.11.6/examples/rdkafka_consumer_example.cpp
--- librdkafka-0.11.3/examples/rdkafka_consumer_example.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/examples/rdkafka_consumer_example.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -210,16 +210,6 @@
}
}
-
-class ExampleConsumeCb : public RdKafka::ConsumeCb {
- public:
- void consume_cb (RdKafka::Message &msg, void *opaque) {
- msg_consume(&msg, opaque);
- }
-};
-
-
-
int main (int argc, char **argv) {
std::string brokers = "localhost";
std::string errstr;
@@ -229,7 +219,6 @@
std::vector topics;
bool do_conf_dump = false;
int opt;
- int use_ccb = 0;
/*
* Create configuration objects
@@ -240,7 +229,7 @@
ExampleRebalanceCb ex_rebalance_cb;
conf->set("rebalance_cb", &ex_rebalance_cb, errstr);
- while ((opt = getopt(argc, argv, "g:b:z:qd:eX:AM:f:qv")) != -1) {
+ while ((opt = getopt(argc, argv, "g:b:z:qd:eX:AM:qv")) != -1) {
switch (opt) {
case 'g':
if (conf->set("group.id", optarg, errstr) != RdKafka::Conf::CONF_OK) {
@@ -306,15 +295,6 @@
}
break;
- case 'f':
- if (!strcmp(optarg, "ccb"))
- use_ccb = 1;
- else {
- std::cerr << "Unknown option: " << optarg << std::endl;
- exit(1);
- }
- break;
-
case 'q':
verbosity--;
break;
@@ -354,8 +334,6 @@
"will be set on topic object.\n"
" Use '-X list' to see the full list\n"
" of supported properties.\n"
- " -f Set option:\n"
- " ccb - use consume_callback\n"
" -q Quiet / Decrease verbosity\n"
" -v Increase verbosity\n"
"\n"
@@ -379,12 +357,6 @@
}
}
- ExampleConsumeCb ex_consume_cb;
-
- if(use_ccb) {
- conf->set("consume_cb", &ex_consume_cb, errstr);
- }
-
ExampleEventCb ex_event_cb;
conf->set("event_cb", &ex_event_cb, errstr);
@@ -453,9 +425,7 @@
*/
while (run) {
RdKafka::Message *msg = consumer->consume(1000);
- if (!use_ccb) {
- msg_consume(msg, NULL);
- }
+ msg_consume(msg, NULL);
delete msg;
}
diff -Nru librdkafka-0.11.3/examples/rdkafka_example.c librdkafka-0.11.6/examples/rdkafka_example.c
--- librdkafka-0.11.3/examples/rdkafka_example.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/examples/rdkafka_example.c 2018-10-10 06:54:38.000000000 +0000
@@ -170,6 +170,8 @@
if (!quiet) {
rd_kafka_timestamp_type_t tstype;
int64_t timestamp;
+ rd_kafka_headers_t *hdrs;
+
fprintf(stdout, "%% Message (offset %"PRId64", %zd bytes):\n",
rkmessage->offset, rkmessage->len);
@@ -187,6 +189,27 @@
!timestamp ? 0 :
(int)time(NULL) - (int)(timestamp/1000));
}
+
+ if (!rd_kafka_message_headers(rkmessage, &hdrs)) {
+ size_t idx = 0;
+ const char *name;
+ const void *val;
+ size_t size;
+
+ fprintf(stdout, "%% Headers:");
+
+ while (!rd_kafka_header_get_all(hdrs, idx++,
+ &name, &val, &size)) {
+ fprintf(stdout, "%s%s=",
+ idx == 1 ? " " : ", ", name);
+ if (val)
+ fprintf(stdout, "\"%.*s\"",
+ (int)size, (const char *)val);
+ else
+ fprintf(stdout, "NULL");
+ }
+ fprintf(stdout, "\n");
+ }
}
if (rkmessage->key_len) {
@@ -210,20 +233,25 @@
static void metadata_print (const char *topic,
const struct rd_kafka_metadata *metadata) {
int i, j, k;
+ int32_t controllerid;
printf("Metadata for %s (from broker %"PRId32": %s):\n",
topic ? : "all topics",
metadata->orig_broker_id,
metadata->orig_broker_name);
+ controllerid = rd_kafka_controllerid(rk, 0);
+
/* Iterate brokers */
printf(" %i brokers:\n", metadata->broker_cnt);
for (i = 0 ; i < metadata->broker_cnt ; i++)
- printf(" broker %"PRId32" at %s:%i\n",
+ printf(" broker %"PRId32" at %s:%i%s\n",
metadata->brokers[i].id,
metadata->brokers[i].host,
- metadata->brokers[i].port);
+ metadata->brokers[i].port,
+ controllerid == metadata->brokers[i].id ?
+ " (controller)" : "");
/* Iterate topics */
printf(" %i topics:\n", metadata->topic_cnt);
@@ -287,6 +315,8 @@
int64_t seek_offset = 0;
int64_t tmp_offset = 0;
int get_wmarks = 0;
+ rd_kafka_headers_t *hdrs = NULL;
+ rd_kafka_resp_err_t err;
/* Kafka configuration */
conf = rd_kafka_conf_new();
@@ -301,7 +331,7 @@
/* Topic configuration */
topic_conf = rd_kafka_topic_conf_new();
- while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:As:")) != -1) {
+ while ((opt = getopt(argc, argv, "PCLt:p:b:z:qd:o:eX:As:H:")) != -1) {
switch (opt) {
case 'P':
case 'C':
@@ -370,6 +400,31 @@
case 'A':
output = OUTPUT_RAW;
break;
+ case 'H':
+ {
+ char *name, *val;
+ size_t name_sz = -1;
+
+ name = optarg;
+ val = strchr(name, '=');
+ if (val) {
+ name_sz = (size_t)(val-name);
+ val++; /* past the '=' */
+ }
+
+ if (!hdrs)
+ hdrs = rd_kafka_headers_new(8);
+
+ err = rd_kafka_header_add(hdrs, name, name_sz, val, -1);
+ if (err) {
+ fprintf(stderr,
+ "%% Failed to add header %s: %s\n",
+ name, rd_kafka_err2str(err));
+ exit(1);
+ }
+ }
+ break;
+
case 'X':
{
char *name, *val;
@@ -503,6 +558,7 @@
" %s\n"
" -q Be quiet\n"
" -A Raw payload output (consumer)\n"
+ " -H Add header to message (producer)\n"
" -X Set arbitrary librdkafka "
"configuration property\n"
" Properties prefixed with \"topic.\" "
@@ -584,22 +640,49 @@
if (buf[len-1] == '\n')
buf[--len] = '\0';
+ err = RD_KAFKA_RESP_ERR_NO_ERROR;
+
/* Send/Produce message. */
- if (rd_kafka_produce(rkt, partition,
- RD_KAFKA_MSG_F_COPY,
- /* Payload and length */
- buf, len,
- /* Optional key and its length */
- NULL, 0,
- /* Message opaque, provided in
- * delivery report callback as
- * msg_opaque. */
- NULL) == -1) {
- fprintf(stderr,
- "%% Failed to produce to topic %s "
+ if (hdrs) {
+ rd_kafka_headers_t *hdrs_copy;
+
+ hdrs_copy = rd_kafka_headers_copy(hdrs);
+
+ err = rd_kafka_producev(
+ rk,
+ RD_KAFKA_V_RKT(rkt),
+ RD_KAFKA_V_PARTITION(partition),
+ RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+ RD_KAFKA_V_VALUE(buf, len),
+ RD_KAFKA_V_HEADERS(hdrs_copy),
+ RD_KAFKA_V_END);
+
+ if (err)
+ rd_kafka_headers_destroy(hdrs_copy);
+
+ } else {
+ if (rd_kafka_produce(
+ rkt, partition,
+ RD_KAFKA_MSG_F_COPY,
+ /* Payload and length */
+ buf, len,
+ /* Optional key and its length */
+ NULL, 0,
+ /* Message opaque, provided in
+ * delivery report callback as
+ * msg_opaque. */
+ NULL) == -1) {
+ err = rd_kafka_last_error();
+ }
+ }
+
+ if (err) {
+ fprintf(stderr,
+ "%% Failed to produce to topic %s "
"partition %i: %s\n",
rd_kafka_topic_name(rkt), partition,
- rd_kafka_err2str(rd_kafka_last_error()));
+ rd_kafka_err2str(err));
+
/* Poll to handle delivery reports */
rd_kafka_poll(rk, 0);
continue;
@@ -792,6 +875,9 @@
exit(err ? 2 : 0);
}
+ if (hdrs)
+ rd_kafka_headers_destroy(hdrs);
+
if (topic_conf)
rd_kafka_topic_conf_destroy(topic_conf);
diff -Nru librdkafka-0.11.3/examples/rdkafka_performance.c librdkafka-0.11.6/examples/rdkafka_performance.c
--- librdkafka-0.11.3/examples/rdkafka_performance.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/examples/rdkafka_performance.c 2018-10-10 06:54:38.000000000 +0000
@@ -75,6 +75,8 @@
static int partition_cnt = 0;
static int eof_cnt = 0;
static int with_dr = 1;
+static int read_hdrs = 0;
+
static void stop (int sig) {
if (!run)
@@ -315,6 +317,12 @@
}
+ if (read_hdrs) {
+ rd_kafka_headers_t *hdrs;
+ /* Force parsing of headers but don't do anything with them. */
+ rd_kafka_message_headers(rkmessage, &hdrs);
+ }
+
if (msgcnt != -1 && (int)cnt.msgs >= msgcnt)
run = 0;
}
@@ -715,6 +723,61 @@
}
+static rd_kafka_resp_err_t do_produce (rd_kafka_t *rk,
+ rd_kafka_topic_t *rkt, int32_t partition,
+ int msgflags,
+ void *payload, size_t size,
+ const void *key, size_t key_size,
+ const rd_kafka_headers_t *hdrs) {
+
+ /* Send/Produce message. */
+ if (hdrs) {
+ rd_kafka_headers_t *hdrs_copy;
+ rd_kafka_resp_err_t err;
+
+ hdrs_copy = rd_kafka_headers_copy(hdrs);
+
+ err = rd_kafka_producev(
+ rk,
+ RD_KAFKA_V_RKT(rkt),
+ RD_KAFKA_V_PARTITION(partition),
+ RD_KAFKA_V_MSGFLAGS(msgflags),
+ RD_KAFKA_V_VALUE(payload, size),
+ RD_KAFKA_V_KEY(key, key_size),
+ RD_KAFKA_V_HEADERS(hdrs_copy),
+ RD_KAFKA_V_END);
+
+ if (err)
+ rd_kafka_headers_destroy(hdrs_copy);
+
+ return err;
+
+ } else {
+ if (rd_kafka_produce(rkt, partition, msgflags, payload, size,
+ key, key_size, NULL) == -1)
+ return rd_kafka_last_error();
+ }
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+/**
+ * @brief Sleep for \p sleep_us microseconds.
+ */
+static void do_sleep (int sleep_us) {
+ if (sleep_us > 100) {
+#ifdef _MSC_VER
+ Sleep(sleep_us / 1000);
+#else
+ usleep(sleep_us);
+#endif
+ } else {
+ rd_ts_t next = rd_clock() + (rd_ts_t)sleep_us;
+ while (next > rd_clock())
+ ;
+ }
+}
+
int main (int argc, char **argv) {
char *brokers = NULL;
@@ -725,7 +788,7 @@
int opt;
int sendflags = 0;
char *msgpattern = "librdkafka_performance testing!";
- int msgsize = (int)strlen(msgpattern);
+ int msgsize = -1;
const char *debug = NULL;
rd_ts_t now;
char errstr[512];
@@ -749,6 +812,8 @@
int rate_sleep = 0;
rd_kafka_topic_partition_list_t *topics;
int exitcode = 0;
+ rd_kafka_headers_t *hdrs = NULL;
+ rd_kafka_resp_err_t err;
/* Kafka configuration */
conf = rd_kafka_conf_new();
@@ -789,7 +854,7 @@
while ((opt =
getopt(argc, argv,
"PCG:t:p:b:s:k:c:fi:MDd:m:S:x:"
- "R:a:z:o:X:B:eT:Y:qvIur:lA:OwN")) != -1) {
+ "R:a:z:o:X:B:eT:Y:qvIur:lA:OwNHH:")) != -1) {
switch (opt) {
case 'G':
if (rd_kafka_conf_set(conf, "group.id", optarg,
@@ -888,6 +953,37 @@
case 'd':
debug = optarg;
break;
+ case 'H':
+ {
+ char *name, *val;
+ size_t name_sz = -1;
+
+ if (!optarg) {
+ read_hdrs = 1;
+ break;
+ }
+
+ name = optarg;
+ val = strchr(name, '=');
+ if (val) {
+ name_sz = (size_t)(val-name);
+ val++; /* past the '=' */
+ }
+
+ if (!hdrs)
+ hdrs = rd_kafka_headers_new(8);
+
+ err = rd_kafka_header_add(hdrs, name, name_sz, val, -1);
+ if (err) {
+ fprintf(stderr,
+ "%% Failed to add header %s: %s\n",
+ name, rd_kafka_err2str(err));
+ exit(1);
+ }
+
+ read_hdrs = 1;
+ }
+ break;
case 'X':
{
char *name, *val;
@@ -1033,6 +1129,8 @@
" -b Broker address list (host[:port],..)\n"
" -s Message size (producer)\n"
" -k Message key (producer)\n"
+ " -H Add header to message (producer)\n"
+ " -H Read message headers (consumer)\n"
" -c Messages to transmit/receive\n"
" -x Hard exit after transmitting messages (producer)\n"
" -D Copy/Duplicate data buffer (producer)\n"
@@ -1150,6 +1248,9 @@
if (msgcnt != -1)
forever = 0;
+ if (msgsize == -1)
+ msgsize = (int)strlen(msgpattern);
+
topic = topics->elems[0].topic;
if (mode == 'P') {
@@ -1261,10 +1362,9 @@
cnt.tx++;
while (run &&
- rd_kafka_produce(rkt, partition,
- sendflags, pbuf, msgsize,
- key, keylen, NULL) == -1) {
- rd_kafka_resp_err_t err = rd_kafka_last_error();
+ (err = do_produce(rk, rkt, partition, sendflags,
+ pbuf, msgsize,
+ key, keylen, hdrs))) {
if (err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION)
printf("%% No such partition: "
"%"PRId32"\n", partition);
@@ -1303,22 +1403,17 @@
cnt.msgs++;
cnt.bytes += msgsize;
- if (rate_sleep) {
- if (rate_sleep > 100) {
-#ifdef _MSC_VER
- Sleep(rate_sleep / 1000);
-#else
- usleep(rate_sleep);
-#endif
- } else {
- rd_ts_t next = rd_clock() + rate_sleep;
- while (next > rd_clock())
- ;
- }
- }
-
/* Must poll to handle delivery reports */
- rd_kafka_poll(rk, 0);
+ if (rate_sleep) {
+ rd_ts_t next = rd_clock() + (rd_ts_t) rate_sleep;
+ do {
+ rd_kafka_poll(rk,
+ (int)RD_MAX(0,
+ (next - rd_clock()) / 1000));
+ } while (next > rd_clock());
+ } else {
+ rd_kafka_poll(rk, 0);
+ }
print_stats(rk, mode, otype, compression);
}
@@ -1443,6 +1538,12 @@
if (r == -1)
fprintf(stderr, "%% Error: %s\n",
rd_kafka_err2str(rd_kafka_last_error()));
+ else if (r > 0 && rate_sleep) {
+ /* Simulate processing time
+ * if `-r ` was set. */
+ do_sleep(rate_sleep);
+ }
+
print_stats(rk, mode, otype, compression);
@@ -1524,6 +1625,11 @@
if (rkmessage) {
msg_consume(rkmessage, NULL);
rd_kafka_message_destroy(rkmessage);
+
+ /* Simulate processing time
+ * if `-r ` was set. */
+ if (rate_sleep)
+ do_sleep(rate_sleep);
}
cnt.t_fetch_latency += rd_clock() - fetch_latency;
@@ -1540,6 +1646,9 @@
rd_kafka_destroy(rk);
}
+ if (hdrs)
+ rd_kafka_headers_destroy(hdrs);
+
print_stats(NULL, mode, otype|_OTYPE_FORCE, compression);
if (cnt.t_fetch_latency && cnt.msgs)
diff -Nru librdkafka-0.11.3/examples/rdkafka_zookeeper_example.c librdkafka-0.11.6/examples/rdkafka_zookeeper_example.c
--- librdkafka-0.11.3/examples/rdkafka_zookeeper_example.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/examples/rdkafka_zookeeper_example.c 2018-10-10 06:54:38.000000000 +0000
@@ -541,8 +541,8 @@
/* Add brokers */
set_brokerlist_from_zookeeper(zh, brokers);
if (rd_kafka_conf_set(conf, "metadata.broker.list",
- brokers, errstr, sizeof(errstr) !=
- RD_KAFKA_CONF_OK)) {
+ brokers, errstr, sizeof(errstr)) !=
+ RD_KAFKA_CONF_OK) {
fprintf(stderr, "%% Failed to set brokers: %s\n", errstr);
exit(1);
}
diff -Nru librdkafka-0.11.3/.github/ISSUE_TEMPLATE librdkafka-0.11.6/.github/ISSUE_TEMPLATE
--- librdkafka-0.11.3/.github/ISSUE_TEMPLATE 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/.github/ISSUE_TEMPLATE 2018-10-10 06:54:38.000000000 +0000
@@ -1,11 +1,16 @@
-Description
-===========
+Read the FAQ first: https://github.com/edenhill/librdkafka/wiki/FAQ
+
+Description
+===========
+
How to reproduce
================
+
+
**IMPORTANT**: Always try to reproduce the issue on the latest released version (see https://github.com/edenhill/librdkafka/releases), if it can't be reproduced on the latest version the issue has been fixed.
@@ -17,10 +22,10 @@
Please provide the following information:
- - [ ] librdkafka version (release number or git tag): `e.g., v0.10.5`
- - [ ] Apache Kafka version: `e.g., 0.10.2.3`
- - [ ] librdkafka client configuration: `e.g., message.timeout.ms=123, auto.reset.offset=earliest, ..`
- - [ ] Operating system: `e.g., Centos 5 (x64)`
+ - [x] librdkafka version (release number or git tag): ``
+ - [ ] Apache Kafka version: ``
+ - [ ] librdkafka client configuration: ``
+ - [ ] Operating system: ``
- [ ] Provide logs (with `debug=..` as necessary) from librdkafka
- [ ] Provide broker log excerpts
- [ ] Critical issue
diff -Nru librdkafka-0.11.3/INTRODUCTION.md librdkafka-0.11.6/INTRODUCTION.md
--- librdkafka-0.11.3/INTRODUCTION.md 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/INTRODUCTION.md 2018-10-10 06:54:38.000000000 +0000
@@ -10,22 +10,22 @@
The following chapters are available in this document
- * Performance
- * Performance numbers
- * High throughput
- * Low latency
- * Compression
- * Message reliability
- * Usage
- * Documentation
- * Initialization
- * Configuration
- * Threads and callbacks
- * Brokers
- * Producer API
- * Consumer API
- * Appendix
- * Test detailts
+ * [Performance](#performance)
+ * [Performance numbers](#performance-numbers)
+ * [High throughput](#high-throughput)
+ * [Low latency](#low-latency)
+ * [Compression](#compression)
+ * [Message reliability](#message-reliability)
+ * [Usage](#usage)
+ * [Documentation](#documentation)
+ * [Initialization](#initialization)
+ * [Configuration](#configuration)
+ * [Threads and callbacks](#threads-and-callbacks)
+ * [Brokers](#brokers)
+ * [Producer API](#producer-api)
+ * [Consumer API](#simple-consumer-api-legacy)
+ * [Appendix](#appendix)
+ * [Test details](#test-details)
@@ -43,10 +43,13 @@
The two most important configuration properties for performance tuning are:
- * batch.num.messages - the minimum number of messages to wait for to
+ * `batch.num.messages` - the maximum number of messages to wait for to
accumulate in the local queue before sending off a message set.
- * queue.buffering.max.ms - how long to wait for batch.num.messages to
- fill up in the local queue.
+ * `queue.buffering.max.ms` - how long to wait for batch.num.messages to
+ fill up in the local queue. A lower value improves latency at the
+ cost of lower throughput and higher per-message overhead.
+ A higher value improves throughput at the expense of latency.
+ The recommended value for high throughput is > 50ms.
### Performance numbers
@@ -98,10 +101,42 @@
one large message set or batch to the peer. This amortizes the messaging
overhead and eliminates the adverse effect of the round trip time (rtt).
-The default settings, batch.num.messages=10000 and queue.buffering.max.ms=1000,
-are suitable for high throughput. This allows librdkafka to wait up to
-1000 ms for up to 10000 messages to accumulate in the local queue before
-sending the accumulate messages to the broker.
+`queue.buffering.max.ms` (also called `linger.ms`) allows librdkafka to
+wait up to the specified amount of time to accumulate up to
+`batch.num.messages` in a single batch (MessageSet) before sending
+to the broker. The larger the batch the higher the throughput.
+Enabling `msg` debugging (set `debug` property to `msg`) will emit log
+messages for the accumulation process which lets you see what batch sizes
+are being produced.
+
+Example using `queue.buffering.max.ms=1`:
+
+```
+... test [0]: MessageSet with 1514 message(s) delivered
+... test [3]: MessageSet with 1690 message(s) delivered
+... test [0]: MessageSet with 1720 message(s) delivered
+... test [3]: MessageSet with 2 message(s) delivered
+... test [3]: MessageSet with 4 message(s) delivered
+... test [0]: MessageSet with 4 message(s) delivered
+... test [3]: MessageSet with 11 message(s) delivered
+```
+
+Example using `queue.buffering.max.ms=1000`:
+```
+... test [0]: MessageSet with 10000 message(s) delivered
+... test [0]: MessageSet with 10000 message(s) delivered
+... test [0]: MessageSet with 4667 message(s) delivered
+... test [3]: MessageSet with 10000 message(s) delivered
+... test [3]: MessageSet with 10000 message(s) delivered
+... test [3]: MessageSet with 4476 message(s) delivered
+
+```
+
+
+The default setting of `queue.buffering.max.ms=1` is not suitable for
+high throughput, it is recommended to set this value to >50ms, with
+throughput leveling out somewhere around 100-1000ms depending on
+message produce pattern and sizes.
These setting are set globally (`rd_kafka_conf_t`) but applies on a
per topic+partition basis.
@@ -109,22 +144,80 @@
### Low latency
-When low latency messaging is required the "queue.buffering.max.ms" should be
+When low latency messaging is required the `queue.buffering.max.ms` should be
tuned to the maximum permitted producer-side latency.
Setting queue.buffering.max.ms to 1 will make sure messages are sent as
soon as possible. You could check out [How to decrease message latency](https://github.com/edenhill/librdkafka/wiki/How-to-decrease-message-latency)
to find more details.
+Lower buffering time leads to smaller batches and larger per-message overheads,
+increasing network, memory and CPU usage for producers, brokers and consumers.
+
+#### Latency measurement
+
+End-to-end latency is preferably measured by synchronizing clocks on producers
+and consumers and using the message timestamp on the consumer to calculate
+the full latency. Make sure the topic's `log.message.timestamp.type` is set to
+the default `CreateTime` (Kafka topic configuration, not librdkafka topic).
+
+Latencies are typically incurred by the producer, network and broker, the
+consumer effect on end-to-end latency is minimal.
+
+To break down the end-to-end latencies and find where latencies are adding up
+there are a number of metrics available through librdkafka statistics
+on the producer:
+
+ * `brokers[].int_latency` is the time, per message, between produce()
+ and the message being written to a MessageSet and ProduceRequest.
+ High `int_latency` indicates CPU core contention: check CPU load and,
+ involuntary context switches (`/proc/<..>/status`).
+ Consider using a machine/instance with more CPU cores.
+ This metric is only relevant on the producer.
+
+ * `brokers[].outbuf_latency` is the time, per protocol request
+ (such as ProduceRequest), between the request being enqueued (which happens
+ right after int_latency) and the time the request is written to the
+ TCP socket connected to the broker.
+ High `outbuf_latency` indicates CPU core contention or network congestion:
+ check CPU load and socket SendQ (`netstat -anp | grep :9092`).
+
+ * `brokers[].rtt` is the time, per protocol request, between the request being
+ written to the TCP socket and the time the response is received from
+ the broker.
+ High `rtt` indicates broker load or network congestion:
+ check broker metrics, local socket SendQ, network performance, etc.
+
+ * `brokers[].throttle` is the time, per throttled protocol request, the
+ broker throttled/delayed handling of a request due to usage quotas.
+ The throttle time will also be reflected in `rtt`.
+
+ * `topics[].batchsize` is the size of individual Producer MessageSet batches.
+ See below.
+
+ * `topics[].batchcnt` is the number of messages in individual Producer
+ MessageSet batches. Due to Kafka protocol overhead a batch with few messages
+ will have a higher relative processing and size overhead than a batch
+ with many messages.
+ Use the `linger.ms` client configuration property to set the maximum
+ amount of time allowed for accumulating a single batch, the larger the
+ value the larger the batches will grow, thus increasing efficiency.
+ When producing messages at a high rate it is recommended to increase
+ linger.ms, which will improve throughput and in some cases also latency.
+
+
+See [STATISTICS.md](STATISTICS.md) for the full definition of metrics.
+A JSON schema for the statistics is available in
+[statistics-schema.json](src/statistics-schema.json).
### Compression
-Producer message compression is enabled through the "compression.codec"
+Producer message compression is enabled through the `compression.codec`
configuration property.
Compression is performed on the batch of messages in the local queue, the
larger the batch the higher likelyhood of a higher compression ratio.
-The local batch queue size is controlled through the "batch.num.messages" and
-"queue.buffering.max.ms" configuration properties as described in the
+The local batch queue size is controlled through the `batch.num.messages` and
+`queue.buffering.max.ms` configuration properties as described in the
**High throughput** chapter above.
@@ -133,21 +226,22 @@
Message reliability is an important factor of librdkafka - an application
can rely fully on librdkafka to deliver a message according to the specified
-configuration ("request.required.acks" and "message.send.max.retries", etc).
+configuration (`request.required.acks` and `message.send.max.retries`, etc).
-If the topic configuration property "request.required.acks" is set to wait
+If the topic configuration property `request.required.acks` is set to wait
for message commit acknowledgements from brokers (any value but 0, see
[`CONFIGURATION.md`](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)
for specifics) then librdkafka will hold on to the message until
all expected acks have been received, gracefully handling the following events:
-
+
* Broker connection failure
* Topic leader change
* Produce errors signaled by the broker
+ * Network problems
This is handled automatically by librdkafka and the application does not need
to take any action at any of the above events.
-The message will be resent up to "message.send.max.retries" times before
+The message will be resent up to `message.send.max.retries` times before
reporting a failure back to the application.
The delivery report callback is used by librdkafka to signal the status of
@@ -160,9 +254,139 @@
See Producer API chapter for more details on delivery report callback usage.
-The delivery report callback is optional.
+The delivery report callback is optional but highly recommended.
+
+
+### Producer message delivery success
+
+When a ProduceRequest is successfully handled by the broker and a
+ProduceResponse is received (also called the ack) without an error code
+the messages from the ProduceRequest are enqueued on the delivery report
+queue (if a delivery report callback has been set) and will be passed to
+the application on the next invocation rd_kafka_poll().
+
+
+### Producer message delivery failure
+
+The following sub-chapters explains how different produce errors
+are handled.
+
+If the error is retryable and there are remaining retry attempts for
+the given message(s), an automatic retry will be scheduled by librdkafka,
+these retries are not visible to the application.
+
+Only permanent errors and temporary errors that have reached their maximum
+retry count will generate a delivery report event to the application with an
+error code set.
+
+The application should typically not attempt to retry producing the message
+on failure, but instead configure librdkafka to perform these retries
+using the `retries` and `retry.backoff.ms` configuration properties.
+
+
+#### Error: Timed out in transmission queue
+
+Internal error ERR__TIMED_OUT_QUEUE.
+
+The connectivity to the broker may be stalled due to networking contention,
+local or remote system issues, etc, and the request has not yet been sent.
+
+The producer can be certain that the message has not been sent to the broker.
+
+This is a retryable error, but is not counted as a retry attempt
+since the message was never actually transmitted.
+
+A retry by librdkafka at this point will not cause duplicate messages.
+
+
+#### Error: Timed out in flight to/from broker
+
+Internal error ERR__TIMED_OUT, ERR__TRANSPORT.
+
+Same reasons as for `Timed out in transmission queue` above, with the
+difference that the message may have been sent to the broker and might
+be stalling waiting for broker replicas to ack the message, or the response
+could be stalled due to networking issues.
+At this point the producer can't know if the message reached the broker,
+nor if the broker wrote the message to disk and replicas.
+
+This is a retryable error.
+
+A retry by librdkafka at this point may cause duplicate messages.
+
+
+#### Error: Temporary broker-side error
+Broker errors ERR_REQUEST_TIMED_OUT, ERR_NOT_ENOUGH_REPLICAS,
+ERR_NOT_ENOUGH_REPLICAS_AFTER_APPEND.
+These errors are considered temporary and librdkafka is will retry them
+if permitted by configuration.
+
+
+#### Error: Temporary errors due to stale metadata
+
+Broker errors ERR_LEADER_NOT_AVAILABLE, ERR_NOT_LEADER_FOR_PARTITION.
+
+These errors are considered temporary and a retry is warranted, a metadata
+request is automatically sent to find a new leader for the partition.
+
+A retry by librdkafka at this point will not cause duplicate messages.
+
+
+#### Error: Local time out
+
+Internal error ERR__MSG_TIMED_OUT.
+
+The message could not be successfully transmitted before `message.timeout.ms`
+expired, typically due to no leader being available or no broker connection.
+The message may have been retried due to other errors but
+those error messages are abstracted by the ERR__MSG_TIMED_OUT error code.
+
+Since the `message.timeout.ms` has passed there will be no more retries
+by librdkafka.
+
+
+#### Error: Permanent errors
+
+Any other error is considered a permanent error and the message
+will fail immediately, generating a delivery report event with the
+distinctive error code.
+
+The full list of permanent errors depend on the broker version and
+will likely grow in the future.
+
+Typical permanent broker errors are:
+ * ERR_CORRUPT_MESSAGE
+ * ERR_MSG_SIZE_TOO_LARGE - adjust client's or broker's `message.max.bytes`.
+ * ERR_UNKNOWN_TOPIC_OR_PART - topic or partition does not exist,
+ automatic topic creation is disabled on the
+ broker or the application is specifying a
+ partition that does not exist.
+ * ERR_RECORD_LIST_TOO_LARGE
+ * ERR_INVALID_REQUIRED_ACKS
+ * ERR_TOPIC_AUTHORIZATION_FAILED
+ * ERR_UNSUPPORTED_FOR_MESSAGE_FORMAT
+ * ERR_CLUSTER_AUTHORIZATION_FAILED
+
+
+### Producer retries
+
+The ProduceRequest itself is not retried, instead the messages
+are put back on the internal partition queue by an insert sort
+that maintains their original position (the message order is defined
+at the time a message is initially appended to a partition queue, i.e., after
+partitioning).
+A backoff time (`retry.backoff.ms`) is set on the retried messages which
+effectively blocks retry attempts until the backoff time has expired.
+
+
+### Reordering
+
+As for all retries, if `max.in.flight` > 1 and `retries` > 0, retried messages
+may be produced out of order, since a sub-sequent message in a sub-sequent
+ProduceRequest may already be in-flight (and accepted by the broker)
+by the time the retry for the failing message is sent.
@@ -173,7 +397,7 @@
The librdkafka API is documented in the
[`rdkafka.h`](https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h)
-header file, the configuration properties are documented in
+header file, the configuration properties are documented in
[`CONFIGURATION.md`](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)
### Initialization
@@ -268,11 +492,11 @@
librdkafka only needs an initial list of brokers (at least one), called the
bootstrap brokers.
It will connect to all the bootstrap brokers, specified by the
-"metadata.broker.list" configuration property or by `rd_kafka_brokers_add()`,
+`metadata.broker.list` configuration property or by `rd_kafka_brokers_add()`,
and query each one for Metadata information which contains the full list of
brokers, topic, partitions and their leaders in the Kafka cluster.
-Broker names are specified as "host[:port]" where the port is optional
+Broker names are specified as `host[:port]` where the port is optional
(default 9092) and the host is either a resolvable hostname or an IPv4 or IPv6
address.
If host resolves to multiple addresses librdkafka will round-robin the
@@ -280,6 +504,28 @@
A DNS record containing all broker address can thus be used to provide a
reliable bootstrap broker.
+#### Connection close
+
+A broker connection may be closed by the broker, intermediary network gear,
+due to network errors, timeouts, etc.
+When a broker connection is closed, librdkafka will wait for
+`reconnect.backoff.jitter.ms` +-50% before reconnecting.
+
+The broker will disconnect clients that have not sent any protocol requests
+within `connections.max.idle.ms` (broker configuration propertion, defaults
+to 10 minutes), but there is no fool proof way for the client to know that it
+was a deliberate close by the broker and not an error. To avoid logging these
+deliberate idle disconnects as errors the client employs some logic to try to
+classify a disconnect as an idle disconnect if no requests have been sent in
+the last `socket.timeout.ms` or there are no outstanding, or
+queued, requests waiting to be sent. In this case the standard "Disconnect"
+error log is silenced (will only be seen with debug enabled).
+
+`log.connection.close=false` may be used to silence all disconnect logs,
+but it is recommended to instead rely on the above heuristics.
+
+
+
### Feature discovery
Apache Kafka broker version 0.10.0 added support for the ApiVersionRequest API
@@ -338,9 +584,10 @@
`rd_kafka_produce()` is a non-blocking API, it will enqueue the message
on an internal queue and return immediately.
-If the number of queued messages would exceed the "queue.buffering.max.messages"
+If the number of queued messages would exceed the `queue.buffering.max.messages`
configuration property then `rd_kafka_produce()` returns -1 and sets errno
-to `ENOBUFS`, thus providing a backpressure mechanism.
+to `ENOBUFS` and last_error to `RD_KAFKA_RESP_ERR__QUEUE_FULL`, thus
+providing a backpressure mechanism.
**Note**: See `examples/rdkafka_performance.c` for a producer implementation.
@@ -369,8 +616,10 @@
`RD_KAFKA_OFFSET_STORED` to use the offset store.
After a topic+partition consumer has been started librdkafka will attempt
-to keep "queued.min.messages" messages in the local queue by repeatedly
-fetching batches of messages from the broker.
+to keep `queued.min.messages` messages in the local queue by repeatedly
+fetching batches of messages from the broker. librdkafka will fetch all
+consumed partitions for which that broker is a leader, through a single
+request.
This local message queue is then served to the application through three
different consume APIs:
@@ -446,7 +695,7 @@
#### Topic auto creation
Topic auto creation is supported by librdkafka.
-The broker needs to be configured with "auto.create.topics.enable=true".
+The broker needs to be configured with `auto.create.topics.enable=true`.
diff -Nru librdkafka-0.11.3/LICENSE librdkafka-0.11.6/LICENSE
--- librdkafka-0.11.3/LICENSE 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/LICENSE 2018-10-10 06:54:38.000000000 +0000
@@ -1,25 +1,25 @@
librdkafka - Apache Kafka C driver library
-Copyright (c) 2012, Magnus Edenhill
+Copyright (c) 2012-2018, Magnus Edenhill
All rights reserved.
Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
+modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
+ this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
+ and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
diff -Nru librdkafka-0.11.3/LICENSE.hdrhistogram librdkafka-0.11.6/LICENSE.hdrhistogram
--- librdkafka-0.11.3/LICENSE.hdrhistogram 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/LICENSE.hdrhistogram 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,27 @@
+This license covers src/rdhdrhistogram.c which is a C port of
+Coda Hale's Golang HdrHistogram https://github.com/codahale/hdrhistogram
+at revision 3a0bb77429bd3a61596f5e8a3172445844342120
+
+-----------------------------------------------------------------------------
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Coda Hale
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE
diff -Nru librdkafka-0.11.3/LICENSE.murmur2 librdkafka-0.11.6/LICENSE.murmur2
--- librdkafka-0.11.3/LICENSE.murmur2 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/LICENSE.murmur2 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,25 @@
+parts of src/rdmurmur2.c: git@github.com:abrandoned/murmur2.git
+
+
+MurMurHash2 Library
+//-----------------------------------------------------------------------------
+// MurmurHash2 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff -Nru librdkafka-0.11.3/LICENSES.txt librdkafka-0.11.6/LICENSES.txt
--- librdkafka-0.11.3/LICENSES.txt 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/LICENSES.txt 2018-10-10 06:54:38.000000000 +0000
@@ -2,26 +2,26 @@
--------------------------------------------------------------
librdkafka - Apache Kafka C driver library
-Copyright (c) 2012, Magnus Edenhill
+Copyright (c) 2012-2018, Magnus Edenhill
All rights reserved.
Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
+modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
+ this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
+ and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
@@ -59,6 +59,37 @@
*/
+LICENSE.hdrhistogram
+--------------------------------------------------------------
+This license covers src/rdhdrhistogram.c which is a C port of
+Coda Hale's Golang HdrHistogram https://github.com/codahale/hdrhistogram
+at revision 3a0bb77429bd3a61596f5e8a3172445844342120
+
+-----------------------------------------------------------------------------
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Coda Hale
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE
+
+
LICENSE.lz4
--------------------------------------------------------------
src/xxhash.[ch] src/lz4*.[ch]: git@github.com:lz4/lz4.git e2827775ee80d2ef985858727575df31fc60f1f3
@@ -89,6 +120,35 @@
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+LICENSE.murmur2
+--------------------------------------------------------------
+parts of src/rdmurmur2.c: git@github.com:abrandoned/murmur2.git
+
+
+MurMurHash2 Library
+//-----------------------------------------------------------------------------
+// MurmurHash2 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
LICENSE.pycrc
--------------------------------------------------------------
The following license applies to the files rdcrc32.c and rdcrc32.h which
diff -Nru librdkafka-0.11.3/Makefile librdkafka-0.11.6/Makefile
--- librdkafka-0.11.3/Makefile 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/Makefile 2018-10-10 06:54:38.000000000 +0000
@@ -31,7 +31,7 @@
check: file-check
@(for d in $(LIBSUBDIRS); do $(MAKE) -C $$d $@ || exit $?; done)
-install:
+install uninstall:
@(for d in $(LIBSUBDIRS); do $(MAKE) -C $$d $@ || exit $?; done)
examples tests: .PHONY libs
diff -Nru librdkafka-0.11.3/mklove/Makefile.base librdkafka-0.11.6/mklove/Makefile.base
--- librdkafka-0.11.3/mklove/Makefile.base 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/mklove/Makefile.base 2018-10-10 06:54:38.000000000 +0000
@@ -109,19 +109,37 @@
Version: $(MKL_APP_VERSION)
Cflags: -I$${includedir}
Libs: -L$${libdir} -l$(LIBNAME0)
-Libs.private: $(patsubst -L%,,$(LIBS))
+Libs.private: $(LIBS)
endef
export _PKG_CONFIG_DEF
+define _PKG_CONFIG_STATIC_DEF
+prefix=$(prefix)
+libdir=$(libdir)
+includedir=$(includedir)
+
+Name: $(LIBNAME)-static
+Description: $(MKL_APP_DESC_ONELINE) (static)
+Version: $(MKL_APP_VERSION)
+Cflags: -I$${includedir}
+Libs: -L$${libdir} $${libdir}/$(LIBNAME).a $(LIBS)
+endef
+
+export _PKG_CONFIG_STATIC_DEF
+
$(LIBNAME0).pc: $(TOPDIR)/Makefile.config
@printf "$(MKL_YELLOW)Generating pkg-config file $@$(MKL_CLR_RESET)\n"
@echo "$$_PKG_CONFIG_DEF" > $@
-lib-gen-pkg-config: $(LIBNAME0).pc
+$(LIBNAME0)-static.pc: $(TOPDIR)/Makefile.config
+ @printf "$(MKL_YELLOW)Generating pkg-config file $@$(MKL_CLR_RESET)\n"
+ @echo "$$_PKG_CONFIG_STATIC_DEF" > $@
+
+lib-gen-pkg-config: $(LIBNAME0).pc $(LIBNAME0)-static.pc
lib-clean-pkg-config:
- rm -f $(LIBNAME0).pc
+ rm -f $(LIBNAME0).pc $(LIBNAME0)-static.pc
else
lib-gen-pkg-config:
lib-clean-pkg-config:
@@ -159,6 +177,10 @@
$(INSTALL) -d $$DESTDIR$(pkgconfigdir) ; \
$(INSTALL) -m 0644 $(LIBNAME0).pc $$DESTDIR$(pkgconfigdir) \
) ; \
+ [ -f "$(LIBNAME0)-static.pc" ] && ( \
+ $(INSTALL) -d $$DESTDIR$(pkgconfigdir) ; \
+ $(INSTALL) -m 0644 $(LIBNAME0)-static.pc $$DESTDIR$(pkgconfigdir) \
+ ) ; \
(cd $$DESTDIR$(libdir) && ln -sf $(LIBFILENAME) $(LIBFILENAMELINK))
lib-uninstall:
@@ -169,8 +191,9 @@
rm -f $$DESTDIR$(libdir)/$(LIBFILENAME)
rm -f $$DESTDIR$(libdir)/$(LIBFILENAMELINK)
rmdir $$DESTDIR$(includedir)/$(PKGNAME) || true
-
-
+ rm -f $$DESTDIR$(pkgconfigdir)/$(LIBNAME0).pc
+ rm -f $$DESTDIR$(pkgconfigdir)/$(LIBNAME0)-static.pc
+ rmdir $$DESTDIR$(pkgconfigdir) || true
bin-install:
@printf "$(MKL_YELLOW)Install $(BIN) to $$DESTDIR$(prefix)$(MKL_CLR_RESET)\n"
@@ -180,6 +203,7 @@
bin-uninstall:
@printf "$(MKL_YELLOW)Uninstall $(BIN) from $$DESTDIR$(prefix)$(MKL_CLR_RESET)\n"
rm -f $$DESTDIR$(bindir)/$(BIN)
+ rmdir $$DESTDIR$(bindir) || true
generic-clean:
diff -Nru librdkafka-0.11.3/mklove/modules/configure.base librdkafka-0.11.6/mklove/modules/configure.base
--- librdkafka-0.11.3/mklove/modules/configure.base 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/mklove/modules/configure.base 2018-10-10 06:54:38.000000000 +0000
@@ -521,10 +521,12 @@
# Generate config.h
mkl_write_h "// Automatically generated by $0 $*"
- mkl_write_h "#pragma once"
+ mkl_write_h "#ifndef _CONFIG_H_"
+ mkl_write_h "#define _CONFIG_H_"
for n in $MKL_DEFINES ; do
mkl_write_h "${!n}"
done
+ mkl_write_h "#endif /* _CONFIG_H_ */"
MKL_OUTH_FINAL=config.h
mv $MKL_OUTH $MKL_OUTH_FINAL
@@ -1098,11 +1100,10 @@
fi
if [[ $pkg_conf_failed == 1 ]]; then
- if [[ $is_static == 1 ]]; then
- mkl_mkvar_prepend "$1" LIBS "$libs"
- else
- mkl_mkvar_append "$1" LIBS "$libs"
- fi
+ # Add libraries in reverse order to make sure inter-dependencies
+ # are resolved in the correct order.
+ # E.g., check for crypto and then ssl should result in -lssl -lcrypto
+ mkl_mkvar_prepend "$1" LIBS "$libs"
fi
return 0
@@ -1160,10 +1161,8 @@
if [[ $WITH_STATIC_LINKING == y && ! -z $staticopt ]]; then
libs=$(mkl_lib_check_static "${staticopt#*=}" "$libs")
- mkl_mkvar_prepend "$1" LIBS "$libs"
- else
- mkl_mkvar_append "$1" LIBS "$libs"
fi
+ mkl_mkvar_prepend "$1" LIBS "$libs"
mkl_check_done "$1" "$2" "$3" "ok"
diff -Nru librdkafka-0.11.3/mklove/modules/configure.cc librdkafka-0.11.6/mklove/modules/configure.cc
--- librdkafka-0.11.3/mklove/modules/configure.cc 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/mklove/modules/configure.cc 2018-10-10 06:54:38.000000000 +0000
@@ -105,7 +105,7 @@
fi
mkl_mkvar_set "pkgconfig" PKG_CONFIG $PKG_CONFIG
- [[ ! -z "$PKG_CONFIG_PATH" ]] && mkl_env_append PKG_CONFIG_PATH "$PKG_CONFIG_PATH"
+ [[ ! -z "$append_PKG_CONFIG_PATH" ]] && mkl_env_append PKG_CONFIG_PATH "$append_PKG_CONFIG_PATH" ":"
# install
if [ -z "$INSTALL" ]; then
@@ -169,7 +169,7 @@
mkl_option "Compiler" "mk:$n" "--$n=$n" "Add $n flags"
done
-mkl_option "Compiler" "env:PKG_CONFIG_PATH" "--pkg-config-path" "Extra paths for pkg-config"
+mkl_option "Compiler" "env:append_PKG_CONFIG_PATH" "--pkg-config-path=EXTRA_PATHS" "Extra paths for pkg-config"
mkl_option "Compiler" "WITH_PROFILING" "--enable-profiling" "Enable profiling"
mkl_option "Compiler" "WITH_STATIC_LINKING" "--enable-static" "Enable static linking"
diff -Nru librdkafka-0.11.3/packaging/alpine/test-build-alpine.sh librdkafka-0.11.6/packaging/alpine/test-build-alpine.sh
--- librdkafka-0.11.3/packaging/alpine/test-build-alpine.sh 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/packaging/alpine/test-build-alpine.sh 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+#
+# Build librdkafka on Alpine.
+# Must only be run from within an Alpine container where
+# the librdkafka root dir is mounted as /v
+#
+
+set -eu
+
+if [ ! -f /.dockerenv ] ; then
+ echo "$0 must be run in the docker container"
+ exit 1
+fi
+
+apk add bash gcc g++ make musl-dev bsd-compat-headers git python
+
+git clone /v /librdkafka
+
+cd /librdkafka
+./configure
+make
+make -C tests run_local
+cd ..
diff -Nru librdkafka-0.11.3/packaging/alpine/test-build.sh librdkafka-0.11.6/packaging/alpine/test-build.sh
--- librdkafka-0.11.3/packaging/alpine/test-build.sh 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/packaging/alpine/test-build.sh 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# Build librdkafka on Alpine using Docker, and run the local test suite.
+#
+
+set -eu
+echo -e "\033[35m### Building on Alpine ###\033[0m"
+exec docker run -v $PWD:/v alpine:3.8 /v/packaging/alpine/test-build-alpine.sh
diff -Nru librdkafka-0.11.3/packaging/cmake/README.md librdkafka-0.11.6/packaging/cmake/README.md
--- librdkafka-0.11.3/packaging/cmake/README.md 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/packaging/cmake/README.md 2018-10-10 06:54:38.000000000 +0000
@@ -3,7 +3,6 @@
The cmake build mode is experimental and not officially supported,
the community is asked to maintain and support this mode through PRs.
-
Set up build environment (from top-level librdkafka directory):
$ cmake -H. -B_cmake_build
@@ -17,6 +16,10 @@
$ cmake --build _cmake_build
+If you want to build static library:
+
+ $ cmake --build _cmake_build -DRDKAFKA_BUILD_STATIC=1
+
Run (local) tests:
@@ -26,3 +29,10 @@
Install library:
$ cmake --build _cmake_build --target install
+
+
+If you use librdkafka as submodule in cmake project and want static link of librdkafka:
+
+ set(RDKAFKA_BUILD_STATIC ON CACHE BOOL "")
+ add_subdirectory(librdkafka)
+ target_link_libraries(your_library_or_executable rdkafka)
diff -Nru librdkafka-0.11.3/packaging/debian/docs librdkafka-0.11.6/packaging/debian/docs
--- librdkafka-0.11.3/packaging/debian/docs 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/packaging/debian/docs 2018-10-10 06:54:38.000000000 +0000
@@ -1,3 +1,4 @@
README.md
INTRODUCTION.md
CONFIGURATION.md
+STATISTICS.md
diff -Nru librdkafka-0.11.3/packaging/debian/rules librdkafka-0.11.6/packaging/debian/rules
--- librdkafka-0.11.3/packaging/debian/rules 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/packaging/debian/rules 2018-10-10 06:54:38.000000000 +0000
@@ -13,5 +13,7 @@
dh_auto_install
install -D -m 0644 rdkafka.pc \
debian/librdkafka-dev/usr/lib/${DEB_HOST_MULTIARCH}/pkgconfig/rdkafka.pc
+ install -D -m 0644 rdkafka-static.pc \
+ debian/librdkafka-dev/usr/lib/${DEB_HOST_MULTIARCH}/pkgconfig/rdkafka-static.pc
.PHONY: override_dh_strip override_dh_auth_install
diff -Nru librdkafka-0.11.3/packaging/nuget/packaging.py librdkafka-0.11.6/packaging/nuget/packaging.py
--- librdkafka-0.11.3/packaging/nuget/packaging.py 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/packaging/nuget/packaging.py 2018-10-10 06:54:38.000000000 +0000
@@ -12,6 +12,7 @@
import shutil
import subprocess
import urllib
+from fnmatch import fnmatch
from string import Template
from collections import defaultdict
import boto3
@@ -19,7 +20,7 @@
# Rename token values
-rename_vals = {'plat': {'windows': 'win7'},
+rename_vals = {'plat': {'windows': 'win'},
'arch': {'x86_64': 'x64',
'i386': 'x86',
'win32': 'x86'}}
@@ -299,40 +300,46 @@
a.info['toolset'] = 'v120'
mappings = [
- [{'arch': 'x64', 'plat': 'linux', 'fname_startswith': 'librdkafka.tar.gz'}, './include/librdkafka/rdkafka.h', 'build/native/include/librdkafka/rdkafka.h'],
- [{'arch': 'x64', 'plat': 'linux', 'fname_startswith': 'librdkafka.tar.gz'}, './include/librdkafka/rdkafkacpp.h', 'build/native/include/librdkafka/rdkafkacpp.h'],
+ [{'arch': 'x64', 'plat': 'linux', 'fname_glob': 'librdkafka.tar.gz'}, './include/librdkafka/rdkafka.h', 'build/native/include/librdkafka/rdkafka.h'],
+ [{'arch': 'x64', 'plat': 'linux', 'fname_glob': 'librdkafka.tar.gz'}, './include/librdkafka/rdkafkacpp.h', 'build/native/include/librdkafka/rdkafkacpp.h'],
- [{'arch': 'x64', 'plat': 'osx', 'fname_startswith': 'librdkafka.tar.gz'}, './lib/librdkafka.dylib', 'runtimes/osx-x64/native/librdkafka.dylib'],
- [{'arch': 'x64', 'plat': 'linux', 'fname_startswith': 'librdkafka-debian9.tgz'}, './lib/librdkafka.so.1', 'runtimes/linux-x64/native/debian9-librdkafka.so'],
- [{'arch': 'x64', 'plat': 'linux', 'fname_startswith': 'librdkafka.tar.gz'}, './lib/librdkafka.so.1', 'runtimes/linux-x64/native/librdkafka.so'],
+ # Travis OSX build
+ [{'arch': 'x64', 'plat': 'osx', 'fname_glob': 'librdkafka.tar.gz'}, './lib/librdkafka.dylib', 'runtimes/osx-x64/native/librdkafka.dylib'],
+ # Travis Debian 9 / Ubuntu 16.04 build
+ [{'arch': 'x64', 'plat': 'linux', 'fname_glob': 'librdkafka-debian9.tgz'}, './lib/librdkafka.so.1', 'runtimes/linux-x64/native/debian9-librdkafka.so'],
+ # Travis Ubuntu 14.04 build
+ [{'arch': 'x64', 'plat': 'linux', 'fname_glob': 'librdkafka.tar.gz'}, './lib/librdkafka.so.1', 'runtimes/linux-x64/native/librdkafka.so'],
+ # Travis CentOS RPM build
+ [{'arch': 'x64', 'plat': 'linux', 'fname_glob': 'librdkafka1*.x86_64.rpm'}, './usr/lib64/librdkafka.so.1', 'runtimes/linux-x64/native/centos7-librdkafka.so'],
- [{'arch': 'x64', 'plat': 'win7', 'fname_startswith': 'msvcr120.zip'}, 'msvcr120.dll', 'runtimes/win7-x64/native/msvcr120.dll'],
+ # Common Win runtime
+ [{'arch': 'x64', 'plat': 'win', 'fname_glob': 'msvcr120.zip'}, 'msvcr120.dll', 'runtimes/win-x64/native/msvcr120.dll'],
# matches librdkafka.redist.{VER}.nupkg
- [{'arch': 'x64', 'plat': 'win7', 'fname_startswith': 'librdkafka.redist'}, 'build/native/bin/v120/x64/Release/librdkafka.dll', 'runtimes/win7-x64/native/librdkafka.dll'],
- [{'arch': 'x64', 'plat': 'win7', 'fname_startswith': 'librdkafka.redist'}, 'build/native/bin/v120/x64/Release/librdkafkacpp.dll', 'runtimes/win7-x64/native/librdkafkacpp.dll'],
- [{'arch': 'x64', 'plat': 'win7', 'fname_startswith': 'librdkafka.redist'}, 'build/native/bin/v120/x64/Release/zlib.dll', 'runtimes/win7-x64/native/zlib.dll'],
+ [{'arch': 'x64', 'plat': 'win', 'fname_glob': 'librdkafka.redist*'}, 'build/native/bin/v120/x64/Release/librdkafka.dll', 'runtimes/win-x64/native/librdkafka.dll'],
+ [{'arch': 'x64', 'plat': 'win', 'fname_glob': 'librdkafka.redist*'}, 'build/native/bin/v120/x64/Release/librdkafkacpp.dll', 'runtimes/win-x64/native/librdkafkacpp.dll'],
+ [{'arch': 'x64', 'plat': 'win', 'fname_glob': 'librdkafka.redist*'}, 'build/native/bin/v120/x64/Release/zlib.dll', 'runtimes/win-x64/native/zlib.dll'],
# matches librdkafka.{VER}.nupkg
- [{'arch': 'x64', 'plat': 'win7', 'fname_startswith': 'librdkafka', 'fname_excludes': ['redist', 'symbols']},
- 'build/native/lib/v120/x64/Release/librdkafka.lib', 'build/native/lib/win7/x64/win7-x64-Release/v120/librdkafka.lib'],
- [{'arch': 'x64', 'plat': 'win7', 'fname_startswith': 'librdkafka', 'fname_excludes': ['redist', 'symbols']},
- 'build/native/lib/v120/x64/Release/librdkafkacpp.lib', 'build/native/lib/win7/x64/win7-x64-Release/v120/librdkafkacpp.lib'],
+ [{'arch': 'x64', 'plat': 'win', 'fname_glob': 'librdkafka*', 'fname_excludes': ['redist', 'symbols']},
+ 'build/native/lib/v120/x64/Release/librdkafka.lib', 'build/native/lib/win/x64/win-x64-Release/v120/librdkafka.lib'],
+ [{'arch': 'x64', 'plat': 'win', 'fname_glob': 'librdkafka*', 'fname_excludes': ['redist', 'symbols']},
+ 'build/native/lib/v120/x64/Release/librdkafkacpp.lib', 'build/native/lib/win/x64/win-x64-Release/v120/librdkafkacpp.lib'],
- [{'arch': 'x86', 'plat': 'win7', 'fname_startswith': 'msvcr120.zip'}, 'msvcr120.dll', 'runtimes/win7-x86/native/msvcr120.dll'],
+ [{'arch': 'x86', 'plat': 'win', 'fname_glob': 'msvcr120.zip'}, 'msvcr120.dll', 'runtimes/win-x86/native/msvcr120.dll'],
# matches librdkafka.redist.{VER}.nupkg
- [{'arch': 'x86', 'plat': 'win7', 'fname_startswith': 'librdkafka.redist'}, 'build/native/bin/v120/Win32/Release/librdkafka.dll', 'runtimes/win7-x86/native/librdkafka.dll'],
- [{'arch': 'x86', 'plat': 'win7', 'fname_startswith': 'librdkafka.redist'}, 'build/native/bin/v120/Win32/Release/librdkafkacpp.dll', 'runtimes/win7-x86/native/librdkafkacpp.dll'],
- [{'arch': 'x86', 'plat': 'win7', 'fname_startswith': 'librdkafka.redist'}, 'build/native/bin/v120/Win32/Release/zlib.dll', 'runtimes/win7-x86/native/zlib.dll'],
+ [{'arch': 'x86', 'plat': 'win', 'fname_glob': 'librdkafka.redist*'}, 'build/native/bin/v120/Win32/Release/librdkafka.dll', 'runtimes/win-x86/native/librdkafka.dll'],
+ [{'arch': 'x86', 'plat': 'win', 'fname_glob': 'librdkafka.redist*'}, 'build/native/bin/v120/Win32/Release/librdkafkacpp.dll', 'runtimes/win-x86/native/librdkafkacpp.dll'],
+ [{'arch': 'x86', 'plat': 'win', 'fname_glob': 'librdkafka.redist*'}, 'build/native/bin/v120/Win32/Release/zlib.dll', 'runtimes/win-x86/native/zlib.dll'],
# matches librdkafka.{VER}.nupkg
- [{'arch': 'x86', 'plat': 'win7', 'fname_startswith': 'librdkafka', 'fname_excludes': ['redist', 'symbols']},
- 'build/native/lib/v120/Win32/Release/librdkafka.lib', 'build/native/lib/win7/x86/win7-x86-Release/v120/librdkafka.lib'],
- [{'arch': 'x86', 'plat': 'win7', 'fname_startswith': 'librdkafka', 'fname_excludes': ['redist', 'symbols']},
- 'build/native/lib/v120/Win32/Release/librdkafkacpp.lib', 'build/native/lib/win7/x86/win7-x86-Release/v120/librdkafkacpp.lib']
+ [{'arch': 'x86', 'plat': 'win', 'fname_glob': 'librdkafka*', 'fname_excludes': ['redist', 'symbols']},
+ 'build/native/lib/v120/Win32/Release/librdkafka.lib', 'build/native/lib/win/x86/win-x86-Release/v120/librdkafka.lib'],
+ [{'arch': 'x86', 'plat': 'win', 'fname_glob': 'librdkafka*', 'fname_excludes': ['redist', 'symbols']},
+ 'build/native/lib/v120/Win32/Release/librdkafkacpp.lib', 'build/native/lib/win/x86/win-x86-Release/v120/librdkafkacpp.lib']
]
for m in mappings:
attributes = m[0]
- fname_startswith = attributes['fname_startswith']
- del attributes['fname_startswith']
+ fname_glob = attributes['fname_glob']
+ del attributes['fname_glob']
fname_excludes = []
if 'fname_excludes' in attributes:
fname_excludes = attributes['fname_excludes']
@@ -347,7 +354,7 @@
found = False
break
- if not a.fname.startswith(fname_startswith):
+ if not fnmatch(a.fname, fname_glob):
found = False
for exclude in fname_excludes:
@@ -360,7 +367,7 @@
break
if artifact is None:
- raise Exception('unable to find file in archive %s with tags %s that starts with "%s"' % (a.fname, str(attributes), fname_startswith))
+ raise Exception('unable to find artifact with tags %s matching "%s"' % (str(attributes), fname_glob))
outf = os.path.join(self.stpath, m[2])
member = m[1]
@@ -389,21 +396,21 @@
"build/native/librdkafka.redist.targets",
"build/native/include/librdkafka/rdkafka.h",
"build/native/include/librdkafka/rdkafkacpp.h",
- "build/native/lib/win7/x64/win7-x64-Release/v120/librdkafka.lib",
- "build/native/lib/win7/x64/win7-x64-Release/v120/librdkafkacpp.lib",
- "build/native/lib/win7/x86/win7-x86-Release/v120/librdkafka.lib",
- "build/native/lib/win7/x86/win7-x86-Release/v120/librdkafkacpp.lib",
+ "build/native/lib/win/x64/win-x64-Release/v120/librdkafka.lib",
+ "build/native/lib/win/x64/win-x64-Release/v120/librdkafkacpp.lib",
+ "build/native/lib/win/x86/win-x86-Release/v120/librdkafka.lib",
+ "build/native/lib/win/x86/win-x86-Release/v120/librdkafkacpp.lib",
"runtimes/linux-x64/native/debian9-librdkafka.so",
"runtimes/linux-x64/native/librdkafka.so",
"runtimes/osx-x64/native/librdkafka.dylib",
- "runtimes/win7-x64/native/librdkafka.dll",
- "runtimes/win7-x64/native/librdkafkacpp.dll",
- "runtimes/win7-x64/native/msvcr120.dll",
- "runtimes/win7-x64/native/zlib.dll",
- "runtimes/win7-x86/native/librdkafka.dll",
- "runtimes/win7-x86/native/librdkafkacpp.dll",
- "runtimes/win7-x86/native/msvcr120.dll",
- "runtimes/win7-x86/native/zlib.dll"]
+ "runtimes/win-x64/native/librdkafka.dll",
+ "runtimes/win-x64/native/librdkafkacpp.dll",
+ "runtimes/win-x64/native/msvcr120.dll",
+ "runtimes/win-x64/native/zlib.dll",
+ "runtimes/win-x86/native/librdkafka.dll",
+ "runtimes/win-x86/native/librdkafkacpp.dll",
+ "runtimes/win-x86/native/msvcr120.dll",
+ "runtimes/win-x86/native/zlib.dll"]
missing = list()
with zfile.ZFile(path, 'r') as zf:
diff -Nru librdkafka-0.11.3/packaging/nuget/templates/librdkafka.redist.props librdkafka-0.11.6/packaging/nuget/templates/librdkafka.redist.props
--- librdkafka-0.11.3/packaging/nuget/templates/librdkafka.redist.props 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/packaging/nuget/templates/librdkafka.redist.props 2018-10-10 06:54:38.000000000 +0000
@@ -1,11 +1,11 @@
-
+
librdkafka\x86\%(Filename)%(Extension)
PreserveNewest
-
+
librdkafka\x64\%(Filename)%(Extension)
PreserveNewest
diff -Nru librdkafka-0.11.3/packaging/nuget/templates/librdkafka.redist.targets librdkafka-0.11.6/packaging/nuget/templates/librdkafka.redist.targets
--- librdkafka-0.11.3/packaging/nuget/templates/librdkafka.redist.targets 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/packaging/nuget/templates/librdkafka.redist.targets 2018-10-10 06:54:38.000000000 +0000
@@ -1,19 +1,19 @@
- $(MSBuildThisFileDirectory)..\..\runtimes\win7-x64\native\librdkafka.lib;%(AdditionalDependencies)
- $(MSBuildThisFileDirectory)..\..\runtimes\win7-x86\native\librdkafka.lib;%(AdditionalDependencies)
- $(MSBuildThisFileDirectory)..\..\runtimes\win7-x64\native;%(AdditionalLibraryDirectories)
- $(MSBuildThisFileDirectory)..\..\runtimes\win7-x86\native;%(AdditionalLibraryDirectories)
+ $(MSBuildThisFileDirectory)lib\win\x64\win-x64-Release\v120\librdkafka.lib;%(AdditionalDependencies)
+ $(MSBuildThisFileDirectory)lib\win\x86\win-x86-Release\v120\librdkafka.lib;%(AdditionalDependencies)
+ $(MSBuildThisFileDirectory)lib\win\x64\win-x64-Release\v120;%(AdditionalLibraryDirectories)
+ $(MSBuildThisFileDirectory)lib\win\x86\win-x86-Release\v120;%(AdditionalLibraryDirectories)
$(MSBuildThisFileDirectory)include;%(AdditionalIncludeDirectories)
-
+
-
+
diff -Nru librdkafka-0.11.3/packaging/rpm/librdkafka.spec librdkafka-0.11.6/packaging/rpm/librdkafka.spec
--- librdkafka-0.11.3/packaging/rpm/librdkafka.spec 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/packaging/rpm/librdkafka.spec 2018-10-10 06:54:38.000000000 +0000
@@ -9,7 +9,7 @@
URL: https://github.com/edenhill/librdkafka
Source: librdkafka-%{version}.tar.gz
-BuildRequires: zlib-devel libstdc++-devel gcc >= 4.1 gcc-c++ openssl-devel cyrus-sasl-devel lz4-devel python
+BuildRequires: zlib-devel libstdc++-devel gcc >= 4.1 gcc-c++ openssl-devel cyrus-sasl-devel python
BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
%define _source_payload w9.gzdio
@@ -69,8 +69,8 @@
%{_libdir}/librdkafka.so.%{soname}
%{_libdir}/librdkafka++.so.%{soname}
%defattr(-,root,root)
-%doc README.md CONFIGURATION.md INTRODUCTION.md
-%doc LICENSE LICENSE.pycrc LICENSE.queue LICENSE.snappy LICENSE.tinycthread LICENSE.wingetopt
+%doc README.md CONFIGURATION.md INTRODUCTION.md STATISTICS.md
+%doc LICENSE LICENSES.txt
%defattr(-,root,root)
#%{_bindir}/rdkafka_example
@@ -87,7 +87,8 @@
%{_libdir}/librdkafka++.so
%{_libdir}/pkgconfig/rdkafka++.pc
%{_libdir}/pkgconfig/rdkafka.pc
-
+%{_libdir}/pkgconfig/rdkafka-static.pc
+%{_libdir}/pkgconfig/rdkafka++-static.pc
%changelog
* Thu Apr 09 2015 Eduard Iskandarov 0.8.6-0
diff -Nru librdkafka-0.11.3/packaging/rpm/Makefile librdkafka-0.11.6/packaging/rpm/Makefile
--- librdkafka-0.11.3/packaging/rpm/Makefile 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/packaging/rpm/Makefile 2018-10-10 06:54:38.000000000 +0000
@@ -34,7 +34,8 @@
--no-clean --no-cleanup-after \
--buildsrpm \
--spec=librdkafka.spec \
- --sources=SOURCES
+ --sources=SOURCES || \
+ (tail -n 100 pkgs-$(VERSION)*/*log ; false)
@echo "======= Source RPM now available in $(RESULT_DIR) ======="
rpm: srpm
@@ -44,7 +45,8 @@
--define "__release $(BUILD_NUMBER)"\
--resultdir=$(RESULT_DIR) \
--no-clean --no-cleanup-after \
- --rebuild $(RESULT_DIR)/$(PACKAGE_NAME)*.src.rpm
+ --rebuild $(RESULT_DIR)/$(PACKAGE_NAME)*.src.rpm || \
+ (tail -n 100 pkgs-$(VERSION)*/*log ; false)
@echo "======= Binary RPMs now available in $(RESULT_DIR) ======="
copy-artifacts:
diff -Nru librdkafka-0.11.3/packaging/rpm/mock-on-docker.sh librdkafka-0.11.6/packaging/rpm/mock-on-docker.sh
--- librdkafka-0.11.3/packaging/rpm/mock-on-docker.sh 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/packaging/rpm/mock-on-docker.sh 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+#
+
+# Run mock in docker
+
+set -ex
+
+_DOCKER_IMAGE=centos:7
+export MOCK_CONFIG=epel-7-x86_64
+
+if [[ ! -f /.dockerenv ]]; then
+ #
+ # Running on host, fire up a docker container a run it.
+ #
+
+ if [[ ! -f configure.librdkafka ]]; then
+ echo "$0 must be run from librdkafka top directory"
+ exit 1
+ fi
+
+ docker run --privileged=true -t -v $(pwd):/io \
+ $_DOCKER_IMAGE /io/packaging/rpm/mock-on-docker.sh
+
+ pushd packaging/rpm
+ make copy-artifacts
+ popd
+
+else
+
+ yum install -y python mock make git
+
+ cfg_file=/etc/mock/${MOCK_CONFIG}.cfg
+ ls -la /etc/mock
+ echo "config_opts['plugin_conf']['bind_mount_enable'] = False" >> $cfg_file
+ echo "config_opts['package_manager'] = 'yum'" >> $cfg_file
+ cat $cfg_file
+ pushd /io/packaging/rpm
+ make all
+ popd
+fi
diff -Nru librdkafka-0.11.3/README.md librdkafka-0.11.6/README.md
--- librdkafka-0.11.3/README.md 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/README.md 2018-10-10 06:54:38.000000000 +0000
@@ -1,7 +1,7 @@
librdkafka - the Apache Kafka C/C++ client library
==================================================
-Copyright (c) 2012-2016, [Magnus Edenhill](http://www.edenhill.se/).
+Copyright (c) 2012-2018, [Magnus Edenhill](http://www.edenhill.se/).
[https://github.com/edenhill/librdkafka](https://github.com/edenhill/librdkafka)
@@ -33,11 +33,11 @@
* [SASL](https://github.com/edenhill/librdkafka/wiki/Using-SASL-with-librdkafka) (GSSAPI/Kerberos/SSPI, PLAIN, SCRAM) support
* Broker version support: >=0.8 (see [Broker version compatibility](https://github.com/edenhill/librdkafka/wiki/Broker-version-compatibility))
* Stable C & C++ APIs (ABI safety guaranteed for C)
- * [Statistics](https://github.com/edenhill/librdkafka/wiki/Statistics) metrics
+ * [Statistics](https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md) metrics
* Debian package: librdkafka1 and librdkafka-dev in Debian and Ubuntu
* RPM package: librdkafka and librdkafka-devel
* Gentoo package: dev-libs/librdkafka
- * Portable: runs on Linux, OSX, Win32, Solaris, FreeBSD, ...
+ * Portable: runs on Linux, OSX, Win32, Solaris, FreeBSD, AIX, ...
# Language bindings #
@@ -83,8 +83,11 @@
* [OVH](http://ovh.com) - [AntiDDOS](http://www.slideshare.net/hugfrance/hugfr-6-oct2014ovhantiddos)
* [otto.de](http://otto.de)'s [trackdrd](https://github.com/otto-de/trackrdrd) - Varnish log reader
* [Microwish](https://github.com/microwish) has a range of Kafka utilites for log aggregation, HDFS integration, etc.
- * [aidp](https://github.com/weiboad/aidp) - kafka consumer embedded Lua scripting language in data process framework
- * large unnamed financial institution
+ * [aidp](https://github.com/weiboad/aidp) - kafka consumer embedded Lua scripting language in data process framework
+ * [Yandex ClickHouse](https://github.com/yandex/ClickHouse)
+ * [NXLog](http://nxlog.co/) - Enterprise logging system, Kafka input/output plugin.
+ * large unnamed financial institutions
+ * and many more..
* *Let [me](mailto:rdkafka@edenhill.se) know if you are using librdkafka*
@@ -95,7 +98,7 @@
The GNU toolchain
GNU make
pthreads
- zlib (optional, for gzip compression support)
+ zlib-dev (optional, for gzip compression support)
libssl-dev (optional, for SSL and SASL SCRAM support)
libsasl2-dev (optional, for SASL GSSAPI support)
@@ -111,6 +114,10 @@
**NOTE**: See [README.win32](README.win32) for instructions how to build
on Windows with Microsoft Visual Studio.
+**NOTE**: See [CMake instructions](packaging/cmake/README.md) for experimental
+ CMake build (unsupported).
+
+
### Usage in code
See [examples/rdkafka_example.c](https://github.com/edenhill/librdkafka/blob/master/examples/rdkafka_example.c) for an example producer and consumer.
diff -Nru librdkafka-0.11.3/README.win32 librdkafka-0.11.6/README.win32
--- librdkafka-0.11.3/README.win32 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/README.win32 2018-10-10 06:54:38.000000000 +0000
@@ -21,7 +21,6 @@
Missing:
- remaining tools (rdkafka_performance, etc)
- - SASL support (no official Cyrus libsasl2 DLLs available)
If you build librdkafka with an external tool (ie CMake) you can get rid of the
__declspec(dllexport) / __declspec(dllimport) decorations by adding a define
diff -Nru librdkafka-0.11.3/src/CMakeLists.txt librdkafka-0.11.6/src/CMakeLists.txt
--- librdkafka-0.11.3/src/CMakeLists.txt 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/CMakeLists.txt 2018-10-10 06:54:38.000000000 +0000
@@ -34,8 +34,13 @@
rdkafka_topic.c
rdkafka_transport.c
rdkafka_interceptor.c
+ rdkafka_header.c
+ rdkafka_admin.c
+ rdkafka_aux.c
+ rdkafka_background.c
rdlist.c
rdlog.c
+ rdmurmur2.c
rdports.c
rdrand.c
rdregex.c
@@ -44,18 +49,19 @@
rdvarint.c
snappy.c
tinycthread.c
+ tinycthread_extra.c
xxhash.c
lz4.c
lz4frame.c
lz4hc.c
)
-if(WITH_LIBDL)
- list(APPEND sources rddl.c)
+if(WITH_LIBDL OR WIN32)
+ list(APPEND sources rddl.c)
endif()
if(WITH_PLUGINS)
- list(APPEND sources rdkafka_plugin.c)
+ list(APPEND sources rdkafka_plugin.c)
endif()
if(WIN32)
@@ -76,10 +82,54 @@
list(APPEND sources regexp.c)
endif()
-add_library(rdkafka SHARED ${sources})
+# Define flags with cmake instead of by defining them on win32_config.h
+if(WITHOUT_WIN32_CONFIG)
+ list(APPEND rdkafka_compile_definitions WITHOUT_WIN32_CONFIG)
+ if(WITH_SSL)
+ list(APPEND rdkafka_compile_definitions WITH_SSL)
+ endif(WITH_SSL)
+ if(WITH_ZLIB)
+ list(APPEND rdkafka_compile_definitions WITH_ZLIB)
+ endif(WITH_ZLIB)
+ if(WITH_SNAPPY)
+ list(APPEND rdkafka_compile_definitions WITH_SNAPPY)
+ endif(WITH_SNAPPY)
+ if(WITH_SASL_SCRAM)
+ list(APPEND rdkafka_compile_definitions WITH_SASL_SCRAM)
+ endif(WITH_SASL_SCRAM)
+ if(ENABLE_DEVEL)
+ list(APPEND rdkafka_compile_definitions ENABLE_DEVEL)
+ endif(ENABLE_DEVEL)
+ if(WITH_PLUGINS)
+ list(APPEND rdkafka_compile_definitions WITH_PLUGINS)
+ endif(WITH_PLUGINS)
+endif()
+
+option(RDKAFKA_BUILD_STATIC "Build static rdkafka library" OFF)
+
+if(RDKAFKA_BUILD_STATIC)
+ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+ set(RDKAFKA_BUILD_MODE STATIC)
+else()
+ set(RDKAFKA_BUILD_MODE SHARED)
+endif()
+
+add_library(rdkafka ${RDKAFKA_BUILD_MODE} ${sources})
# Support '#include '
target_include_directories(rdkafka PUBLIC "$")
+target_compile_definitions(rdkafka PUBLIC ${rdkafka_compile_definitions})
+if(RDKAFKA_BUILD_STATIC)
+ target_compile_definitions(rdkafka PUBLIC LIBRDKAFKA_STATICLIB)
+endif()
+
+if(WIN32)
+ if(RDKAFKA_BUILD_STATIC)
+ target_link_libraries(rdkafka PUBLIC crypt32)
+ else()
+ target_compile_definitions(rdkafka PRIVATE LIBRDKAFKA_EXPORTS)
+ endif()
+endif()
# We need 'dummy' directory to support `#include "../config.h"` path
set(dummy "${GENERATED_DIR}/dummy")
diff -Nru librdkafka-0.11.3/src/crc32c.c librdkafka-0.11.6/src/crc32c.c
--- librdkafka-0.11.3/src/crc32c.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/crc32c.c 2018-10-10 06:54:38.000000000 +0000
@@ -105,7 +105,7 @@
len--;
}
while (len >= 8) {
-#if defined(__sparc) || defined(__sparc__) || defined(__APPLE__) || defined(__mips__)
+#if defined(__sparc) || defined(__sparc__) || defined(__APPLE__) || defined(__mips__) || defined(__arm__)
/* Alignment-safe alternative.
* This is also needed on Apple to avoid compilation warnings for
* non-appearant alignment reasons. */
diff -Nru librdkafka-0.11.3/src/crc32c.h librdkafka-0.11.6/src/crc32c.h
--- librdkafka-0.11.3/src/crc32c.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/crc32c.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,10 +26,13 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RD_CRC32C_H_
+#define _RD_CRC32C_H_
uint32_t crc32c(uint32_t crc, const void *buf, size_t len);
void crc32c_global_init (void);
int unittest_crc32c (void);
+
+#endif /* _RD_CRC32C_H_ */
diff -Nru librdkafka-0.11.3/src/Makefile librdkafka-0.11.6/src/Makefile
--- librdkafka-0.11.3/src/Makefile 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/Makefile 2018-10-10 06:54:38.000000000 +0000
@@ -13,6 +13,7 @@
SRCS_$(WITH_SASL_SCRAM) += rdkafka_sasl_scram.c
SRCS_$(WITH_SNAPPY) += snappy.c
SRCS_$(WITH_ZLIB) += rdgz.c
+SRCS_$(WITH_HDRHISTOGRAM) += rdhdrhistogram.c
SRCS_LZ4 = xxhash.c
ifneq ($(WITH_LZ4_EXT), y)
@@ -35,11 +36,14 @@
rdkafka_partition.c rdkafka_subscription.c \
rdkafka_assignor.c rdkafka_range_assignor.c \
rdkafka_roundrobin_assignor.c rdkafka_feature.c \
- rdcrc32.c crc32c.c rdaddr.c rdrand.c rdlist.c tinycthread.c \
+ rdcrc32.c crc32c.c rdmurmur2.c rdaddr.c rdrand.c rdlist.c \
+ tinycthread.c tinycthread_extra.c \
rdlog.c rdstring.c rdkafka_event.c rdkafka_metadata.c \
rdregex.c rdports.c rdkafka_metadata_cache.c rdavl.c \
rdkafka_sasl.c rdkafka_sasl_plain.c rdkafka_interceptor.c \
rdkafka_msgset_writer.c rdkafka_msgset_reader.c \
+ rdkafka_header.c rdkafka_admin.c rdkafka_aux.c \
+ rdkafka_background.c \
rdvarint.c rdbuf.c rdunittest.c \
$(SRCS_y)
@@ -63,6 +67,7 @@
printf "$(MKL_RED)FAILED$(MKL_CLR_RESET)\n")
install: lib-install
+uninstall: lib-uninstall
clean: lib-clean
@@ -72,10 +77,10 @@
ifeq ($(WITH_LDS),y)
# Enable linker script if supported by platform
LIB_LDFLAGS+= $(LDFLAG_LINKERSCRIPT)$(LIBNAME).lds
-endif
$(LIBNAME).lds: $(HDRS)
@(printf "$(MKL_YELLOW)Generating linker script $@ from $(HDRS)$(MKL_CLR_RESET)\n" ; \
cat $(HDRS) | ../lds-gen.py > $@)
+endif
-include $(DEPS)
diff -Nru librdkafka-0.11.3/src/rdaddr.c librdkafka-0.11.6/src/rdaddr.c
--- librdkafka-0.11.3/src/rdaddr.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdaddr.c 2018-10-10 06:54:38.000000000 +0000
@@ -131,7 +131,7 @@
if (nodelen) {
/* Truncate nodename if necessary. */
nodelen = RD_MIN(nodelen, sizeof(snode)-1);
- strncpy(snode, nodesvc, nodelen);
+ memcpy(snode, nodesvc, nodelen);
snode[nodelen] = '\0';
}
diff -Nru librdkafka-0.11.3/src/rdaddr.h librdkafka-0.11.6/src/rdaddr.h
--- librdkafka-0.11.3/src/rdaddr.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdaddr.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDADDR_H_
+#define _RDADDR_H_
#ifndef _MSC_VER
#include
@@ -182,3 +183,5 @@
return "af?";
};
}
+
+#endif /* _RDADDR_H_ */
diff -Nru librdkafka-0.11.3/src/rdatomic.h librdkafka-0.11.6/src/rdatomic.h
--- librdkafka-0.11.3/src/rdatomic.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdatomic.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,20 +25,21 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDATOMIC_H_
+#define _RDATOMIC_H_
#include "tinycthread.h"
typedef struct {
int32_t val;
-#ifndef HAVE_ATOMICS_32
+#if !HAVE_ATOMICS_32
mtx_t lock;
#endif
} rd_atomic32_t;
typedef struct {
int64_t val;
-#ifndef HAVE_ATOMICS_64
+#if !HAVE_ATOMICS_64
mtx_t lock;
#endif
} rd_atomic64_t;
@@ -46,7 +47,7 @@
static RD_INLINE RD_UNUSED void rd_atomic32_init (rd_atomic32_t *ra, int32_t v) {
ra->val = v;
-#if !defined(_MSC_VER) && !defined(HAVE_ATOMICS_32)
+#if !defined(_MSC_VER) && !HAVE_ATOMICS_32
mtx_init(&ra->lock, mtx_plain);
#endif
}
@@ -57,7 +58,7 @@
return atomic_add_32_nv(&ra->val, v);
#elif defined(_MSC_VER)
return InterlockedAdd(&ra->val, v);
-#elif !defined(HAVE_ATOMICS_32)
+#elif !HAVE_ATOMICS_32
int32_t r;
mtx_lock(&ra->lock);
ra->val += v;
@@ -74,7 +75,7 @@
return atomic_add_32_nv(&ra->val, -v);
#elif defined(_MSC_VER)
return InterlockedAdd(&ra->val, -v);
-#elif !defined(HAVE_ATOMICS_32)
+#elif !HAVE_ATOMICS_32
int32_t r;
mtx_lock(&ra->lock);
ra->val -= v;
@@ -89,7 +90,7 @@
static RD_INLINE int32_t RD_UNUSED rd_atomic32_get(rd_atomic32_t *ra) {
#if defined(_MSC_VER) || defined(__SUNPRO_C)
return ra->val;
-#elif !defined(HAVE_ATOMICS_32)
+#elif !HAVE_ATOMICS_32
int32_t r;
mtx_lock(&ra->lock);
r = ra->val;
@@ -103,12 +104,18 @@
static RD_INLINE int32_t RD_UNUSED rd_atomic32_set(rd_atomic32_t *ra, int32_t v) {
#ifdef _MSC_VER
return InterlockedExchange(&ra->val, v);
-#elif !defined(HAVE_ATOMICS_32)
+#elif !HAVE_ATOMICS_32
int32_t r;
mtx_lock(&ra->lock);
r = ra->val = v;
mtx_unlock(&ra->lock);
return r;
+#elif HAVE_ATOMICS_32_ATOMIC
+ __atomic_store_n(&ra->val, v, __ATOMIC_SEQ_CST);
+ return v;
+#elif HAVE_ATOMICS_32_SYNC
+ (void)__sync_lock_test_and_set(&ra->val, v);
+ return v;
#else
return ra->val = v; // FIXME
#endif
@@ -118,7 +125,7 @@
static RD_INLINE RD_UNUSED void rd_atomic64_init (rd_atomic64_t *ra, int64_t v) {
ra->val = v;
-#if !defined(_MSC_VER) && !defined(HAVE_ATOMICS_64)
+#if !defined(_MSC_VER) && !HAVE_ATOMICS_64
mtx_init(&ra->lock, mtx_plain);
#endif
}
@@ -128,7 +135,7 @@
return atomic_add_64_nv(&ra->val, v);
#elif defined(_MSC_VER)
return InterlockedAdd64(&ra->val, v);
-#elif !defined(HAVE_ATOMICS_64)
+#elif !HAVE_ATOMICS_64
int64_t r;
mtx_lock(&ra->lock);
ra->val += v;
@@ -145,7 +152,7 @@
return atomic_add_64_nv(&ra->val, -v);
#elif defined(_MSC_VER)
return InterlockedAdd64(&ra->val, -v);
-#elif !defined(HAVE_ATOMICS_64)
+#elif !HAVE_ATOMICS_64
int64_t r;
mtx_lock(&ra->lock);
ra->val -= v;
@@ -160,7 +167,7 @@
static RD_INLINE int64_t RD_UNUSED rd_atomic64_get(rd_atomic64_t *ra) {
#if defined(_MSC_VER) || defined(__SUNPRO_C)
return ra->val;
-#elif !defined(HAVE_ATOMICS_64)
+#elif !HAVE_ATOMICS_64
int64_t r;
mtx_lock(&ra->lock);
r = ra->val;
@@ -175,14 +182,22 @@
static RD_INLINE int64_t RD_UNUSED rd_atomic64_set(rd_atomic64_t *ra, int64_t v) {
#ifdef _MSC_VER
return InterlockedExchange64(&ra->val, v);
-#elif !defined(HAVE_ATOMICS_64)
+#elif !HAVE_ATOMICS_64
int64_t r;
mtx_lock(&ra->lock);
ra->val = v;
r = ra->val;
mtx_unlock(&ra->lock);
return r;
+#elif HAVE_ATOMICS_64_ATOMIC
+ __atomic_store_n(&ra->val, v, __ATOMIC_SEQ_CST);
+ return v;
+#elif HAVE_ATOMICS_64_SYNC
+ (void)__sync_lock_test_and_set(&ra->val, v);
+ return v;
#else
return ra->val = v; // FIXME
#endif
}
+
+#endif /* _RDATOMIC_H_ */
diff -Nru librdkafka-0.11.3/src/rdavg.h librdkafka-0.11.6/src/rdavg.h
--- librdkafka-0.11.3/src/rdavg.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdavg.h 2018-10-10 06:54:38.000000000 +0000
@@ -1,6 +1,11 @@
-#pragma once
+#ifndef _RDAVG_H_
+#define _RDAVG_H_
+#if WITH_HDRHISTOGRAM
+#include "rdhdrhistogram.h"
+#endif
+
typedef struct rd_avg_s {
struct {
int64_t maxv;
@@ -11,24 +16,51 @@
rd_ts_t start;
} ra_v;
mtx_t ra_lock;
+ int ra_enabled;
enum {
RD_AVG_GAUGE,
RD_AVG_COUNTER,
} ra_type;
+#if WITH_HDRHISTOGRAM
+ rd_hdr_histogram_t *ra_hdr;
+#endif
+ /* Histogram results, calculated for dst in rollover().
+ * Will be all zeroes if histograms are not supported. */
+ struct {
+ /* Quantiles */
+ int64_t p50;
+ int64_t p75;
+ int64_t p90;
+ int64_t p95;
+ int64_t p99;
+ int64_t p99_99;
+
+ int64_t oor; /**< Values out of range */
+ int32_t hdrsize; /**< hdr.allocatedSize */
+ double stddev;
+ double mean;
+ } ra_hist;
} rd_avg_t;
/**
- * Add timestamp 'ts' to averager 'ra'.
+ * @brief Add value \p v to averager \p ra.
*/
static RD_UNUSED void rd_avg_add (rd_avg_t *ra, int64_t v) {
mtx_lock(&ra->ra_lock);
+ if (!ra->ra_enabled) {
+ mtx_unlock(&ra->ra_lock);
+ return;
+ }
if (v > ra->ra_v.maxv)
ra->ra_v.maxv = v;
if (ra->ra_v.minv == 0 || v < ra->ra_v.minv)
ra->ra_v.minv = v;
ra->ra_v.sum += v;
ra->ra_v.cnt++;
+#if WITH_HDRHISTOGRAM
+ rd_hdr_histogram_record(ra->ra_hdr, v);
+#endif
mtx_unlock(&ra->ra_lock);
}
@@ -56,18 +88,107 @@
/**
- * Rolls over statistics in 'src' and stores the average in 'dst'.
- * 'src' is cleared and ready to be reused.
+ * @returns the quantile \q for \p ra, or 0 if histograms are not supported
+ * in this build.
+ *
+ * @remark ra will be not locked by this function.
*/
-static RD_UNUSED void rd_avg_rollover (rd_avg_t *dst,
- rd_avg_t *src) {
- rd_ts_t now = rd_clock();
+static RD_UNUSED int64_t
+rd_avg_quantile (const rd_avg_t *ra, double q) {
+#if WITH_HDRHISTOGRAM
+ return rd_hdr_histogram_quantile(ra->ra_hdr, q);
+#else
+ return 0;
+#endif
+}
+
+/**
+ * @brief Rolls over statistics in \p src and stores the average in \p dst.
+ * \p src is cleared and ready to be reused.
+ *
+ * Caller must free avg internal members by calling rd_avg_destroy()
+ * on the \p dst.
+ */
+static RD_UNUSED void rd_avg_rollover (rd_avg_t *dst, rd_avg_t *src) {
+ rd_ts_t now;
mtx_lock(&src->ra_lock);
+ if (!src->ra_enabled) {
+ memset(dst, 0, sizeof(*dst));
+ dst->ra_type = src->ra_type;
+ mtx_unlock(&src->ra_lock);
+ return;
+ }
+
+ mtx_init(&dst->ra_lock, mtx_plain);
dst->ra_type = src->ra_type;
dst->ra_v = src->ra_v;
+#if WITH_HDRHISTOGRAM
+ dst->ra_hdr = NULL;
+
+ dst->ra_hist.stddev = rd_hdr_histogram_stddev(src->ra_hdr);
+ dst->ra_hist.mean = rd_hdr_histogram_mean(src->ra_hdr);
+ dst->ra_hist.oor = src->ra_hdr->outOfRangeCount;
+ dst->ra_hist.hdrsize = src->ra_hdr->allocatedSize;
+ dst->ra_hist.p50 = rd_hdr_histogram_quantile(src->ra_hdr, 50.0);
+ dst->ra_hist.p75 = rd_hdr_histogram_quantile(src->ra_hdr, 75.0);
+ dst->ra_hist.p90 = rd_hdr_histogram_quantile(src->ra_hdr, 90.0);
+ dst->ra_hist.p95 = rd_hdr_histogram_quantile(src->ra_hdr, 95.0);
+ dst->ra_hist.p99 = rd_hdr_histogram_quantile(src->ra_hdr, 99.0);
+ dst->ra_hist.p99_99 = rd_hdr_histogram_quantile(src->ra_hdr, 99.99);
+#else
+ memset(&dst->ra_hist, 0, sizeof(dst->ra_hist));
+#endif
memset(&src->ra_v, 0, sizeof(src->ra_v));
+
+ now = rd_clock();
src->ra_v.start = now;
+
+#if WITH_HDRHISTOGRAM
+ /* Adapt histogram span to fit future out of range entries
+ * from this period. */
+ if (src->ra_hdr->totalCount > 0) {
+ int64_t vmin = src->ra_hdr->lowestTrackableValue;
+ int64_t vmax = src->ra_hdr->highestTrackableValue;
+ int64_t mindiff, maxdiff;
+
+ mindiff = src->ra_hdr->lowestTrackableValue -
+ src->ra_hdr->lowestOutOfRange;
+
+ if (mindiff > 0) {
+ /* There were low out of range values, grow lower
+ * span to fit lowest out of range value + 20%. */
+ vmin = src->ra_hdr->lowestOutOfRange +
+ (int64_t)((double)mindiff * 0.2);
+ }
+
+ maxdiff = src->ra_hdr->highestOutOfRange -
+ src->ra_hdr->highestTrackableValue;
+
+ if (maxdiff > 0) {
+ /* There were high out of range values, grow higher
+ * span to fit highest out of range value + 20%. */
+ vmax = src->ra_hdr->highestOutOfRange +
+ (int64_t)((double)maxdiff * 0.2);
+ }
+
+ if (vmin == src->ra_hdr->lowestTrackableValue &&
+ vmax == src->ra_hdr->highestTrackableValue) {
+ /* No change in min,max, use existing hdr */
+ rd_hdr_histogram_reset(src->ra_hdr);
+
+ } else {
+ int sigfigs = (int)src->ra_hdr->significantFigures;
+ /* Create new hdr for adapted range */
+ rd_hdr_histogram_destroy(src->ra_hdr);
+ src->ra_hdr = rd_hdr_histogram_new(vmin, vmax, sigfigs);
+ }
+
+ } else {
+ /* No records, no need to reset. */
+ }
+#endif
+
mtx_unlock(&src->ra_lock);
rd_avg_calc(dst, now);
@@ -77,19 +198,33 @@
/**
* Initialize an averager
*/
-static RD_UNUSED void rd_avg_init (rd_avg_t *ra, int type) {
- rd_avg_t dummy;
+static RD_UNUSED void rd_avg_init (rd_avg_t *ra, int type,
+ int64_t exp_min, int64_t exp_max,
+ int sigfigs, int enable) {
memset(ra, 0, sizeof(*ra));
mtx_init(&ra->ra_lock, 0);
+ ra->ra_enabled = enable;
+ if (!enable)
+ return;
ra->ra_type = type;
-
- rd_avg_rollover(&dummy, ra);
+ ra->ra_v.start = rd_clock();
+#if WITH_HDRHISTOGRAM
+ /* Start off the histogram with expected min,max span,
+ * we'll adapt the size on each rollover. */
+ ra->ra_hdr = rd_hdr_histogram_new(exp_min, exp_max, sigfigs);
+#endif
}
+
/**
* Destroy averager
*/
static RD_UNUSED void rd_avg_destroy (rd_avg_t *ra) {
+#if WITH_HDRHISTOGRAM
+ if (ra->ra_hdr)
+ rd_hdr_histogram_destroy(ra->ra_hdr);
+#endif
mtx_destroy(&ra->ra_lock);
}
+#endif /* _RDAVG_H_ */
diff -Nru librdkafka-0.11.3/src/rdavl.h librdkafka-0.11.6/src/rdavl.h
--- librdkafka-0.11.3/src/rdavl.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdavl.h 2018-10-10 06:54:38.000000000 +0000
@@ -32,7 +32,8 @@
* Inspired by Ian Piumarta's tree.h implementation.
*/
-#pragma once
+#ifndef _RDAVL_H_
+#define _RDAVL_H_
#include "tinycthread.h"
@@ -251,3 +252,5 @@
return ret;
}
+
+#endif /* _RDAVL_H_ */
diff -Nru librdkafka-0.11.3/src/rdbuf.c librdkafka-0.11.6/src/rdbuf.c
--- librdkafka-0.11.3/src/rdbuf.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdbuf.c 2018-10-10 06:54:38.000000000 +0000
@@ -616,8 +616,11 @@
* destroy_segment() length checks are correct.
* Will decrement rbuf_len et.al. */
for (next = TAILQ_LAST(&rbuf->rbuf_segments, rd_segment_head) ;
- next != seg ; next = TAILQ_PREV(next, rd_segment_head, seg_link))
- rd_buf_destroy_segment(rbuf, next);
+ next != seg ; ) {
+ rd_segment_t *this = next;
+ next = TAILQ_PREV(this, rd_segment_head, seg_link);
+ rd_buf_destroy_segment(rbuf, this);
+ }
/* Update relative write offset */
seg->seg_of = relof;
diff -Nru librdkafka-0.11.3/src/rdendian.h librdkafka-0.11.6/src/rdendian.h
--- librdkafka-0.11.3/src/rdendian.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdendian.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDENDIAN_H_
+#define _RDENDIAN_H_
/**
* Provides portable endian-swapping macros/functions.
@@ -51,11 +52,13 @@
#define be32toh(x) (x)
#define be64toh(x) (x)
#define le64toh(x) __bswap_64 (x)
+ #define le32toh(x) __bswap_32 (x)
#else
#define be16toh(x) __bswap_16 (x)
#define be32toh(x) __bswap_32 (x)
#define be64toh(x) __bswap_64 (x)
#define le64toh(x) (x)
+ #define le32toh(x) (x)
#endif
#endif
@@ -76,9 +79,6 @@
#define le16toh(x) ((uint16_t)BSWAP_16(x))
#define le32toh(x) BSWAP_32(x)
#define le64toh(x) BSWAP_64(x)
-#define htole16(x) ((uint16_t)BSWAP_16(x))
-#define htole32(x) BSWAP_32(x)
-#define htole64(x) BSWAP_64(x)
# else
#define __BYTE_ORDER __LITTLE_ENDIAN
#define be64toh(x) BSWAP_64(x)
@@ -88,7 +88,6 @@
#define le32toh(x) (x)
#define le64toh(x) (x)
#define htole16(x) (x)
-#define htole32(x) (x)
#define htole64(x) (x)
#endif /* sun */
@@ -125,6 +124,11 @@
#define be64toh(x) (x)
#define be32toh(x) (x)
#define be16toh(x) (x)
+#define le32toh(x) \
+ ((((x) & 0xff) << 24) | \
+ (((x) & 0xff00) << 8) | \
+ (((x) & 0xff0000) >> 8) | \
+ (((x) & 0xff000000) >> 24))
#else
#include
@@ -157,3 +161,9 @@
#ifndef htobe16
#define htobe16(x) be16toh(x)
#endif
+
+#ifndef htole32
+#define htole32(x) le32toh(x)
+#endif
+
+#endif /* _RDENDIAN_H_ */
diff -Nru librdkafka-0.11.3/src/rdfloat.h librdkafka-0.11.6/src/rdfloat.h
--- librdkafka-0.11.3/src/rdfloat.h 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdfloat.h 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,67 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2018, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+
+/**
+ * rd_dbl_eq0(a,b,prec)
+ * Check two doubles for equality with the specified precision.
+ * Use this instead of != and == for all floats/doubles.
+ * More info:
+ * http://docs.sun.com/source/806-3568/ncg_goldberg.html
+ */
+static RD_INLINE RD_UNUSED
+int rd_dbl_eq0 (double a, double b, double prec) {
+ return fabs(a - b) < prec;
+}
+
+/* A default 'good' double-equality precision value.
+ * This rather timid epsilon value is useful for tenths, hundreths,
+ * and thousands parts, but not anything more precis than that.
+ * If a higher precision is needed, use dbl_eq0 and dbl_eq0 directly
+ * and specify your own precision. */
+#define RD_DBL_EPSILON 0.00001
+
+/**
+ * rd_dbl_eq(a,b)
+ * Same as rd_dbl_eq0() above but with a predefined 'good' precision.
+ */
+#define rd_dbl_eq(a,b) rd_dbl_eq0(a,b,RD_DBL_EPSILON)
+
+/**
+ * rd_dbl_ne(a,b)
+ * Same as rd_dbl_eq() above but with reversed logic: not-equal.
+ */
+#define rd_dbl_ne(a,b) (!rd_dbl_eq0(a,b,RD_DBL_EPSILON))
+
+/**
+ * rd_dbl_zero(a)
+ * Checks if the double `a' is zero (or close enough).
+ */
+#define rd_dbl_zero(a) rd_dbl_eq0(a,0.0,RD_DBL_EPSILON)
diff -Nru librdkafka-0.11.3/src/rdgz.h librdkafka-0.11.6/src/rdgz.h
--- librdkafka-0.11.3/src/rdgz.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdgz.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDGZ_H_
+#define _RDGZ_H_
/**
* Simple gzip decompression returning the inflated data
@@ -40,3 +41,5 @@
*/
void *rd_gz_decompress (const void *compressed, int compressed_len,
uint64_t *decompressed_lenp);
+
+#endif /* _RDGZ_H_ */
diff -Nru librdkafka-0.11.3/src/rd.h librdkafka-0.11.6/src/rd.h
--- librdkafka-0.11.3/src/rd.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rd.h 2018-10-10 06:54:38.000000000 +0000
@@ -27,8 +27,8 @@
*/
-
-#pragma once
+#ifndef _RD_H_
+#define _RD_H_
#ifndef _MSC_VER
#ifndef _GNU_SOURCE
@@ -80,7 +80,7 @@
/** Assert if reached */
-#define RD_NOTREACHED() rd_kafka_assert(NULL, !*"/* NOTREACHED */ violated")
+#define RD_NOTREACHED() rd_assert(!*"/* NOTREACHED */ violated")
@@ -144,14 +144,24 @@
#ifdef strndupa
#define rd_strndupa(DESTPTR,PTR,LEN) (*(DESTPTR) = strndupa(PTR,LEN))
#else
-#define rd_strndupa(DESTPTR,PTR,LEN) (*(DESTPTR) = rd_alloca(LEN+1), \
- memcpy(*(DESTPTR), (PTR), LEN), *((*(DESTPTR))+(LEN)) = 0)
+#define rd_strndupa(DESTPTR,PTR,LEN) do { \
+ const char *_src = (PTR); \
+ size_t _srclen = (LEN); \
+ char *_dst = rd_alloca(_srclen + 1); \
+ memcpy(_dst, _src, _srclen); \
+ _dst[_srclen] = '\0'; \
+ *(DESTPTR) = _dst; \
+ } while (0)
#endif
#ifdef strdupa
#define rd_strdupa(DESTPTR,PTR) (*(DESTPTR) = strdupa(PTR))
#else
-#define rd_strdupa(DESTPTR,PTR) rd_strndupa(DESTPTR,PTR,strlen(PTR))
+#define rd_strdupa(DESTPTR,PTR) do { \
+ const char *_src1 = (PTR); \
+ size_t _srclen1 = strlen(_src1); \
+ rd_strndupa(DESTPTR, _src1, _srclen1); \
+ } while (0)
#endif
#ifndef IOV_MAX
@@ -220,8 +230,8 @@
/**
* Generic refcnt interface
*/
-#ifndef _MSC_VER
-/* Mutexes (critical sections) are slow, even when uncontended, on Windows */
+
+#if !HAVE_ATOMICS_32
#define RD_REFCNT_USE_LOCKS 1
#endif
@@ -453,3 +463,5 @@
char *ptr;
size_t size;
} rd_chariov_t;
+
+#endif /* _RD_H_ */
diff -Nru librdkafka-0.11.3/src/rdhdrhistogram.c librdkafka-0.11.6/src/rdhdrhistogram.c
--- librdkafka-0.11.3/src/rdhdrhistogram.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdhdrhistogram.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,729 @@
+/*
+ * This license covers this C port of
+ * Coda Hale's Golang HdrHistogram https://github.com/codahale/hdrhistogram
+ * at revision 3a0bb77429bd3a61596f5e8a3172445844342120
+ *
+ * ----------------------------------------------------------------------------
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2014 Coda Hale
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+*/
+
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * Minimal C Hdr_Histogram based on Coda Hale's Golang implementation.
+ * https://github.com/codahale/hdr_histogram
+ *
+ *
+ * A Histogram is a lossy data structure used to record the distribution of
+ * non-normally distributed data (like latency) with a high degree of accuracy
+ * and a bounded degree of precision.
+ *
+ *
+ */
+
+#include "rd.h"
+
+#include
+#include
+#include
+
+#include "rdhdrhistogram.h"
+#include "rdunittest.h"
+#include "rdfloat.h"
+
+void rd_hdr_histogram_destroy (rd_hdr_histogram_t *hdr) {
+ free(hdr);
+}
+
+rd_hdr_histogram_t *rd_hdr_histogram_new (int64_t minValue, int64_t maxValue,
+ int significantFigures) {
+ rd_hdr_histogram_t *hdr;
+ int64_t largestValueWithSingleUnitResolution;
+ int32_t subBucketCountMagnitude;
+ int32_t subBucketHalfCountMagnitude;
+ int32_t unitMagnitude;
+ int32_t subBucketCount;
+ int32_t subBucketHalfCount;
+ int64_t subBucketMask;
+ int64_t smallestUntrackableValue;
+ int32_t bucketsNeeded = 1;
+ int32_t bucketCount;
+ int32_t countsLen;
+
+ if (significantFigures < 1 || significantFigures > 5)
+ return NULL;
+
+ largestValueWithSingleUnitResolution =
+ (int64_t)(2.0 * pow(10.0, (double)significantFigures));
+
+ subBucketCountMagnitude =
+ (int32_t)ceil(
+ log2((double)largestValueWithSingleUnitResolution));
+
+ subBucketHalfCountMagnitude = RD_MAX(subBucketCountMagnitude, 1) - 1;
+
+ unitMagnitude = RD_MAX(floor(log2((double)minValue)), 0);
+
+ subBucketCount = (int32_t)pow(2,
+ (double)subBucketHalfCountMagnitude+1.0);
+
+ subBucketHalfCount = subBucketCount / 2;
+
+ subBucketMask = (int64_t)(subBucketCount-1) << unitMagnitude;
+
+ /* Determine exponent range needed to support the trackable
+ * value with no overflow: */
+ smallestUntrackableValue = (int64_t)subBucketCount << unitMagnitude;
+ while (smallestUntrackableValue < maxValue) {
+ smallestUntrackableValue <<= 1;
+ bucketsNeeded++;
+ }
+
+ bucketCount = bucketsNeeded;
+ countsLen = (bucketCount + 1) * (subBucketCount / 2);
+ hdr = calloc(1, sizeof(*hdr) + (sizeof(*hdr->counts) * countsLen));
+ hdr->counts = (int64_t *)(hdr+1);
+ hdr->allocatedSize = sizeof(*hdr) + (sizeof(*hdr->counts) * countsLen);
+
+ hdr->lowestTrackableValue = minValue;
+ hdr->highestTrackableValue = maxValue;
+ hdr->unitMagnitude = unitMagnitude;
+ hdr->significantFigures = significantFigures;
+ hdr->subBucketHalfCountMagnitude = subBucketHalfCountMagnitude;
+ hdr->subBucketHalfCount = subBucketHalfCount;
+ hdr->subBucketMask = subBucketMask;
+ hdr->subBucketCount = subBucketCount;
+ hdr->bucketCount = bucketCount;
+ hdr->countsLen = countsLen;
+ hdr->totalCount = 0;
+ hdr->lowestOutOfRange = minValue;
+ hdr->highestOutOfRange = maxValue;
+
+ return hdr;
+}
+
+/**
+ * @brief Deletes all recorded values and resets histogram.
+ */
+void rd_hdr_histogram_reset (rd_hdr_histogram_t *hdr) {
+ int32_t i;
+ hdr->totalCount = 0;
+ for (i = 0 ; i < hdr->countsLen ; i++)
+ hdr->counts[i] = 0;
+}
+
+
+
+static int32_t
+rd_hdr_countsIndex (const rd_hdr_histogram_t *hdr,
+ int32_t bucketIdx, int32_t subBucketIdx) {
+ int32_t bucketBaseIdx = (bucketIdx + 1) <<
+ hdr->subBucketHalfCountMagnitude;
+ int32_t offsetInBucket = subBucketIdx - hdr->subBucketHalfCount;
+ return bucketBaseIdx + offsetInBucket;
+}
+
+static __inline int64_t
+rd_hdr_getCountAtIndex (const rd_hdr_histogram_t *hdr,
+ int32_t bucketIdx, int32_t subBucketIdx) {
+ return hdr->counts[rd_hdr_countsIndex(hdr, bucketIdx, subBucketIdx)];
+}
+
+
+static __inline int64_t bitLen (int64_t x) {
+ int64_t n = 0;
+ for (; x >= 0x8000; x >>= 16)
+ n += 16;
+ if (x >= 0x80) {
+ x >>= 8;
+ n += 8;
+ }
+ if (x >= 0x8) {
+ x >>= 4;
+ n += 4;
+ }
+ if (x >= 0x2) {
+ x >>= 2;
+ n += 2;
+ }
+ if (x >= 0x1)
+ n++;
+ return n;
+}
+
+
+static __inline int32_t
+rd_hdr_getBucketIndex (const rd_hdr_histogram_t *hdr, int64_t v) {
+ int64_t pow2Ceiling = bitLen(v | hdr->subBucketMask);
+ return (int32_t)(pow2Ceiling - (int64_t)hdr->unitMagnitude -
+ (int64_t)(hdr->subBucketHalfCountMagnitude+1));
+}
+
+static __inline int32_t
+rd_hdr_getSubBucketIdx (const rd_hdr_histogram_t *hdr, int64_t v, int32_t idx) {
+ return (int32_t)(v >> ((int64_t)idx + (int64_t)hdr->unitMagnitude));
+}
+
+static __inline int64_t
+rd_hdr_valueFromIndex (const rd_hdr_histogram_t *hdr,
+ int32_t bucketIdx, int32_t subBucketIdx) {
+ return (int64_t)subBucketIdx <<
+ ((int64_t)bucketIdx + hdr->unitMagnitude);
+}
+
+static __inline int64_t
+rd_hdr_sizeOfEquivalentValueRange (const rd_hdr_histogram_t *hdr, int64_t v) {
+ int32_t bucketIdx = rd_hdr_getBucketIndex(hdr, v);
+ int32_t subBucketIdx = rd_hdr_getSubBucketIdx(hdr, v, bucketIdx);
+ int32_t adjustedBucket = bucketIdx;
+ if (subBucketIdx >= hdr->subBucketCount)
+ adjustedBucket++;
+ return (int64_t)1 << (hdr->unitMagnitude + (int64_t)adjustedBucket);
+}
+
+static __inline int64_t
+rd_hdr_lowestEquivalentValue (const rd_hdr_histogram_t *hdr, int64_t v) {
+ int32_t bucketIdx = rd_hdr_getBucketIndex(hdr, v);
+ int32_t subBucketIdx = rd_hdr_getSubBucketIdx(hdr, v, bucketIdx);
+ return rd_hdr_valueFromIndex(hdr, bucketIdx, subBucketIdx);
+}
+
+
+static __inline int64_t
+rd_hdr_nextNonEquivalentValue (const rd_hdr_histogram_t *hdr, int64_t v) {
+ return rd_hdr_lowestEquivalentValue(hdr, v) +
+ rd_hdr_sizeOfEquivalentValueRange(hdr, v);
+}
+
+
+static __inline int64_t
+rd_hdr_highestEquivalentValue (const rd_hdr_histogram_t *hdr, int64_t v) {
+ return rd_hdr_nextNonEquivalentValue(hdr, v) - 1;
+}
+
+static __inline int64_t
+rd_hdr_medianEquivalentValue (const rd_hdr_histogram_t *hdr, int64_t v) {
+ return rd_hdr_lowestEquivalentValue(hdr, v) +
+ (rd_hdr_sizeOfEquivalentValueRange(hdr, v) >> 1);
+}
+
+
+static __inline int32_t
+rd_hdr_countsIndexFor (const rd_hdr_histogram_t *hdr, int64_t v) {
+ int32_t bucketIdx = rd_hdr_getBucketIndex(hdr, v);
+ int32_t subBucketIdx = rd_hdr_getSubBucketIdx(hdr, v, bucketIdx);
+ return rd_hdr_countsIndex(hdr, bucketIdx, subBucketIdx);
+}
+
+
+
+typedef struct rd_hdr_iter_s {
+ const rd_hdr_histogram_t *hdr;
+ int bucketIdx;
+ int subBucketIdx;
+ int64_t countAtIdx;
+ int64_t countToIdx;
+ int64_t valueFromIdx;
+ int64_t highestEquivalentValue;
+} rd_hdr_iter_t;
+
+#define RD_HDR_ITER_INIT(hdr) { .hdr = hdr, .subBucketIdx = -1 }
+
+static int rd_hdr_iter_next (rd_hdr_iter_t *it) {
+ const rd_hdr_histogram_t *hdr = it->hdr;
+
+ if (it->countToIdx >= hdr->totalCount)
+ return 0;
+
+ it->subBucketIdx++;
+ if (it->subBucketIdx >= hdr->subBucketCount) {
+ it->subBucketIdx = hdr->subBucketHalfCount;
+ it->bucketIdx++;
+ }
+
+ if (it->bucketIdx >= hdr->bucketCount)
+ return 0;
+
+ it->countAtIdx = rd_hdr_getCountAtIndex(hdr,
+ it->bucketIdx,
+ it->subBucketIdx);
+ it->countToIdx += it->countAtIdx;
+ it->valueFromIdx = rd_hdr_valueFromIndex(hdr,
+ it->bucketIdx,
+ it->subBucketIdx);
+ it->highestEquivalentValue =
+ rd_hdr_highestEquivalentValue(hdr, it->valueFromIdx);
+
+ return 1;
+}
+
+
+double rd_hdr_histogram_stddev (rd_hdr_histogram_t *hdr) {
+ double mean;
+ double geometricDevTotal = 0.0;
+ rd_hdr_iter_t it = RD_HDR_ITER_INIT(hdr);
+
+ if (hdr->totalCount == 0)
+ return 0;
+
+ mean = rd_hdr_histogram_mean(hdr);
+
+
+ while (rd_hdr_iter_next(&it)) {
+ double dev;
+
+ if (it.countAtIdx == 0)
+ continue;
+
+ dev = (double)rd_hdr_medianEquivalentValue(
+ hdr, it.valueFromIdx) - mean;
+ geometricDevTotal += (dev * dev) * (double)it.countAtIdx;
+ }
+
+ return sqrt(geometricDevTotal / (double)hdr->totalCount);
+}
+
+
+/**
+ * @returns the approximate maximum recorded value.
+ */
+int64_t rd_hdr_histogram_max (const rd_hdr_histogram_t *hdr) {
+ int64_t vmax = 0;
+ rd_hdr_iter_t it = RD_HDR_ITER_INIT(hdr);
+
+ while (rd_hdr_iter_next(&it)) {
+ if (it.countAtIdx != 0)
+ vmax = it.highestEquivalentValue;
+ }
+ return rd_hdr_highestEquivalentValue(hdr, vmax);
+}
+
+/**
+ * @returns the approximate minimum recorded value.
+ */
+int64_t rd_hdr_histogram_min (const rd_hdr_histogram_t *hdr) {
+ int64_t vmin = 0;
+ rd_hdr_iter_t it = RD_HDR_ITER_INIT(hdr);
+
+ while (rd_hdr_iter_next(&it)) {
+ if (it.countAtIdx != 0 && vmin == 0) {
+ vmin = it.highestEquivalentValue;
+ break;
+ }
+ }
+ return rd_hdr_lowestEquivalentValue(hdr, vmin);
+}
+
+/**
+ * @returns the approximate arithmetic mean of the recorded values.
+ */
+double rd_hdr_histogram_mean (const rd_hdr_histogram_t *hdr) {
+ int64_t total = 0;
+ rd_hdr_iter_t it = RD_HDR_ITER_INIT(hdr);
+
+ if (hdr->totalCount == 0)
+ return 0.0;
+
+ while (rd_hdr_iter_next(&it)) {
+ if (it.countAtIdx != 0)
+ total += it.countAtIdx *
+ rd_hdr_medianEquivalentValue(hdr,
+ it.valueFromIdx);
+ }
+ return (double)total / (double)hdr->totalCount;
+}
+
+
+
+/**
+ * @brief Records the given value.
+ *
+ * @returns 1 if value was recorded or 0 if value is out of range.
+ */
+
+int rd_hdr_histogram_record (rd_hdr_histogram_t *hdr, int64_t v) {
+ int32_t idx = rd_hdr_countsIndexFor(hdr, v);
+
+ if (idx < 0 || hdr->countsLen <= idx) {
+ hdr->outOfRangeCount++;
+ if (v > hdr->highestOutOfRange)
+ hdr->highestOutOfRange = v;
+ if (v < hdr->lowestOutOfRange)
+ hdr->lowestOutOfRange = v;
+ return 0;
+ }
+
+ hdr->counts[idx]++;
+ hdr->totalCount++;
+
+ return 1;
+}
+
+
+/**
+ * @returns the recorded value at the given quantile (0..100).
+ */
+int64_t rd_hdr_histogram_quantile (const rd_hdr_histogram_t *hdr, double q) {
+ int64_t total = 0;
+ int64_t countAtPercentile;
+ rd_hdr_iter_t it = RD_HDR_ITER_INIT(hdr);
+
+ if (q > 100.0)
+ q = 100.0;
+
+ countAtPercentile =
+ (int64_t)(((q / 100.0) * (double)hdr->totalCount) + 0.5);
+
+ while (rd_hdr_iter_next(&it)) {
+ total += it.countAtIdx;
+ if (total >= countAtPercentile)
+ return rd_hdr_highestEquivalentValue(
+ hdr, it.valueFromIdx);
+ }
+
+ return 0;
+}
+
+
+
+/**
+ * @name Unit tests
+ * @{
+ *
+ *
+ *
+ */
+
+/**
+ * @returns 0 on success or 1 on failure.
+ */
+static int ut_high_sigfig (void) {
+ rd_hdr_histogram_t *hdr;
+ const int64_t input[] = {
+ 459876, 669187, 711612, 816326, 931423,
+ 1033197, 1131895, 2477317, 3964974, 12718782,
+ };
+ size_t i;
+ int64_t v;
+ const int64_t exp = 1048575;
+
+ hdr = rd_hdr_histogram_new(459876, 12718782, 5);
+ for (i = 0 ; i < RD_ARRAYSIZE(input) ; i++) {
+ /* Ignore errors (some should fail) */
+ rd_hdr_histogram_record(hdr, input[i]);
+ }
+
+ v = rd_hdr_histogram_quantile(hdr, 50);
+ RD_UT_ASSERT(v == exp, "Median is %"PRId64", expected %"PRId64,
+ v, exp);
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+static int ut_quantile (void) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(1, 10000000, 3);
+ size_t i;
+ const struct {
+ double q;
+ int64_t v;
+ } exp[] = {
+ { 50, 500223 },
+ { 75, 750079 },
+ { 90, 900095 },
+ { 95, 950271 },
+ { 99, 990207 },
+ { 99.9, 999423 },
+ { 99.99, 999935 },
+ };
+
+ for (i = 0 ; i < 1000000 ; i++) {
+ int r = rd_hdr_histogram_record(hdr, (int64_t)i);
+ RD_UT_ASSERT(r, "record(%"PRId64") failed\n", (int64_t)i);
+ }
+
+ for (i = 0 ; i < RD_ARRAYSIZE(exp) ; i++) {
+ int64_t v = rd_hdr_histogram_quantile(hdr, exp[i].q);
+ RD_UT_ASSERT(v == exp[i].v,
+ "P%.2f is %"PRId64", expected %"PRId64,
+ exp[i].q, v, exp[i].v);
+ }
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+static int ut_mean (void) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(1, 10000000, 3);
+ size_t i;
+ const double exp = 500000.013312;
+ double v;
+
+ for (i = 0 ; i < 1000000 ; i++) {
+ int r = rd_hdr_histogram_record(hdr, (int64_t)i);
+ RD_UT_ASSERT(r, "record(%"PRId64") failed\n", (int64_t)i);
+ }
+
+ v = rd_hdr_histogram_mean(hdr);
+ RD_UT_ASSERT(rd_dbl_eq0(v, exp, 0.0000001),
+ "Mean is %f, expected %f", v, exp);
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+
+static int ut_stddev (void) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(1, 10000000, 3);
+ size_t i;
+ const double exp = 288675.140368;
+ const double epsilon = 0.000001;
+ double v;
+
+ for (i = 0 ; i < 1000000 ; i++) {
+ int r = rd_hdr_histogram_record(hdr, (int64_t)i);
+ RD_UT_ASSERT(r, "record(%"PRId64") failed\n", (int64_t)i);
+ }
+
+ v = rd_hdr_histogram_stddev(hdr);
+ RD_UT_ASSERT(rd_dbl_eq0(v, exp, epsilon),
+ "StdDev is %.6f, expected %.6f: diff %.6f vs epsilon %.6f",
+ v, exp, fabs(v - exp), epsilon);
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+static int ut_totalcount (void) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(1, 10000000, 3);
+ int64_t i;
+
+ for (i = 0 ; i < 1000000 ; i++) {
+ int64_t v;
+ int r = rd_hdr_histogram_record(hdr, i);
+ RD_UT_ASSERT(r, "record(%"PRId64") failed\n", i);
+
+ v = hdr->totalCount;
+ RD_UT_ASSERT(v == i+1,
+ "total_count is %"PRId64", expected %"PRId64,
+ v, i+1);
+ }
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+
+static int ut_max (void) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(1, 10000000, 3);
+ int64_t i, v;
+ const int64_t exp = 1000447;
+
+ for (i = 0 ; i < 1000000 ; i++) {
+ int r = rd_hdr_histogram_record(hdr, i);
+ RD_UT_ASSERT(r, "record(%"PRId64") failed\n", i);
+ }
+
+ v = rd_hdr_histogram_max(hdr);
+ RD_UT_ASSERT(v == exp,
+ "Max is %"PRId64", expected %"PRId64, v, exp);
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+static int ut_min (void) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(1, 10000000, 3);
+ int64_t i, v;
+ const int64_t exp = 0;
+
+ for (i = 0 ; i < 1000000 ; i++) {
+ int r = rd_hdr_histogram_record(hdr, i);
+ RD_UT_ASSERT(r, "record(%"PRId64") failed\n", i);
+ }
+
+ v = rd_hdr_histogram_min(hdr);
+ RD_UT_ASSERT(v == exp,
+ "Min is %"PRId64", expected %"PRId64, v, exp);
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+static int ut_reset (void) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(1, 10000000, 3);
+ int64_t i, v;
+ const int64_t exp = 0;
+
+ for (i = 0 ; i < 1000000 ; i++) {
+ int r = rd_hdr_histogram_record(hdr, i);
+ RD_UT_ASSERT(r, "record(%"PRId64") failed\n", i);
+ }
+
+ rd_hdr_histogram_reset(hdr);
+
+ v = rd_hdr_histogram_max(hdr);
+ RD_UT_ASSERT(v == exp,
+ "Max is %"PRId64", expected %"PRId64, v, exp);
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+
+static int ut_nan (void) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(1, 100000, 3);
+ double v;
+
+ v = rd_hdr_histogram_mean(hdr);
+ RD_UT_ASSERT(!isnan(v), "Mean is %f, expected NaN", v);
+ v = rd_hdr_histogram_stddev(hdr);
+ RD_UT_ASSERT(!isnan(v), "StdDev is %f, expected NaN", v);
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+
+static int ut_sigfigs (void) {
+ int sigfigs;
+
+ for (sigfigs = 1 ; sigfigs <= 5 ; sigfigs++) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(1, 10, sigfigs);
+ RD_UT_ASSERT(hdr->significantFigures == sigfigs,
+ "Significant figures is %"PRId64", expected %d",
+ hdr->significantFigures, sigfigs);
+ rd_hdr_histogram_destroy(hdr);
+ }
+
+ RD_UT_PASS();
+}
+
+static int ut_minmax_trackable (void) {
+ const int64_t minval = 2;
+ const int64_t maxval = 11;
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(minval, maxval, 3);
+
+ RD_UT_ASSERT(hdr->lowestTrackableValue == minval,
+ "lowestTrackableValue is %"PRId64", expected %"PRId64,
+ hdr->lowestTrackableValue, minval);
+ RD_UT_ASSERT(hdr->highestTrackableValue == maxval,
+ "highestTrackableValue is %"PRId64", expected %"PRId64,
+ hdr->highestTrackableValue, maxval);
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+
+static int ut_unitmagnitude_overflow (void) {
+ rd_hdr_histogram_t *hdr = rd_hdr_histogram_new(0, 200, 4);
+ int r = rd_hdr_histogram_record(hdr, 11);
+ RD_UT_ASSERT(r, "record(11) failed\n");
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+static int ut_subbucketmask_overflow (void) {
+ rd_hdr_histogram_t *hdr;
+ const int64_t input[] = { (int64_t)1e8, (int64_t)2e7, (int64_t)3e7 };
+ const struct {
+ double q;
+ int64_t v;
+ } exp[] = {
+ { 50, 33554431 },
+ { 83.33, 33554431 },
+ { 83.34, 100663295 },
+ { 99, 100663295 },
+ };
+ size_t i;
+
+ hdr = rd_hdr_histogram_new((int64_t)2e7, (int64_t)1e8, 5);
+
+ for (i = 0 ; i < RD_ARRAYSIZE(input) ; i++) {
+ /* Ignore errors (some should fail) */
+ int r = rd_hdr_histogram_record(hdr, input[i]);
+ RD_UT_ASSERT(r, "record(%"PRId64") failed\n", input[i]);
+ }
+
+ for (i = 0 ; i < RD_ARRAYSIZE(exp) ; i++) {
+ int64_t v = rd_hdr_histogram_quantile(hdr, exp[i].q);
+ RD_UT_ASSERT(v == exp[i].v,
+ "P%.2f is %"PRId64", expected %"PRId64,
+ exp[i].q, v, exp[i].v);
+ }
+
+ rd_hdr_histogram_destroy(hdr);
+ RD_UT_PASS();
+}
+
+
+int unittest_rdhdrhistogram (void) {
+ int fails = 0;
+
+ fails += ut_high_sigfig();
+ fails += ut_quantile();
+ fails += ut_mean();
+ fails += ut_stddev();
+ fails += ut_totalcount();
+ fails += ut_max();
+ fails += ut_min();
+ fails += ut_reset();
+ fails += ut_nan();
+ fails += ut_sigfigs();
+ fails += ut_minmax_trackable();
+ fails += ut_unitmagnitude_overflow();
+ fails += ut_subbucketmask_overflow();
+
+ return fails;
+}
+
+/**@}*/
diff -Nru librdkafka-0.11.3/src/rdhdrhistogram.h librdkafka-0.11.6/src/rdhdrhistogram.h
--- librdkafka-0.11.3/src/rdhdrhistogram.h 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdhdrhistogram.h 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,86 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef _RDHDR_HISTOGRAM_H_
+#define _RDHDR_HISTOGRAM_H_
+
+#include
+
+
+typedef struct rd_hdr_histogram_s {
+ int64_t lowestTrackableValue;
+ int64_t highestTrackableValue;
+ int64_t unitMagnitude;
+ int64_t significantFigures;
+ int32_t subBucketHalfCountMagnitude;
+ int32_t subBucketHalfCount;
+ int64_t subBucketMask;
+ int32_t subBucketCount;
+ int32_t bucketCount;
+ int32_t countsLen;
+ int64_t totalCount;
+ int64_t *counts;
+ int64_t outOfRangeCount; /**< Number of rejected records due to
+ * value being out of range. */
+ int64_t lowestOutOfRange; /**< Lowest value that was out of range.
+ * Initialized to lowestTrackableValue */
+ int64_t highestOutOfRange; /**< Highest value that was out of range.
+ * Initialized to highestTrackableValue */
+ int32_t allocatedSize; /**< Allocated size of histogram, for
+ * sigfigs tuning. */
+} rd_hdr_histogram_t;
+
+
+#endif /* !_RDHDR_HISTOGRAM_H_ */
+
+
+void rd_hdr_histogram_destroy (rd_hdr_histogram_t *hdr);
+
+/**
+ * @brief Create a new Hdr_Histogram.
+ *
+ * @param significant_figures must be between 1..5
+ *
+ * @returns a newly allocated histogram, or NULL on error.
+ *
+ * @sa rd_hdr_histogram_destroy()
+ */
+rd_hdr_histogram_t *rd_hdr_histogram_new (int64_t minValue, int64_t maxValue,
+ int significantFigures);
+
+void rd_hdr_histogram_reset (rd_hdr_histogram_t *hdr);
+
+int rd_hdr_histogram_record (rd_hdr_histogram_t *hdr, int64_t v);
+
+double rd_hdr_histogram_stddev (rd_hdr_histogram_t *hdr);
+double rd_hdr_histogram_mean (const rd_hdr_histogram_t *hdr);
+int64_t rd_hdr_histogram_max (const rd_hdr_histogram_t *hdr);
+int64_t rd_hdr_histogram_min (const rd_hdr_histogram_t *hdr);
+int64_t rd_hdr_histogram_quantile (const rd_hdr_histogram_t *hdr, double q);
+
+
+int unittest_rdhdrhistogram (void);
diff -Nru librdkafka-0.11.3/src/rdinterval.h librdkafka-0.11.6/src/rdinterval.h
--- librdkafka-0.11.3/src/rdinterval.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdinterval.h 2018-10-10 06:54:38.000000000 +0000
@@ -1,6 +1,5 @@
-#pragma once
-
-
+#ifndef _RDINTERVAL_H_
+#define _RDINTERVAL_H_
#include "rd.h"
@@ -114,3 +113,5 @@
static RD_INLINE RD_UNUSED int rd_interval_disabled (const rd_interval_t *ri) {
return ri->ri_ts_last == 6000000000000000000LL;
}
+
+#endif /* _RDINTERVAL_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_admin.c librdkafka-0.11.6/src/rdkafka_admin.c
--- librdkafka-0.11.3/src/rdkafka_admin.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_admin.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,2740 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "rdkafka_int.h"
+#include "rdkafka_admin.h"
+#include "rdkafka_request.h"
+#include "rdkafka_aux.h"
+
+#include
+
+
+
+/** @brief Descriptive strings for rko_u.admin_request.state */
+static const char *rd_kafka_admin_state_desc[] = {
+ "initializing",
+ "waiting for broker",
+ "waiting for controller",
+ "constructing request",
+ "waiting for response from broker",
+};
+
+
+
+/**
+ * @brief Admin API implementation.
+ *
+ * The public Admin API in librdkafka exposes a completely asynchronous
+ * interface where the initial request API (e.g., ..CreateTopics())
+ * is non-blocking and returns immediately, and the application polls
+ * a ..queue_t for the result.
+ *
+ * The underlying handling of the request is also completely asynchronous
+ * inside librdkafka, for two reasons:
+ * - everything is async in librdkafka so adding something new that isn't
+ * would mean that existing functionality will need to be changed if
+ * it should be able to work simultaneously (such as statistics, timers,
+ * etc). There is no functional value to making the admin API
+ * synchronous internally, even if it would simplify its implementation.
+ * So making it async allows the Admin API to be used with existing
+ * client types in existing applications without breakage.
+ * - the async approach allows multiple outstanding Admin API requests
+ * simultaneously.
+ *
+ * The internal async implementation relies on the following concepts:
+ * - it uses a single rko (rd_kafka_op_t) to maintain state.
+ * - the rko has a callback attached - called the worker callback.
+ * - the worker callback is a small state machine that triggers
+ * async operations (be it controller lookups, timeout timers,
+ * protocol transmits, etc).
+ * - the worker callback is only called on the rdkafka main thread.
+ * - the callback is triggered by different events and sources by enqueuing
+ * the rko on the rdkafka main ops queue.
+ *
+ *
+ * Let's illustrate this with a DeleteTopics example. This might look
+ * daunting, but it boils down to an asynchronous state machine being
+ * triggered by enqueuing the rko op.
+ *
+ * 1. [app thread] The user constructs the input arguments,
+ * including a response rkqu queue and then calls DeleteTopics().
+ *
+ * 2. [app thread] DeleteTopics() creates a new internal op (rko) of type
+ * RD_KAFKA_OP_DELETETOPICS, makes a **copy** on the rko of all the
+ * input arguments (which allows the caller to free the originals
+ * whenever she likes). The rko op worker callback is set to the
+ * generic admin worker callback rd_kafka_admin_worker()
+ *
+ * 3. [app thread] DeleteTopics() enqueues the rko on librdkafka's main ops
+ * queue that is served by the rdkafka main thread in rd_kafka_thread_main()
+ *
+ * 4. [rdkafka main thread] The rko is dequeued by rd_kafka_q_serve and
+ * the rd_kafka_poll_cb() is called.
+ *
+ * 5. [rdkafka main thread] The rko_type switch case identifies the rko
+ * as an RD_KAFKA_OP_DELETETOPICS which is served by the op callback
+ * set in step 2.
+ *
+ * 6. [rdkafka main thread] The worker callback is called.
+ * After some initial checking of err==ERR__DESTROY events
+ * (which is used to clean up outstanding ops (etc) on termination),
+ * the code hits a state machine using rko_u.admin.request_state.
+ *
+ * 7. [rdkafka main thread] The initial state is RD_KAFKA_ADMIN_STATE_INIT
+ * where the worker validates the user input.
+ * An enqueue once (eonce) object is created - the use of this object
+ * allows having multiple outstanding async functions referencing the
+ * same underlying rko object, but only allowing the first one
+ * to trigger an event.
+ * A timeout timer is set up to trigger the eonce object when the
+ * full options.request_timeout has elapsed.
+ *
+ * 8. [rdkafka main thread] After initialization the state is updated
+ * to WAIT_BROKER or WAIT_CONTROLLER and the code falls through to
+ * looking up a specific broker or the controller broker and waiting for
+ * an active connection.
+ * Both the lookup and the waiting for an active connection are
+ * fully asynchronous, and the same eonce used for the timer is passed
+ * to the rd_kafka_broker_controller_async() or broker_async() functions
+ * which will trigger the eonce when a broker state change occurs.
+ * If the controller is already known (from metadata) and the connection
+ * is up a rkb broker object is returned and the eonce is not used,
+ * skip to step 11.
+ *
+ * 9. [rdkafka main thread] Upon metadata retrieval (which is triggered
+ * automatically by other parts of the code) the controller_id may be
+ * updated in which case the eonce is triggered.
+ * The eonce triggering enqueues the original rko on the rdkafka main
+ * ops queue again and we go to step 8 which will check if the controller
+ * connection is up.
+ *
+ * 10. [broker thread] If the controller_id is now known we wait for
+ * the corresponding broker's connection to come up. This signaling
+ * is performed from the broker thread upon broker state changes
+ * and uses the same eonce. The eonce triggering enqueues the original
+ * rko on the rdkafka main ops queue again we go to back to step 8
+ * to check if broker is now available.
+ *
+ * 11. [rdkafka main thread] Back in the worker callback we now have an
+ * rkb broker pointer (with reference count increased) for the controller
+ * with the connection up (it might go down while we're referencing it,
+ * but that does not stop us from enqueuing a protocol request).
+ *
+ * 12. [rdkafka main thread] A DeleteTopics protocol request buffer is
+ * constructed using the input parameters saved on the rko and the
+ * buffer is enqueued on the broker's transmit queue.
+ * The buffer is set up to provide the reply buffer on the rdkafka main
+ * ops queue (the same queue we are operating from) with a handler
+ * callback of rd_kafka_admin_handle_response().
+ * The state is updated to the RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE.
+ *
+ * 13. [broker thread] If the request times out, a response with error code
+ * (ERR__TIMED_OUT) is enqueued. Go to 16.
+ *
+ * 14. [broker thread] If a response is received, the response buffer
+ * is enqueued. Go to 16.
+ *
+ * 15. [rdkafka main thread] The buffer callback (..handle_response())
+ * is called, which attempts to extra the original rko from the eonce,
+ * but if the eonce has already been triggered by some other source
+ * (the timeout timer) the buffer callback simply returns and does nothing
+ * since the admin request is over and a result (probably a timeout)
+ * has been enqueued for the application.
+ * If the rko was still intact we temporarily set the reply buffer
+ * in the rko struct and call the worker callback. Go to 17.
+ *
+ * 16. [rdkafka main thread] The worker callback is called in state
+ * RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE without a response but with an error.
+ * An error result op is created and enqueued on the application's
+ * provided response rkqu queue.
+ *
+ * 17. [rdkafka main thread] The worker callback is called in state
+ * RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE with a response buffer with no
+ * error set.
+ * The worker calls the response `parse()` callback to parse the response
+ * buffer and populates a result op (rko_result) with the response
+ * information (such as per-topic error codes, etc).
+ * The result op is returned to the worker.
+ *
+ * 18. [rdkafka main thread] The worker enqueues the result up (rko_result)
+ * on the application's provided response rkqu queue.
+ *
+ * 19. [app thread] The application calls rd_kafka_queue_poll() to
+ * receive the result of the operation. The result may have been
+ * enqueued in step 18 thanks to succesful completion, or in any
+ * of the earlier stages when an error was encountered.
+ *
+ * 20. [app thread] The application uses rd_kafka_event_DeleteTopics_result()
+ * to retrieve the request-specific result type.
+ *
+ * 21. Done.
+ *
+ */
+
+
+/**
+ * @brief Admin op callback types
+ */
+typedef rd_kafka_resp_err_t (rd_kafka_admin_Request_cb_t) (
+ rd_kafka_broker_t *rkb,
+ const rd_list_t *configs /*(ConfigResource_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque)
+ RD_WARN_UNUSED_RESULT;
+
+typedef rd_kafka_resp_err_t (rd_kafka_admin_Response_parse_cb_t) (
+ rd_kafka_op_t *rko_req,
+ rd_kafka_op_t **rko_resultp,
+ rd_kafka_buf_t *reply,
+ char *errstr, size_t errstr_size)
+ RD_WARN_UNUSED_RESULT;
+
+
+
+/**
+ * @struct Request-specific worker callbacks.
+ */
+struct rd_kafka_admin_worker_cbs {
+ /**< Protocol request callback which is called
+ * to construct and send the request. */
+ rd_kafka_admin_Request_cb_t *request;
+
+ /**< Protocol response parser callback which is called
+ * to translate the response to a rko_result op. */
+ rd_kafka_admin_Response_parse_cb_t *parse;
+};
+
+
+/* Forward declarations */
+static void rd_kafka_AdminOptions_init (rd_kafka_t *rk,
+ rd_kafka_AdminOptions_t *options);
+static rd_kafka_op_res_t
+rd_kafka_admin_worker (rd_kafka_t *rk, rd_kafka_q_t *rkq, rd_kafka_op_t *rko);
+static rd_kafka_ConfigEntry_t *
+rd_kafka_ConfigEntry_copy (const rd_kafka_ConfigEntry_t *src);
+static void rd_kafka_ConfigEntry_free (void *ptr);
+static void *rd_kafka_ConfigEntry_list_copy (const void *src, void *opaque);
+
+
+/**
+ * @name Common admin request code
+ * @{
+ *
+ *
+ */
+
+/**
+ * @brief Create a new admin_result op based on the request op \p rko_req
+ */
+static rd_kafka_op_t *rd_kafka_admin_result_new (const rd_kafka_op_t *rko_req) {
+ rd_kafka_op_t *rko_result;
+
+ rko_result = rd_kafka_op_new(RD_KAFKA_OP_ADMIN_RESULT);
+ rko_result->rko_rk = rko_req->rko_rk;
+
+ rko_result->rko_u.admin_result.opaque =
+ rd_kafka_confval_get_ptr(&rko_req->rko_u.admin_request.
+ options.opaque);
+ rko_result->rko_u.admin_result.reqtype = rko_req->rko_type;
+ rko_result->rko_evtype = rko_req->rko_u.admin_request.reply_event_type;
+
+ return rko_result;
+}
+
+
+/**
+ * @brief Set error code and error string on admin_result op \p rko.
+ */
+static void rd_kafka_admin_result_set_err0 (rd_kafka_op_t *rko,
+ rd_kafka_resp_err_t err,
+ const char *fmt, va_list ap) {
+ char buf[512];
+
+ rd_vsnprintf(buf, sizeof(buf), fmt, ap);
+
+ rko->rko_err = err;
+
+ if (rko->rko_u.admin_result.errstr)
+ rd_free(rko->rko_u.admin_result.errstr);
+ rko->rko_u.admin_result.errstr = rd_strdup(buf);
+
+ rd_kafka_dbg(rko->rko_rk, ADMIN, "ADMINFAIL",
+ "Admin %s result error: %s",
+ rd_kafka_op2str(rko->rko_u.admin_result.reqtype),
+ rko->rko_u.admin_result.errstr);
+}
+
+/**
+ * @sa rd_kafka_admin_result_set_err0
+ */
+static RD_UNUSED void rd_kafka_admin_result_set_err (rd_kafka_op_t *rko,
+ rd_kafka_resp_err_t err,
+ const char *fmt, ...) {
+ va_list ap;
+
+ va_start(ap, fmt);
+ rd_kafka_admin_result_set_err0(rko, err, fmt, ap);
+ va_end(ap);
+}
+
+/**
+ * @brief Enqueue admin_result on application's queue.
+ */
+static RD_INLINE
+void rd_kafka_admin_result_enq (rd_kafka_op_t *rko_req,
+ rd_kafka_op_t *rko_result) {
+ rd_kafka_replyq_enq(&rko_req->rko_u.admin_request.replyq, rko_result,
+ rko_req->rko_u.admin_request.replyq.version);
+}
+
+/**
+ * @brief Set request-level error code and string in reply op.
+ */
+static void rd_kafka_admin_result_fail (rd_kafka_op_t *rko_req,
+ rd_kafka_resp_err_t err,
+ const char *fmt, ...) {
+ va_list ap;
+ rd_kafka_op_t *rko_result;
+
+ rko_result = rd_kafka_admin_result_new(rko_req);
+
+ va_start(ap, fmt);
+ rd_kafka_admin_result_set_err0(rko_result, err, fmt, ap);
+ va_end(ap);
+
+ rd_kafka_admin_result_enq(rko_req, rko_result);
+}
+
+
+
+/**
+ * @brief Return the topics list from a topic-related result object.
+ */
+static const rd_kafka_topic_result_t **
+rd_kafka_admin_result_ret_topics (const rd_kafka_op_t *rko,
+ size_t *cntp) {
+ rd_kafka_op_type_t reqtype =
+ rko->rko_u.admin_result.reqtype & ~RD_KAFKA_OP_FLAGMASK;
+ rd_assert(reqtype == RD_KAFKA_OP_CREATETOPICS ||
+ reqtype == RD_KAFKA_OP_DELETETOPICS ||
+ reqtype == RD_KAFKA_OP_CREATEPARTITIONS);
+
+ *cntp = rd_list_cnt(&rko->rko_u.admin_result.results);
+ return (const rd_kafka_topic_result_t **)rko->rko_u.admin_result.
+ results.rl_elems;
+}
+
+/**
+ * @brief Return the ConfigResource list from a config-related result object.
+ */
+static const rd_kafka_ConfigResource_t **
+rd_kafka_admin_result_ret_resources (const rd_kafka_op_t *rko,
+ size_t *cntp) {
+ rd_kafka_op_type_t reqtype =
+ rko->rko_u.admin_result.reqtype & ~RD_KAFKA_OP_FLAGMASK;
+ rd_assert(reqtype == RD_KAFKA_OP_ALTERCONFIGS ||
+ reqtype == RD_KAFKA_OP_DESCRIBECONFIGS);
+
+ *cntp = rd_list_cnt(&rko->rko_u.admin_result.results);
+ return (const rd_kafka_ConfigResource_t **)rko->rko_u.admin_result.
+ results.rl_elems;
+}
+
+
+
+
+/**
+ * @brief Create a new admin_request op of type \p optype and sets up the
+ * generic (type independent files).
+ *
+ * The caller shall then populate the admin_request.args list
+ * and enqueue the op on rk_ops for further processing work.
+ *
+ * @param cbs Callbacks, must reside in .data segment.
+ * @param options Optional options, may be NULL to use defaults.
+ *
+ * @locks none
+ * @locality application thread
+ */
+static rd_kafka_op_t *
+rd_kafka_admin_request_op_new (rd_kafka_t *rk,
+ rd_kafka_op_type_t optype,
+ rd_kafka_event_type_t reply_event_type,
+ const struct rd_kafka_admin_worker_cbs *cbs,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu) {
+ rd_kafka_op_t *rko;
+
+ rd_assert(rk);
+ rd_assert(rkqu);
+ rd_assert(cbs);
+
+ rko = rd_kafka_op_new_cb(rk, optype, rd_kafka_admin_worker);
+
+ rko->rko_u.admin_request.reply_event_type = reply_event_type;
+
+ rko->rko_u.admin_request.cbs = (struct rd_kafka_admin_worker_cbs *)cbs;
+
+ /* Make a copy of the options */
+ if (options)
+ rko->rko_u.admin_request.options = *options;
+ else
+ rd_kafka_AdminOptions_init(rk,
+ &rko->rko_u.admin_request.options);
+
+ /* Default to controller */
+ rko->rko_u.admin_request.broker_id = -1;
+
+ /* Calculate absolute timeout */
+ rko->rko_u.admin_request.abs_timeout =
+ rd_timeout_init(
+ rd_kafka_confval_get_int(&rko->rko_u.admin_request.
+ options.request_timeout));
+
+ /* Setup enq-op-once, which is triggered by either timer code
+ * or future wait-controller code. */
+ rko->rko_u.admin_request.eonce =
+ rd_kafka_enq_once_new(rko, RD_KAFKA_REPLYQ(rk->rk_ops, 0));
+
+ /* The timer itself must be started from the rdkafka main thread,
+ * not here. */
+
+ /* Set up replyq */
+ rd_kafka_set_replyq(&rko->rko_u.admin_request.replyq,
+ rkqu->rkqu_q, 0);
+
+ rko->rko_u.admin_request.state = RD_KAFKA_ADMIN_STATE_INIT;
+ return rko;
+}
+
+
+/**
+ * @brief Timer timeout callback for the admin rko's eonce object.
+ */
+static void rd_kafka_admin_eonce_timeout_cb (rd_kafka_timers_t *rkts,
+ void *arg) {
+ rd_kafka_enq_once_t *eonce = arg;
+
+ rd_kafka_enq_once_trigger(eonce, RD_KAFKA_RESP_ERR__TIMED_OUT,
+ "timer timeout");
+}
+
+
+
+/**
+ * @brief Common worker destroy to be called in destroy: label
+ * in worker.
+ */
+static void rd_kafka_admin_common_worker_destroy (rd_kafka_t *rk,
+ rd_kafka_op_t *rko) {
+ int timer_was_stopped;
+
+ /* Free resources for this op. */
+ timer_was_stopped =
+ rd_kafka_timer_stop(&rk->rk_timers,
+ &rko->rko_u.admin_request.tmr, rd_true);
+
+
+ if (rko->rko_u.admin_request.eonce) {
+ /* Remove the stopped timer's eonce reference since its
+ * callback will not have fired if we stopped the timer. */
+ if (timer_was_stopped)
+ rd_kafka_enq_once_del_source(rko->rko_u.admin_request.
+ eonce, "timeout timer");
+
+ /* This is thread-safe to do even if there are outstanding
+ * timers or wait-controller references to the eonce
+ * since they only hold direct reference to the eonce,
+ * not the rko (the eonce holds a reference to the rko but
+ * it is cleared here). */
+ rd_kafka_enq_once_destroy(rko->rko_u.admin_request.eonce);
+ rko->rko_u.admin_request.eonce = NULL;
+ }
+}
+
+
+
+/**
+ * @brief Asynchronously look up a broker.
+ * To be called repeatedly from each invocation of the worker
+ * when in state RD_KAFKA_ADMIN_STATE_WAIT_BROKER until
+ * a valid rkb is returned.
+ *
+ * @returns the broker rkb with refcount increased, or NULL if not yet
+ * available.
+ */
+static rd_kafka_broker_t *
+rd_kafka_admin_common_get_broker (rd_kafka_t *rk,
+ rd_kafka_op_t *rko,
+ int32_t broker_id) {
+ rd_kafka_broker_t *rkb;
+
+ rd_kafka_dbg(rk, ADMIN, "ADMIN", "%s: looking up broker %"PRId32,
+ rd_kafka_op2str(rko->rko_type), broker_id);
+
+ /* Since we're iterating over this broker_async() call
+ * (asynchronously) until a broker is availabe (or timeout)
+ * we need to re-enable the eonce to be triggered again (which
+ * is not necessary the first time we get here, but there
+ * is no harm doing it then either). */
+ rd_kafka_enq_once_reenable(rko->rko_u.admin_request.eonce,
+ rko, RD_KAFKA_REPLYQ(rk->rk_ops, 0));
+
+ /* Look up the broker asynchronously, if the broker
+ * is not available the eonce is registered for broker
+ * state changes which will cause our function to be called
+ * again as soon as (any) broker state changes.
+ * When we are called again we perform the broker lookup
+ * again and hopefully get an rkb back, otherwise defer a new
+ * async wait. Repeat until success or timeout. */
+ if (!(rkb = rd_kafka_broker_get_async(
+ rk, broker_id, RD_KAFKA_BROKER_STATE_UP,
+ rko->rko_u.admin_request.eonce))) {
+ /* Broker not available, wait asynchronously
+ * for broker metadata code to trigger eonce. */
+ return NULL;
+ }
+
+ rd_kafka_dbg(rk, ADMIN, "ADMIN", "%s: broker %"PRId32" is %s",
+ rd_kafka_op2str(rko->rko_type), broker_id, rkb->rkb_name);
+
+ return rkb;
+}
+
+
+/**
+ * @brief Asynchronously look up the controller.
+ * To be called repeatedly from each invocation of the worker
+ * when in state RD_KAFKA_ADMIN_STATE_WAIT_CONTROLLER until
+ * a valid rkb is returned.
+ *
+ * @returns the controller rkb with refcount increased, or NULL if not yet
+ * available.
+ */
+static rd_kafka_broker_t *
+rd_kafka_admin_common_get_controller (rd_kafka_t *rk,
+ rd_kafka_op_t *rko) {
+ rd_kafka_broker_t *rkb;
+
+ rd_kafka_dbg(rk, ADMIN, "ADMIN", "%s: looking up controller",
+ rd_kafka_op2str(rko->rko_type));
+
+ /* Since we're iterating over this controller_async() call
+ * (asynchronously) until a controller is availabe (or timeout)
+ * we need to re-enable the eonce to be triggered again (which
+ * is not necessary the first time we get here, but there
+ * is no harm doing it then either). */
+ rd_kafka_enq_once_reenable(rko->rko_u.admin_request.eonce,
+ rko, RD_KAFKA_REPLYQ(rk->rk_ops, 0));
+
+ /* Look up the controller asynchronously, if the controller
+ * is not available the eonce is registered for broker
+ * state changes which will cause our function to be called
+ * again as soon as (any) broker state changes.
+ * When we are called again we perform the controller lookup
+ * again and hopefully get an rkb back, otherwise defer a new
+ * async wait. Repeat until success or timeout. */
+ if (!(rkb = rd_kafka_broker_controller_async(
+ rk, RD_KAFKA_BROKER_STATE_UP,
+ rko->rko_u.admin_request.eonce))) {
+ /* Controller not available, wait asynchronously
+ * for controller code to trigger eonce. */
+ return NULL;
+ }
+
+ rd_kafka_dbg(rk, ADMIN, "ADMIN", "%s: controller %s",
+ rd_kafka_op2str(rko->rko_type), rkb->rkb_name);
+
+ return rkb;
+}
+
+
+
+/**
+ * @brief Handle response from broker by triggering worker callback.
+ *
+ * @param opaque is the eonce from the worker protocol request call.
+ */
+static void rd_kafka_admin_handle_response (rd_kafka_t *rk,
+ rd_kafka_broker_t *rkb,
+ rd_kafka_resp_err_t err,
+ rd_kafka_buf_t *reply,
+ rd_kafka_buf_t *request,
+ void *opaque) {
+ rd_kafka_enq_once_t *eonce = opaque;
+ rd_kafka_op_t *rko;
+
+ /* From ...add_source("send") */
+ rko = rd_kafka_enq_once_disable(eonce);
+
+ if (!rko) {
+ /* The operation timed out and the worker was
+ * dismantled while we were waiting for broker response,
+ * do nothing - everything has been cleaned up. */
+ rd_kafka_dbg(rk, ADMIN, "ADMIN",
+ "Dropping outdated %sResponse with return code %s",
+ request ?
+ rd_kafka_ApiKey2str(request->rkbuf_reqhdr.ApiKey):
+ "???",
+ rd_kafka_err2str(err));
+ return;
+ }
+
+ /* Attach reply buffer to rko for parsing in the worker. */
+ rd_assert(!rko->rko_u.admin_request.reply_buf);
+ rko->rko_u.admin_request.reply_buf = reply;
+ rko->rko_err = err;
+
+ if (rko->rko_op_cb(rk, NULL, rko) == RD_KAFKA_OP_RES_HANDLED)
+ rd_kafka_op_destroy(rko);
+
+}
+
+
+
+/**
+ * @brief Common worker state machine handling regardless of request type.
+ *
+ * Tasks:
+ * - Sets up timeout on first call.
+ * - Checks for timeout.
+ * - Checks for and fails on errors.
+ * - Async Controller and broker lookups
+ * - Calls the Request callback
+ * - Calls the parse callback
+ * - Result reply
+ * - Destruction of rko
+ *
+ * rko->rko_err may be one of:
+ * RD_KAFKA_RESP_ERR_NO_ERROR, or
+ * RD_KAFKA_RESP_ERR__DESTROY for queue destruction cleanup, or
+ * RD_KAFKA_RESP_ERR__TIMED_OUT if request has timed out,
+ * or any other error code triggered by other parts of the code.
+ *
+ * @returns a hint to the op code whether the rko should be destroyed or not.
+ */
+static rd_kafka_op_res_t
+rd_kafka_admin_worker (rd_kafka_t *rk, rd_kafka_q_t *rkq, rd_kafka_op_t *rko) {
+ const char *name = rd_kafka_op2str(rko->rko_type);
+ rd_ts_t timeout_in;
+ rd_kafka_broker_t *rkb = NULL;
+ rd_kafka_resp_err_t err;
+ char errstr[512];
+
+ if (rd_kafka_terminating(rk)) {
+ rd_kafka_dbg(rk, ADMIN, name,
+ "%s worker called in state %s: "
+ "handle is terminating: %s",
+ name,
+ rd_kafka_admin_state_desc[rko->rko_u.
+ admin_request.state],
+ rd_kafka_err2str(rko->rko_err));
+ goto destroy;
+ }
+
+ if (rko->rko_err == RD_KAFKA_RESP_ERR__DESTROY)
+ goto destroy; /* rko being destroyed (silent) */
+
+ rd_kafka_dbg(rk, ADMIN, name,
+ "%s worker called in state %s: %s",
+ name,
+ rd_kafka_admin_state_desc[rko->rko_u.admin_request.state],
+ rd_kafka_err2str(rko->rko_err));
+
+ rd_assert(thrd_is_current(rko->rko_rk->rk_thread));
+
+ /* Check for errors raised asynchronously (e.g., by timer) */
+ if (rko->rko_err) {
+ rd_kafka_admin_result_fail(
+ rko, rko->rko_err,
+ "Failed while %s: %s",
+ rd_kafka_admin_state_desc[rko->rko_u.
+ admin_request.state],
+ rd_kafka_err2str(rko->rko_err));
+ goto destroy;
+ }
+
+ /* Check for timeout */
+ timeout_in = rd_timeout_remains_us(rko->rko_u.admin_request.
+ abs_timeout);
+ if (timeout_in <= 0) {
+ rd_kafka_admin_result_fail(
+ rko, RD_KAFKA_RESP_ERR__TIMED_OUT,
+ "Timed out %s",
+ rd_kafka_admin_state_desc[rko->rko_u.
+ admin_request.state]);
+ goto destroy;
+ }
+
+ redo:
+ switch (rko->rko_u.admin_request.state)
+ {
+ case RD_KAFKA_ADMIN_STATE_INIT:
+ {
+ int32_t broker_id;
+
+ /* First call. */
+
+ /* Set up timeout timer. */
+ rd_kafka_enq_once_add_source(rko->rko_u.admin_request.eonce,
+ "timeout timer");
+ rd_kafka_timer_start_oneshot(&rk->rk_timers,
+ &rko->rko_u.admin_request.tmr,
+ timeout_in,
+ rd_kafka_admin_eonce_timeout_cb,
+ rko->rko_u.admin_request.eonce);
+
+ /* Use explicitly specified broker_id, if available. */
+ broker_id = (int32_t)rd_kafka_confval_get_int(
+ &rko->rko_u.admin_request.options.broker);
+
+ if (broker_id != -1) {
+ rd_kafka_dbg(rk, ADMIN, name,
+ "%s using explicitly "
+ "set broker id %"PRId32
+ " rather than %"PRId32,
+ name, broker_id,
+ rko->rko_u.admin_request.broker_id);
+ rko->rko_u.admin_request.broker_id = broker_id;
+ }
+
+ /* Look up controller or specific broker. */
+ if (rko->rko_u.admin_request.broker_id != -1) {
+ /* Specific broker */
+ rko->rko_u.admin_request.state =
+ RD_KAFKA_ADMIN_STATE_WAIT_BROKER;
+ } else {
+ /* Controller */
+ rko->rko_u.admin_request.state =
+ RD_KAFKA_ADMIN_STATE_WAIT_CONTROLLER;
+ }
+ goto redo; /* Trigger next state immediately */
+ }
+
+
+ case RD_KAFKA_ADMIN_STATE_WAIT_BROKER:
+ /* Broker lookup */
+ if (!(rkb = rd_kafka_admin_common_get_broker(
+ rk, rko, rko->rko_u.admin_request.broker_id))) {
+ /* Still waiting for broker to become available */
+ return RD_KAFKA_OP_RES_KEEP;
+ }
+
+ rko->rko_u.admin_request.state =
+ RD_KAFKA_ADMIN_STATE_CONSTRUCT_REQUEST;
+ goto redo;
+
+ case RD_KAFKA_ADMIN_STATE_WAIT_CONTROLLER:
+ if (!(rkb = rd_kafka_admin_common_get_controller(rk, rko))) {
+ /* Still waiting for controller to become available. */
+ return RD_KAFKA_OP_RES_KEEP;
+ }
+
+ rko->rko_u.admin_request.state =
+ RD_KAFKA_ADMIN_STATE_CONSTRUCT_REQUEST;
+ goto redo;
+
+
+ case RD_KAFKA_ADMIN_STATE_CONSTRUCT_REQUEST:
+ /* Got broker, send protocol request. */
+
+ /* Make sure we're called from a 'goto redo' where
+ * the rkb was set. */
+ rd_assert(rkb);
+
+ /* Still need to use the eonce since this worker may
+ * time out while waiting for response from broker, in which
+ * case the broker response will hit an empty eonce (ok). */
+ rd_kafka_enq_once_add_source(rko->rko_u.admin_request.eonce,
+ "send");
+
+ /* Send request (async) */
+ err = rko->rko_u.admin_request.cbs->request(
+ rkb,
+ &rko->rko_u.admin_request.args,
+ &rko->rko_u.admin_request.options,
+ errstr, sizeof(errstr),
+ RD_KAFKA_REPLYQ(rk->rk_ops, 0),
+ rd_kafka_admin_handle_response,
+ rko->rko_u.admin_request.eonce);
+
+ /* Loose broker refcount from get_broker(), get_controller() */
+ rd_kafka_broker_destroy(rkb);
+
+ if (err) {
+ rd_kafka_enq_once_del_source(
+ rko->rko_u.admin_request.eonce, "send");
+ rd_kafka_admin_result_fail(rko, err, "%s", errstr);
+ goto destroy;
+ }
+
+ rko->rko_u.admin_request.state =
+ RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE;
+
+ /* Wait asynchronously for broker response, which will
+ * trigger the eonce and worker to be called again. */
+ return RD_KAFKA_OP_RES_KEEP;
+
+
+ case RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE:
+ {
+ rd_kafka_op_t *rko_result;
+
+ /* Response received.
+ * Parse response and populate result to application */
+ err = rko->rko_u.admin_request.cbs->parse(
+ rko, &rko_result,
+ rko->rko_u.admin_request.reply_buf,
+ errstr, sizeof(errstr));
+ if (err) {
+ rd_kafka_admin_result_fail(
+ rko, err,
+ "%s worker failed to parse response: %s",
+ name, errstr);
+ goto destroy;
+ }
+
+ /* Enqueue result on application queue, we're done. */
+ rd_kafka_admin_result_enq(rko, rko_result);
+
+ goto destroy;
+ }
+ }
+
+ return RD_KAFKA_OP_RES_KEEP;
+
+ destroy:
+ rd_kafka_admin_common_worker_destroy(rk, rko);
+ return RD_KAFKA_OP_RES_HANDLED; /* trigger's op_destroy() */
+
+}
+
+
+/**@}*/
+
+
+/**
+ * @name Generic AdminOptions
+ * @{
+ *
+ *
+ */
+
+rd_kafka_resp_err_t
+rd_kafka_AdminOptions_set_request_timeout (rd_kafka_AdminOptions_t *options,
+ int timeout_ms,
+ char *errstr, size_t errstr_size) {
+ return rd_kafka_confval_set_type(&options->request_timeout,
+ RD_KAFKA_CONFVAL_INT, &timeout_ms,
+ errstr, errstr_size);
+}
+
+
+rd_kafka_resp_err_t
+rd_kafka_AdminOptions_set_operation_timeout (rd_kafka_AdminOptions_t *options,
+ int timeout_ms,
+ char *errstr, size_t errstr_size) {
+ return rd_kafka_confval_set_type(&options->operation_timeout,
+ RD_KAFKA_CONFVAL_INT, &timeout_ms,
+ errstr, errstr_size);
+}
+
+
+rd_kafka_resp_err_t
+rd_kafka_AdminOptions_set_validate_only (rd_kafka_AdminOptions_t *options,
+ int true_or_false,
+ char *errstr, size_t errstr_size) {
+ return rd_kafka_confval_set_type(&options->validate_only,
+ RD_KAFKA_CONFVAL_INT, &true_or_false,
+ errstr, errstr_size);
+}
+
+rd_kafka_resp_err_t
+rd_kafka_AdminOptions_set_incremental (rd_kafka_AdminOptions_t *options,
+ int true_or_false,
+ char *errstr, size_t errstr_size) {
+ rd_snprintf(errstr, errstr_size,
+ "Incremental updates currently not supported, see KIP-248");
+ return RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED;
+
+ return rd_kafka_confval_set_type(&options->incremental,
+ RD_KAFKA_CONFVAL_INT, &true_or_false,
+ errstr, errstr_size);
+}
+
+rd_kafka_resp_err_t
+rd_kafka_AdminOptions_set_broker (rd_kafka_AdminOptions_t *options,
+ int32_t broker_id,
+ char *errstr, size_t errstr_size) {
+ int ibroker_id = (int)broker_id;
+
+ return rd_kafka_confval_set_type(&options->broker,
+ RD_KAFKA_CONFVAL_INT,
+ &ibroker_id,
+ errstr, errstr_size);
+}
+
+void
+rd_kafka_AdminOptions_set_opaque (rd_kafka_AdminOptions_t *options,
+ void *opaque) {
+ rd_kafka_confval_set_type(&options->opaque,
+ RD_KAFKA_CONFVAL_PTR, opaque, NULL, 0);
+}
+
+
+/**
+ * @brief Initialize and set up defaults for AdminOptions
+ */
+static void rd_kafka_AdminOptions_init (rd_kafka_t *rk,
+ rd_kafka_AdminOptions_t *options) {
+ rd_kafka_confval_init_int(&options->request_timeout, "request_timeout",
+ 0, 3600*1000,
+ rk->rk_conf.admin.request_timeout_ms);
+
+ if (options->for_api == RD_KAFKA_ADMIN_OP_ANY ||
+ options->for_api == RD_KAFKA_ADMIN_OP_CREATETOPICS ||
+ options->for_api == RD_KAFKA_ADMIN_OP_DELETETOPICS ||
+ options->for_api == RD_KAFKA_ADMIN_OP_CREATEPARTITIONS)
+ rd_kafka_confval_init_int(&options->operation_timeout,
+ "operation_timeout",
+ -1, 3600*1000, 0);
+ else
+ rd_kafka_confval_disable(&options->operation_timeout,
+ "operation_timeout");
+
+ if (options->for_api == RD_KAFKA_ADMIN_OP_ANY ||
+ options->for_api == RD_KAFKA_ADMIN_OP_CREATETOPICS ||
+ options->for_api == RD_KAFKA_ADMIN_OP_CREATEPARTITIONS ||
+ options->for_api == RD_KAFKA_ADMIN_OP_ALTERCONFIGS)
+ rd_kafka_confval_init_int(&options->validate_only,
+ "validate_only",
+ 0, 1, 0);
+ else
+ rd_kafka_confval_disable(&options->validate_only,
+ "validate_only");
+
+ if (options->for_api == RD_KAFKA_ADMIN_OP_ANY ||
+ options->for_api == RD_KAFKA_ADMIN_OP_ALTERCONFIGS)
+ rd_kafka_confval_init_int(&options->incremental,
+ "incremental",
+ 0, 1, 0);
+ else
+ rd_kafka_confval_disable(&options->incremental,
+ "incremental");
+
+ rd_kafka_confval_init_int(&options->broker, "broker",
+ 0, INT32_MAX, -1);
+ rd_kafka_confval_init_ptr(&options->opaque, "opaque");
+}
+
+
+rd_kafka_AdminOptions_t *
+rd_kafka_AdminOptions_new (rd_kafka_t *rk, rd_kafka_admin_op_t for_api) {
+ rd_kafka_AdminOptions_t *options;
+
+ if ((int)for_api < 0 || for_api >= RD_KAFKA_ADMIN_OP__CNT)
+ return NULL;
+
+ options = rd_calloc(1, sizeof(*options));
+
+ options->for_api = for_api;
+
+ rd_kafka_AdminOptions_init(rk, options);
+
+ return options;
+}
+
+void rd_kafka_AdminOptions_destroy (rd_kafka_AdminOptions_t *options) {
+ rd_free(options);
+}
+
+/**@}*/
+
+
+
+
+
+
+/**
+ * @name CreateTopics
+ * @{
+ *
+ *
+ *
+ */
+
+
+
+rd_kafka_NewTopic_t *
+rd_kafka_NewTopic_new (const char *topic,
+ int num_partitions,
+ int replication_factor,
+ char *errstr, size_t errstr_size) {
+ rd_kafka_NewTopic_t *new_topic;
+
+ if (!topic) {
+ rd_snprintf(errstr, errstr_size, "Invalid topic name");
+ return NULL;
+ }
+
+ if (num_partitions < 1 || num_partitions > RD_KAFKAP_PARTITIONS_MAX) {
+ rd_snprintf(errstr, errstr_size, "num_partitions out of "
+ "expected range %d..%d",
+ 1, RD_KAFKAP_PARTITIONS_MAX);
+ return NULL;
+ }
+
+ if (replication_factor < -1 ||
+ replication_factor > RD_KAFKAP_BROKERS_MAX) {
+ rd_snprintf(errstr, errstr_size,
+ "replication_factor out of expected range %d..%d",
+ -1, RD_KAFKAP_BROKERS_MAX);
+ return NULL;
+ }
+
+ new_topic = rd_calloc(1, sizeof(*new_topic));
+ new_topic->topic = rd_strdup(topic);
+ new_topic->num_partitions = num_partitions;
+ new_topic->replication_factor = replication_factor;
+
+ /* List of int32 lists */
+ rd_list_init(&new_topic->replicas, 0, rd_list_destroy_free);
+ rd_list_prealloc_elems(&new_topic->replicas, 0,
+ num_partitions, 0/*nozero*/);
+
+ /* List of ConfigEntrys */
+ rd_list_init(&new_topic->config, 0, rd_kafka_ConfigEntry_free);
+
+ return new_topic;
+
+}
+
+
+/**
+ * @brief Topic name comparator for NewTopic_t
+ */
+static int rd_kafka_NewTopic_cmp (const void *_a, const void *_b) {
+ const rd_kafka_NewTopic_t *a = _a, *b = _b;
+ return strcmp(a->topic, b->topic);
+}
+
+
+
+/**
+ * @brief Allocate a new NewTopic and make a copy of \p src
+ */
+static rd_kafka_NewTopic_t *
+rd_kafka_NewTopic_copy (const rd_kafka_NewTopic_t *src) {
+ rd_kafka_NewTopic_t *dst;
+
+ dst = rd_kafka_NewTopic_new(src->topic, src->num_partitions,
+ src->replication_factor, NULL, 0);
+ rd_assert(dst);
+
+ rd_list_destroy(&dst->replicas); /* created in .._new() */
+ rd_list_init_copy(&dst->replicas, &src->replicas);
+ rd_list_copy_to(&dst->replicas, &src->replicas,
+ rd_list_copy_preallocated, NULL);
+
+ rd_list_init_copy(&dst->config, &src->config);
+ rd_list_copy_to(&dst->config, &src->config,
+ rd_kafka_ConfigEntry_list_copy, NULL);
+
+ return dst;
+}
+
+void rd_kafka_NewTopic_destroy (rd_kafka_NewTopic_t *new_topic) {
+ rd_list_destroy(&new_topic->replicas);
+ rd_list_destroy(&new_topic->config);
+ rd_free(new_topic->topic);
+ rd_free(new_topic);
+}
+
+static void rd_kafka_NewTopic_free (void *ptr) {
+ rd_kafka_NewTopic_destroy(ptr);
+}
+
+void
+rd_kafka_NewTopic_destroy_array (rd_kafka_NewTopic_t **new_topics,
+ size_t new_topic_cnt) {
+ size_t i;
+ for (i = 0 ; i < new_topic_cnt ; i++)
+ rd_kafka_NewTopic_destroy(new_topics[i]);
+}
+
+
+rd_kafka_resp_err_t
+rd_kafka_NewTopic_set_replica_assignment (rd_kafka_NewTopic_t *new_topic,
+ int32_t partition,
+ int32_t *broker_ids,
+ size_t broker_id_cnt,
+ char *errstr, size_t errstr_size) {
+ rd_list_t *rl;
+ int i;
+
+ if (new_topic->replication_factor != -1) {
+ rd_snprintf(errstr, errstr_size,
+ "Specifying a replication factor and "
+ "a replica assignment are mutually exclusive");
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ /* Replica partitions must be added consecutively starting from 0. */
+ if (partition != rd_list_cnt(&new_topic->replicas)) {
+ rd_snprintf(errstr, errstr_size,
+ "Partitions must be added in order, "
+ "starting at 0: expecting partition %d, "
+ "not %"PRId32,
+ rd_list_cnt(&new_topic->replicas), partition);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ if (broker_id_cnt > RD_KAFKAP_BROKERS_MAX) {
+ rd_snprintf(errstr, errstr_size,
+ "Too many brokers specified "
+ "(RD_KAFKAP_BROKERS_MAX=%d)",
+ RD_KAFKAP_BROKERS_MAX);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+
+ rl = rd_list_init_int32(rd_list_new(0, NULL), (int)broker_id_cnt);
+
+ for (i = 0 ; i < (int)broker_id_cnt ; i++)
+ rd_list_set_int32(rl, i, broker_ids[i]);
+
+ rd_list_add(&new_topic->replicas, rl);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+/**
+ * @brief Generic constructor of ConfigEntry which is also added to \p rl
+ */
+static rd_kafka_resp_err_t
+rd_kafka_admin_add_config0 (rd_list_t *rl,
+ const char *name, const char *value,
+ rd_kafka_AlterOperation_t operation) {
+ rd_kafka_ConfigEntry_t *entry;
+
+ if (!name)
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+
+ entry = rd_calloc(1, sizeof(*entry));
+ entry->kv = rd_strtup_new(name, value);
+ entry->a.operation = operation;
+
+ rd_list_add(rl, entry);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+rd_kafka_resp_err_t
+rd_kafka_NewTopic_set_config (rd_kafka_NewTopic_t *new_topic,
+ const char *name, const char *value) {
+ return rd_kafka_admin_add_config0(&new_topic->config, name, value,
+ RD_KAFKA_ALTER_OP_ADD);
+}
+
+
+
+/**
+ * @brief Parse CreateTopicsResponse and create ADMIN_RESULT op.
+ */
+static rd_kafka_resp_err_t
+rd_kafka_CreateTopicsResponse_parse (rd_kafka_op_t *rko_req,
+ rd_kafka_op_t **rko_resultp,
+ rd_kafka_buf_t *reply,
+ char *errstr, size_t errstr_size) {
+ const int log_decode_errors = LOG_ERR;
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ rd_kafka_broker_t *rkb = reply->rkbuf_rkb;
+ rd_kafka_t *rk = rkb->rkb_rk;
+ rd_kafka_op_t *rko_result = NULL;
+ int32_t topic_cnt;
+ int i;
+
+ if (rd_kafka_buf_ApiVersion(reply) >= 2) {
+ int32_t Throttle_Time;
+ rd_kafka_buf_read_i32(reply, &Throttle_Time);
+ rd_kafka_op_throttle_time(rkb, rk->rk_rep, Throttle_Time);
+ }
+
+ /* #topics */
+ rd_kafka_buf_read_i32(reply, &topic_cnt);
+
+ if (topic_cnt > rd_list_cnt(&rko_req->rko_u.admin_request.args))
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Received %"PRId32" topics in response "
+ "when only %d were requested", topic_cnt,
+ rd_list_cnt(&rko_req->rko_u.admin_request.args));
+
+
+ rko_result = rd_kafka_admin_result_new(rko_req);
+
+ rd_list_init(&rko_result->rko_u.admin_result.results, topic_cnt,
+ rd_kafka_topic_result_free);
+
+ for (i = 0 ; i < (int)topic_cnt ; i++) {
+ rd_kafkap_str_t ktopic;
+ int16_t error_code;
+ rd_kafkap_str_t error_msg = RD_KAFKAP_STR_INITIALIZER;
+ char *errstr = NULL;
+ rd_kafka_topic_result_t *terr;
+ rd_kafka_NewTopic_t skel;
+ int orig_pos;
+
+ rd_kafka_buf_read_str(reply, &ktopic);
+ rd_kafka_buf_read_i16(reply, &error_code);
+
+ if (rd_kafka_buf_ApiVersion(reply) >= 1)
+ rd_kafka_buf_read_str(reply, &error_msg);
+
+ /* For non-blocking CreateTopicsRequests the broker
+ * will returned REQUEST_TIMED_OUT for topics
+ * that were triggered for creation -
+ * we hide this error code from the application
+ * since the topic creation is in fact in progress. */
+ if (error_code == RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT &&
+ rd_kafka_confval_get_int(&rko_req->rko_u.
+ admin_request.options.
+ operation_timeout) <= 0) {
+ error_code = RD_KAFKA_RESP_ERR_NO_ERROR;
+ errstr = NULL;
+ }
+
+ if (error_code) {
+ if (RD_KAFKAP_STR_IS_NULL(&error_msg) ||
+ RD_KAFKAP_STR_LEN(&error_msg) == 0)
+ errstr = (char *)rd_kafka_err2str(error_code);
+ else
+ RD_KAFKAP_STR_DUPA(&errstr, &error_msg);
+
+ } else {
+ errstr = NULL;
+ }
+
+ terr = rd_kafka_topic_result_new(ktopic.str,
+ RD_KAFKAP_STR_LEN(&ktopic),
+ error_code, errstr);
+
+ /* As a convenience to the application we insert topic result
+ * in the same order as they were requested. The broker
+ * does not maintain ordering unfortunately. */
+ skel.topic = terr->topic;
+ orig_pos = rd_list_index(&rko_req->rko_u.admin_request.args,
+ &skel, rd_kafka_NewTopic_cmp);
+ if (orig_pos == -1) {
+ rd_kafka_topic_result_destroy(terr);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned topic %.*s that was not "
+ "included in the original request",
+ RD_KAFKAP_STR_PR(&ktopic));
+ }
+
+ if (rd_list_elem(&rko_result->rko_u.admin_result.results,
+ orig_pos) != NULL) {
+ rd_kafka_topic_result_destroy(terr);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned topic %.*s multiple times",
+ RD_KAFKAP_STR_PR(&ktopic));
+ }
+
+ rd_list_set(&rko_result->rko_u.admin_result.results, orig_pos,
+ terr);
+ }
+
+ *rko_resultp = rko_result;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ err_parse:
+ if (rko_result)
+ rd_kafka_op_destroy(rko_result);
+
+ rd_snprintf(errstr, errstr_size,
+ "CreateTopics response protocol parse failure: %s",
+ rd_kafka_err2str(err));
+
+ return err;
+}
+
+
+void rd_kafka_CreateTopics (rd_kafka_t *rk,
+ rd_kafka_NewTopic_t **new_topics,
+ size_t new_topic_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu) {
+ rd_kafka_op_t *rko;
+ size_t i;
+ static const struct rd_kafka_admin_worker_cbs cbs = {
+ rd_kafka_CreateTopicsRequest,
+ rd_kafka_CreateTopicsResponse_parse,
+ };
+
+ rko = rd_kafka_admin_request_op_new(rk,
+ RD_KAFKA_OP_CREATETOPICS,
+ RD_KAFKA_EVENT_CREATETOPICS_RESULT,
+ &cbs, options, rkqu);
+
+ rd_list_init(&rko->rko_u.admin_request.args, (int)new_topic_cnt,
+ rd_kafka_NewTopic_free);
+
+ for (i = 0 ; i < new_topic_cnt ; i++)
+ rd_list_add(&rko->rko_u.admin_request.args,
+ rd_kafka_NewTopic_copy(new_topics[i]));
+
+ rd_kafka_q_enq(rk->rk_ops, rko);
+}
+
+
+/**
+ * @brief Get an array of topic results from a CreateTopics result.
+ *
+ * The returned \p topics life-time is the same as the \p result object.
+ * @param cntp is updated to the number of elements in the array.
+ */
+const rd_kafka_topic_result_t **
+rd_kafka_CreateTopics_result_topics (
+ const rd_kafka_CreateTopics_result_t *result,
+ size_t *cntp) {
+ return rd_kafka_admin_result_ret_topics((const rd_kafka_op_t *)result,
+ cntp);
+}
+
+/**@}*/
+
+
+
+
+/**
+ * @name Delete topics
+ * @{
+ *
+ *
+ *
+ *
+ */
+
+rd_kafka_DeleteTopic_t *rd_kafka_DeleteTopic_new (const char *topic) {
+ size_t tsize = strlen(topic) + 1;
+ rd_kafka_DeleteTopic_t *del_topic;
+
+ /* Single allocation */
+ del_topic = rd_malloc(sizeof(*del_topic) + tsize);
+ del_topic->topic = del_topic->data;
+ memcpy(del_topic->topic, topic, tsize);
+
+ return del_topic;
+}
+
+void rd_kafka_DeleteTopic_destroy (rd_kafka_DeleteTopic_t *del_topic) {
+ rd_free(del_topic);
+}
+
+static void rd_kafka_DeleteTopic_free (void *ptr) {
+ rd_kafka_DeleteTopic_destroy(ptr);
+}
+
+
+void rd_kafka_DeleteTopic_destroy_array (rd_kafka_DeleteTopic_t **del_topics,
+ size_t del_topic_cnt) {
+ size_t i;
+ for (i = 0 ; i < del_topic_cnt ; i++)
+ rd_kafka_DeleteTopic_destroy(del_topics[i]);
+}
+
+
+/**
+ * @brief Topic name comparator for DeleteTopic_t
+ */
+static int rd_kafka_DeleteTopic_cmp (const void *_a, const void *_b) {
+ const rd_kafka_DeleteTopic_t *a = _a, *b = _b;
+ return strcmp(a->topic, b->topic);
+}
+
+/**
+ * @brief Allocate a new DeleteTopic and make a copy of \p src
+ */
+static rd_kafka_DeleteTopic_t *
+rd_kafka_DeleteTopic_copy (const rd_kafka_DeleteTopic_t *src) {
+ return rd_kafka_DeleteTopic_new(src->topic);
+}
+
+
+
+
+
+
+
+/**
+ * @brief Parse DeleteTopicsResponse and create ADMIN_RESULT op.
+ */
+static rd_kafka_resp_err_t
+rd_kafka_DeleteTopicsResponse_parse (rd_kafka_op_t *rko_req,
+ rd_kafka_op_t **rko_resultp,
+ rd_kafka_buf_t *reply,
+ char *errstr, size_t errstr_size) {
+ const int log_decode_errors = LOG_ERR;
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ rd_kafka_broker_t *rkb = reply->rkbuf_rkb;
+ rd_kafka_t *rk = rkb->rkb_rk;
+ rd_kafka_op_t *rko_result = NULL;
+ int32_t topic_cnt;
+ int i;
+
+ if (rd_kafka_buf_ApiVersion(reply) >= 1) {
+ int32_t Throttle_Time;
+ rd_kafka_buf_read_i32(reply, &Throttle_Time);
+ rd_kafka_op_throttle_time(rkb, rk->rk_rep, Throttle_Time);
+ }
+
+ /* #topics */
+ rd_kafka_buf_read_i32(reply, &topic_cnt);
+
+ if (topic_cnt > rd_list_cnt(&rko_req->rko_u.admin_request.args))
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Received %"PRId32" topics in response "
+ "when only %d were requested", topic_cnt,
+ rd_list_cnt(&rko_req->rko_u.admin_request.args));
+
+ rko_result = rd_kafka_admin_result_new(rko_req);
+
+ rd_list_init(&rko_result->rko_u.admin_result.results, topic_cnt,
+ rd_kafka_topic_result_free);
+
+ for (i = 0 ; i < (int)topic_cnt ; i++) {
+ rd_kafkap_str_t ktopic;
+ int16_t error_code;
+ rd_kafka_topic_result_t *terr;
+ rd_kafka_NewTopic_t skel;
+ int orig_pos;
+
+ rd_kafka_buf_read_str(reply, &ktopic);
+ rd_kafka_buf_read_i16(reply, &error_code);
+
+ /* For non-blocking DeleteTopicsRequests the broker
+ * will returned REQUEST_TIMED_OUT for topics
+ * that were triggered for creation -
+ * we hide this error code from the application
+ * since the topic creation is in fact in progress. */
+ if (error_code == RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT &&
+ rd_kafka_confval_get_int(&rko_req->rko_u.
+ admin_request.options.
+ operation_timeout) <= 0) {
+ error_code = RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+
+ terr = rd_kafka_topic_result_new(ktopic.str,
+ RD_KAFKAP_STR_LEN(&ktopic),
+ error_code,
+ error_code ?
+ rd_kafka_err2str(error_code) :
+ NULL);
+
+ /* As a convenience to the application we insert topic result
+ * in the same order as they were requested. The broker
+ * does not maintain ordering unfortunately. */
+ skel.topic = terr->topic;
+ orig_pos = rd_list_index(&rko_req->rko_u.admin_request.args,
+ &skel, rd_kafka_DeleteTopic_cmp);
+ if (orig_pos == -1) {
+ rd_kafka_topic_result_destroy(terr);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned topic %.*s that was not "
+ "included in the original request",
+ RD_KAFKAP_STR_PR(&ktopic));
+ }
+
+ if (rd_list_elem(&rko_result->rko_u.admin_result.results,
+ orig_pos) != NULL) {
+ rd_kafka_topic_result_destroy(terr);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned topic %.*s multiple times",
+ RD_KAFKAP_STR_PR(&ktopic));
+ }
+
+ rd_list_set(&rko_result->rko_u.admin_result.results, orig_pos,
+ terr);
+ }
+
+ *rko_resultp = rko_result;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ err_parse:
+ if (rko_result)
+ rd_kafka_op_destroy(rko_result);
+
+ rd_snprintf(errstr, errstr_size,
+ "DeleteTopics response protocol parse failure: %s",
+ rd_kafka_err2str(err));
+
+ return err;
+}
+
+
+
+
+
+
+void rd_kafka_DeleteTopics (rd_kafka_t *rk,
+ rd_kafka_DeleteTopic_t **del_topics,
+ size_t del_topic_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu) {
+ rd_kafka_op_t *rko;
+ size_t i;
+ static const struct rd_kafka_admin_worker_cbs cbs = {
+ rd_kafka_DeleteTopicsRequest,
+ rd_kafka_DeleteTopicsResponse_parse,
+ };
+
+ rko = rd_kafka_admin_request_op_new(rk,
+ RD_KAFKA_OP_DELETETOPICS,
+ RD_KAFKA_EVENT_DELETETOPICS_RESULT,
+ &cbs, options, rkqu);
+
+ rd_list_init(&rko->rko_u.admin_request.args, (int)del_topic_cnt,
+ rd_kafka_DeleteTopic_free);
+
+ for (i = 0 ; i < del_topic_cnt ; i++)
+ rd_list_add(&rko->rko_u.admin_request.args,
+ rd_kafka_DeleteTopic_copy(del_topics[i]));
+
+ rd_kafka_q_enq(rk->rk_ops, rko);
+}
+
+
+/**
+ * @brief Get an array of topic results from a DeleteTopics result.
+ *
+ * The returned \p topics life-time is the same as the \p result object.
+ * @param cntp is updated to the number of elements in the array.
+ */
+const rd_kafka_topic_result_t **
+rd_kafka_DeleteTopics_result_topics (
+ const rd_kafka_DeleteTopics_result_t *result,
+ size_t *cntp) {
+ return rd_kafka_admin_result_ret_topics((const rd_kafka_op_t *)result,
+ cntp);
+}
+
+
+
+
+/**
+ * @name Create partitions
+ * @{
+ *
+ *
+ *
+ *
+ */
+
+rd_kafka_NewPartitions_t *rd_kafka_NewPartitions_new (const char *topic,
+ size_t new_total_cnt,
+ char *errstr,
+ size_t errstr_size) {
+ size_t tsize = strlen(topic) + 1;
+ rd_kafka_NewPartitions_t *newps;
+
+ if (new_total_cnt < 1 || new_total_cnt > RD_KAFKAP_PARTITIONS_MAX) {
+ rd_snprintf(errstr, errstr_size, "new_total_cnt out of "
+ "expected range %d..%d",
+ 1, RD_KAFKAP_PARTITIONS_MAX);
+ return NULL;
+ }
+
+ /* Single allocation */
+ newps = rd_malloc(sizeof(*newps) + tsize);
+ newps->total_cnt = new_total_cnt;
+ newps->topic = newps->data;
+ memcpy(newps->topic, topic, tsize);
+
+ /* List of int32 lists */
+ rd_list_init(&newps->replicas, 0, rd_list_destroy_free);
+ rd_list_prealloc_elems(&newps->replicas, 0, new_total_cnt, 0/*nozero*/);
+
+ return newps;
+}
+
+/**
+ * @brief Topic name comparator for NewPartitions_t
+ */
+static int rd_kafka_NewPartitions_cmp (const void *_a, const void *_b) {
+ const rd_kafka_NewPartitions_t *a = _a, *b = _b;
+ return strcmp(a->topic, b->topic);
+}
+
+
+/**
+ * @brief Allocate a new CreatePartitions and make a copy of \p src
+ */
+static rd_kafka_NewPartitions_t *
+rd_kafka_NewPartitions_copy (const rd_kafka_NewPartitions_t *src) {
+ rd_kafka_NewPartitions_t *dst;
+
+ dst = rd_kafka_NewPartitions_new(src->topic, src->total_cnt, NULL, 0);
+
+ rd_list_destroy(&dst->replicas); /* created in .._new() */
+ rd_list_init_copy(&dst->replicas, &src->replicas);
+ rd_list_copy_to(&dst->replicas, &src->replicas,
+ rd_list_copy_preallocated, NULL);
+
+ return dst;
+}
+
+void rd_kafka_NewPartitions_destroy (rd_kafka_NewPartitions_t *newps) {
+ rd_list_destroy(&newps->replicas);
+ rd_free(newps);
+}
+
+static void rd_kafka_NewPartitions_free (void *ptr) {
+ rd_kafka_NewPartitions_destroy(ptr);
+}
+
+
+void rd_kafka_NewPartitions_destroy_array (rd_kafka_NewPartitions_t **newps,
+ size_t newps_cnt) {
+ size_t i;
+ for (i = 0 ; i < newps_cnt ; i++)
+ rd_kafka_NewPartitions_destroy(newps[i]);
+}
+
+
+
+
+
+rd_kafka_resp_err_t
+rd_kafka_NewPartitions_set_replica_assignment (rd_kafka_NewPartitions_t *newp,
+ int32_t new_partition_idx,
+ int32_t *broker_ids,
+ size_t broker_id_cnt,
+ char *errstr,
+ size_t errstr_size) {
+ rd_list_t *rl;
+ int i;
+
+ /* Replica partitions must be added consecutively starting from 0. */
+ if (new_partition_idx != rd_list_cnt(&newp->replicas)) {
+ rd_snprintf(errstr, errstr_size,
+ "Partitions must be added in order, "
+ "starting at 0: expecting partition "
+ "index %d, not %"PRId32,
+ rd_list_cnt(&newp->replicas), new_partition_idx);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ if (broker_id_cnt > RD_KAFKAP_BROKERS_MAX) {
+ rd_snprintf(errstr, errstr_size,
+ "Too many brokers specified "
+ "(RD_KAFKAP_BROKERS_MAX=%d)",
+ RD_KAFKAP_BROKERS_MAX);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ rl = rd_list_init_int32(rd_list_new(0, NULL), (int)broker_id_cnt);
+
+ for (i = 0 ; i < (int)broker_id_cnt ; i++)
+ rd_list_set_int32(rl, i, broker_ids[i]);
+
+ rd_list_add(&newp->replicas, rl);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+
+
+/**
+ * @brief Parse CreatePartitionsResponse and create ADMIN_RESULT op.
+ */
+static rd_kafka_resp_err_t
+rd_kafka_CreatePartitionsResponse_parse (rd_kafka_op_t *rko_req,
+ rd_kafka_op_t **rko_resultp,
+ rd_kafka_buf_t *reply,
+ char *errstr,
+ size_t errstr_size) {
+ const int log_decode_errors = LOG_ERR;
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ rd_kafka_broker_t *rkb = reply->rkbuf_rkb;
+ rd_kafka_t *rk = rkb->rkb_rk;
+ rd_kafka_op_t *rko_result = NULL;
+ int32_t topic_cnt;
+ int i;
+ int32_t Throttle_Time;
+
+ rd_kafka_buf_read_i32(reply, &Throttle_Time);
+ rd_kafka_op_throttle_time(rkb, rk->rk_rep, Throttle_Time);
+
+ /* #topics */
+ rd_kafka_buf_read_i32(reply, &topic_cnt);
+
+ if (topic_cnt > rd_list_cnt(&rko_req->rko_u.admin_request.args))
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Received %"PRId32" topics in response "
+ "when only %d were requested", topic_cnt,
+ rd_list_cnt(&rko_req->rko_u.admin_request.args));
+
+ rko_result = rd_kafka_admin_result_new(rko_req);
+
+ rd_list_init(&rko_result->rko_u.admin_result.results, topic_cnt,
+ rd_kafka_topic_result_free);
+
+ for (i = 0 ; i < (int)topic_cnt ; i++) {
+ rd_kafkap_str_t ktopic;
+ int16_t error_code;
+ char *errstr = NULL;
+ rd_kafka_topic_result_t *terr;
+ rd_kafka_NewTopic_t skel;
+ rd_kafkap_str_t error_msg;
+ int orig_pos;
+
+ rd_kafka_buf_read_str(reply, &ktopic);
+ rd_kafka_buf_read_i16(reply, &error_code);
+ rd_kafka_buf_read_str(reply, &error_msg);
+
+ /* For non-blocking CreatePartitionsRequests the broker
+ * will returned REQUEST_TIMED_OUT for topics
+ * that were triggered for creation -
+ * we hide this error code from the application
+ * since the topic creation is in fact in progress. */
+ if (error_code == RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT &&
+ rd_kafka_confval_get_int(&rko_req->rko_u.
+ admin_request.options.
+ operation_timeout) <= 0) {
+ error_code = RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+
+ if (error_code) {
+ if (RD_KAFKAP_STR_IS_NULL(&error_msg) ||
+ RD_KAFKAP_STR_LEN(&error_msg) == 0)
+ errstr = (char *)rd_kafka_err2str(error_code);
+ else
+ RD_KAFKAP_STR_DUPA(&errstr, &error_msg);
+ }
+
+
+ terr = rd_kafka_topic_result_new(ktopic.str,
+ RD_KAFKAP_STR_LEN(&ktopic),
+ error_code,
+ error_code ?
+ rd_kafka_err2str(error_code) :
+ NULL);
+
+ /* As a convenience to the application we insert topic result
+ * in the same order as they were requested. The broker
+ * does not maintain ordering unfortunately. */
+ skel.topic = terr->topic;
+ orig_pos = rd_list_index(&rko_req->rko_u.admin_request.args,
+ &skel, rd_kafka_NewPartitions_cmp);
+ if (orig_pos == -1) {
+ rd_kafka_topic_result_destroy(terr);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned topic %.*s that was not "
+ "included in the original request",
+ RD_KAFKAP_STR_PR(&ktopic));
+ }
+
+ if (rd_list_elem(&rko_result->rko_u.admin_result.results,
+ orig_pos) != NULL) {
+ rd_kafka_topic_result_destroy(terr);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned topic %.*s multiple times",
+ RD_KAFKAP_STR_PR(&ktopic));
+ }
+
+ rd_list_set(&rko_result->rko_u.admin_result.results, orig_pos,
+ terr);
+ }
+
+ *rko_resultp = rko_result;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ err_parse:
+ if (rko_result)
+ rd_kafka_op_destroy(rko_result);
+
+ rd_snprintf(errstr, errstr_size,
+ "CreatePartitions response protocol parse failure: %s",
+ rd_kafka_err2str(err));
+
+ return err;
+}
+
+
+
+
+
+
+
+void rd_kafka_CreatePartitions (rd_kafka_t *rk,
+ rd_kafka_NewPartitions_t **newps,
+ size_t newps_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu) {
+ rd_kafka_op_t *rko;
+ size_t i;
+ static const struct rd_kafka_admin_worker_cbs cbs = {
+ rd_kafka_CreatePartitionsRequest,
+ rd_kafka_CreatePartitionsResponse_parse,
+ };
+
+ rko = rd_kafka_admin_request_op_new(
+ rk,
+ RD_KAFKA_OP_CREATEPARTITIONS,
+ RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT,
+ &cbs, options, rkqu);
+
+ rd_list_init(&rko->rko_u.admin_request.args, (int)newps_cnt,
+ rd_kafka_NewPartitions_free);
+
+ for (i = 0 ; i < newps_cnt ; i++)
+ rd_list_add(&rko->rko_u.admin_request.args,
+ rd_kafka_NewPartitions_copy(newps[i]));
+
+ rd_kafka_q_enq(rk->rk_ops, rko);
+}
+
+
+/**
+ * @brief Get an array of topic results from a CreatePartitions result.
+ *
+ * The returned \p topics life-time is the same as the \p result object.
+ * @param cntp is updated to the number of elements in the array.
+ */
+const rd_kafka_topic_result_t **
+rd_kafka_CreatePartitions_result_topics (
+ const rd_kafka_CreatePartitions_result_t *result,
+ size_t *cntp) {
+ return rd_kafka_admin_result_ret_topics((const rd_kafka_op_t *)result,
+ cntp);
+}
+
+/**@}*/
+
+
+
+
+/**
+ * @name ConfigEntry
+ * @{
+ *
+ *
+ *
+ */
+
+static void rd_kafka_ConfigEntry_destroy (rd_kafka_ConfigEntry_t *entry) {
+ rd_strtup_destroy(entry->kv);
+ rd_list_destroy(&entry->synonyms);
+ rd_free(entry);
+}
+
+
+static void rd_kafka_ConfigEntry_free (void *ptr) {
+ rd_kafka_ConfigEntry_destroy((rd_kafka_ConfigEntry_t *)ptr);
+}
+
+
+/**
+ * @brief Create new ConfigEntry
+ *
+ * @param name Config entry name
+ * @param name_len Length of name, or -1 to use strlen()
+ * @param value Config entry value, or NULL
+ * @param value_len Length of value, or -1 to use strlen()
+ */
+static rd_kafka_ConfigEntry_t *
+rd_kafka_ConfigEntry_new0 (const char *name, size_t name_len,
+ const char *value, size_t value_len) {
+ rd_kafka_ConfigEntry_t *entry;
+
+ if (!name)
+ return NULL;
+
+ entry = rd_calloc(1, sizeof(*entry));
+ entry->kv = rd_strtup_new0(name, name_len, value, value_len);
+
+ rd_list_init(&entry->synonyms, 0, rd_kafka_ConfigEntry_free);
+
+ entry->a.source = RD_KAFKA_CONFIG_SOURCE_UNKNOWN_CONFIG;
+
+ return entry;
+}
+
+/**
+ * @sa rd_kafka_ConfigEntry_new0
+ */
+static rd_kafka_ConfigEntry_t *
+rd_kafka_ConfigEntry_new (const char *name, const char *value) {
+ return rd_kafka_ConfigEntry_new0(name, -1, value, -1);
+}
+
+
+
+
+/**
+ * @brief Allocate a new AlterConfigs and make a copy of \p src
+ */
+static rd_kafka_ConfigEntry_t *
+rd_kafka_ConfigEntry_copy (const rd_kafka_ConfigEntry_t *src) {
+ rd_kafka_ConfigEntry_t *dst;
+
+ dst = rd_kafka_ConfigEntry_new(src->kv->name, src->kv->value);
+ dst->a = src->a;
+
+ rd_list_destroy(&dst->synonyms); /* created in .._new() */
+ rd_list_init_copy(&dst->synonyms, &src->synonyms);
+ rd_list_copy_to(&dst->synonyms, &src->synonyms,
+ rd_kafka_ConfigEntry_list_copy, NULL);
+
+ return dst;
+}
+
+static void *rd_kafka_ConfigEntry_list_copy (const void *src, void *opaque) {
+ return rd_kafka_ConfigEntry_copy((const rd_kafka_ConfigEntry_t *)src);
+}
+
+
+const char *rd_kafka_ConfigEntry_name (const rd_kafka_ConfigEntry_t *entry) {
+ return entry->kv->name;
+}
+
+const char *
+rd_kafka_ConfigEntry_value (const rd_kafka_ConfigEntry_t *entry) {
+ return entry->kv->value;
+}
+
+rd_kafka_ConfigSource_t
+rd_kafka_ConfigEntry_source (const rd_kafka_ConfigEntry_t *entry) {
+ return entry->a.source;
+}
+
+int rd_kafka_ConfigEntry_is_read_only (const rd_kafka_ConfigEntry_t *entry) {
+ return entry->a.is_readonly;
+}
+
+int rd_kafka_ConfigEntry_is_default (const rd_kafka_ConfigEntry_t *entry) {
+ return entry->a.is_default;
+}
+
+int rd_kafka_ConfigEntry_is_sensitive (const rd_kafka_ConfigEntry_t *entry) {
+ return entry->a.is_sensitive;
+}
+
+int rd_kafka_ConfigEntry_is_synonym (const rd_kafka_ConfigEntry_t *entry) {
+ return entry->a.is_synonym;
+}
+
+const rd_kafka_ConfigEntry_t **
+rd_kafka_ConfigEntry_synonyms (const rd_kafka_ConfigEntry_t *entry,
+ size_t *cntp) {
+ *cntp = rd_list_cnt(&entry->synonyms);
+ if (!*cntp)
+ return NULL;
+ return (const rd_kafka_ConfigEntry_t **)entry->synonyms.rl_elems;
+
+}
+
+
+/**@}*/
+
+
+
+/**
+ * @name ConfigSource
+ * @{
+ *
+ *
+ *
+ */
+
+const char *
+rd_kafka_ConfigSource_name (rd_kafka_ConfigSource_t confsource) {
+ static const char *names[] = {
+ "UNKNOWN_CONFIG",
+ "DYNAMIC_TOPIC_CONFIG",
+ "DYNAMIC_BROKER_CONFIG",
+ "DYNAMIC_DEFAULT_BROKER_CONFIG",
+ "STATIC_BROKER_CONFIG",
+ "DEFAULT_CONFIG",
+ };
+
+ if ((unsigned int)confsource >=
+ (unsigned int)RD_KAFKA_CONFIG_SOURCE__CNT)
+ return "UNSUPPORTED";
+
+ return names[confsource];
+}
+
+/**@}*/
+
+
+
+/**
+ * @name ConfigResource
+ * @{
+ *
+ *
+ *
+ */
+
+const char *
+rd_kafka_ResourceType_name (rd_kafka_ResourceType_t restype) {
+ static const char *names[] = {
+ "UNKNOWN",
+ "ANY",
+ "TOPIC",
+ "GROUP",
+ "BROKER",
+ };
+
+ if ((unsigned int)restype >=
+ (unsigned int)RD_KAFKA_RESOURCE__CNT)
+ return "UNSUPPORTED";
+
+ return names[restype];
+}
+
+
+rd_kafka_ConfigResource_t *
+rd_kafka_ConfigResource_new (rd_kafka_ResourceType_t restype,
+ const char *resname) {
+ rd_kafka_ConfigResource_t *config;
+ size_t namesz = resname ? strlen(resname) : 0;
+
+ if (!namesz || (int)restype < 0)
+ return NULL;
+
+ config = rd_calloc(1, sizeof(*config) + namesz + 1);
+ config->name = config->data;
+ memcpy(config->name, resname, namesz + 1);
+ config->restype = restype;
+
+ rd_list_init(&config->config, 8, rd_kafka_ConfigEntry_free);
+
+ return config;
+}
+
+void rd_kafka_ConfigResource_destroy (rd_kafka_ConfigResource_t *config) {
+ rd_list_destroy(&config->config);
+ if (config->errstr)
+ rd_free(config->errstr);
+ rd_free(config);
+}
+
+static void rd_kafka_ConfigResource_free (void *ptr) {
+ rd_kafka_ConfigResource_destroy((rd_kafka_ConfigResource_t *)ptr);
+}
+
+
+void rd_kafka_ConfigResource_destroy_array (rd_kafka_ConfigResource_t **config,
+ size_t config_cnt) {
+ size_t i;
+ for (i = 0 ; i < config_cnt ; i++)
+ rd_kafka_ConfigResource_destroy(config[i]);
+}
+
+
+/**
+ * @brief Type and name comparator for ConfigResource_t
+ */
+static int rd_kafka_ConfigResource_cmp (const void *_a, const void *_b) {
+ const rd_kafka_ConfigResource_t *a = _a, *b = _b;
+ if (a->restype != b->restype)
+ return a->restype - b->restype;
+ return strcmp(a->name, b->name);
+}
+
+/**
+ * @brief Allocate a new AlterConfigs and make a copy of \p src
+ */
+static rd_kafka_ConfigResource_t *
+rd_kafka_ConfigResource_copy (const rd_kafka_ConfigResource_t *src) {
+ rd_kafka_ConfigResource_t *dst;
+
+ dst = rd_kafka_ConfigResource_new(src->restype, src->name);
+
+ rd_list_destroy(&dst->config); /* created in .._new() */
+ rd_list_init_copy(&dst->config, &src->config);
+ rd_list_copy_to(&dst->config, &src->config,
+ rd_kafka_ConfigEntry_list_copy, NULL);
+
+ return dst;
+}
+
+
+static void
+rd_kafka_ConfigResource_add_ConfigEntry (rd_kafka_ConfigResource_t *config,
+ rd_kafka_ConfigEntry_t *entry) {
+ rd_list_add(&config->config, entry);
+}
+
+
+rd_kafka_resp_err_t
+rd_kafka_ConfigResource_add_config (rd_kafka_ConfigResource_t *config,
+ const char *name, const char *value) {
+ if (!name || !*name || !value)
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+
+ return rd_kafka_admin_add_config0(&config->config, name, value,
+ RD_KAFKA_ALTER_OP_ADD);
+}
+
+rd_kafka_resp_err_t
+rd_kafka_ConfigResource_set_config (rd_kafka_ConfigResource_t *config,
+ const char *name, const char *value) {
+ if (!name || !*name || !value)
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+
+ return rd_kafka_admin_add_config0(&config->config, name, value,
+ RD_KAFKA_ALTER_OP_SET);
+}
+
+rd_kafka_resp_err_t
+rd_kafka_ConfigResource_delete_config (rd_kafka_ConfigResource_t *config,
+ const char *name) {
+ if (!name || !*name)
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+
+ return rd_kafka_admin_add_config0(&config->config, name, NULL,
+ RD_KAFKA_ALTER_OP_DELETE);
+}
+
+
+const rd_kafka_ConfigEntry_t **
+rd_kafka_ConfigResource_configs (const rd_kafka_ConfigResource_t *config,
+ size_t *cntp) {
+ *cntp = rd_list_cnt(&config->config);
+ if (!*cntp)
+ return NULL;
+ return (const rd_kafka_ConfigEntry_t **)config->config.rl_elems;
+}
+
+
+
+
+rd_kafka_ResourceType_t
+rd_kafka_ConfigResource_type (const rd_kafka_ConfigResource_t *config) {
+ return config->restype;
+}
+
+const char *
+rd_kafka_ConfigResource_name (const rd_kafka_ConfigResource_t *config) {
+ return config->name;
+}
+
+rd_kafka_resp_err_t
+rd_kafka_ConfigResource_error (const rd_kafka_ConfigResource_t *config) {
+ return config->err;
+}
+
+const char *
+rd_kafka_ConfigResource_error_string (const rd_kafka_ConfigResource_t *config) {
+ if (!config->err)
+ return NULL;
+ if (config->errstr)
+ return config->errstr;
+ return rd_kafka_err2str(config->err);
+}
+
+
+/**
+ * @brief Look in the provided ConfigResource_t* list for a resource of
+ * type BROKER and set its broker id in \p broker_id, returning
+ * RD_KAFKA_RESP_ERR_NO_ERROR.
+ *
+ * If multiple BROKER resources are found RD_KAFKA_RESP_ERR__CONFLICT
+ * is returned and an error string is written to errstr.
+ *
+ * If no BROKER resources are found RD_KAFKA_RESP_ERR_NO_ERROR
+ * is returned and \p broker_idp is set to -1.
+ */
+static rd_kafka_resp_err_t
+rd_kafka_ConfigResource_get_single_broker_id (const rd_list_t *configs,
+ int32_t *broker_idp,
+ char *errstr,
+ size_t errstr_size) {
+ const rd_kafka_ConfigResource_t *config;
+ int i;
+ int32_t broker_id = -1;
+
+ RD_LIST_FOREACH(config, configs, i) {
+ char *endptr;
+ long int r;
+
+ if (config->restype != RD_KAFKA_RESOURCE_BROKER)
+ continue;
+
+ if (broker_id != -1) {
+ rd_snprintf(errstr, errstr_size,
+ "Only one ConfigResource of type BROKER "
+ "is allowed per call");
+ return RD_KAFKA_RESP_ERR__CONFLICT;
+ }
+
+ /* Convert string broker-id to int32 */
+ r = (int32_t)strtol(config->name, &endptr, 10);
+ if (r == LONG_MIN || r == LONG_MAX || config->name == endptr ||
+ r < 0) {
+ rd_snprintf(errstr, errstr_size,
+ "Expected an int32 broker_id for "
+ "ConfigResource(type=BROKER, name=%s)",
+ config->name);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ broker_id = r;
+
+ /* Keep scanning to make sure there are no duplicate
+ * BROKER resources. */
+ }
+
+ *broker_idp = broker_id;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+/**@}*/
+
+
+
+/**
+ * @name AlterConfigs
+ * @{
+ *
+ *
+ *
+ */
+
+
+
+/**
+ * @brief Parse AlterConfigsResponse and create ADMIN_RESULT op.
+ */
+static rd_kafka_resp_err_t
+rd_kafka_AlterConfigsResponse_parse (rd_kafka_op_t *rko_req,
+ rd_kafka_op_t **rko_resultp,
+ rd_kafka_buf_t *reply,
+ char *errstr, size_t errstr_size) {
+ const int log_decode_errors = LOG_ERR;
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ rd_kafka_broker_t *rkb = reply->rkbuf_rkb;
+ rd_kafka_t *rk = rkb->rkb_rk;
+ rd_kafka_op_t *rko_result = NULL;
+ int32_t res_cnt;
+ int i;
+ int32_t Throttle_Time;
+
+ rd_kafka_buf_read_i32(reply, &Throttle_Time);
+ rd_kafka_op_throttle_time(rkb, rk->rk_rep, Throttle_Time);
+
+ rd_kafka_buf_read_i32(reply, &res_cnt);
+
+ if (res_cnt > rd_list_cnt(&rko_req->rko_u.admin_request.args)) {
+ rd_snprintf(errstr, errstr_size,
+ "Received %"PRId32" ConfigResources in response "
+ "when only %d were requested", res_cnt,
+ rd_list_cnt(&rko_req->rko_u.admin_request.args));
+ return RD_KAFKA_RESP_ERR__BAD_MSG;
+ }
+
+ rko_result = rd_kafka_admin_result_new(rko_req);
+
+ rd_list_init(&rko_result->rko_u.admin_result.results, res_cnt,
+ rd_kafka_ConfigResource_free);
+
+ for (i = 0 ; i < (int)res_cnt ; i++) {
+ int16_t error_code;
+ rd_kafkap_str_t error_msg;
+ int8_t res_type;
+ rd_kafkap_str_t kres_name;
+ char *res_name;
+ char *errstr = NULL;
+ rd_kafka_ConfigResource_t *config;
+ rd_kafka_ConfigResource_t skel;
+ int orig_pos;
+
+ rd_kafka_buf_read_i16(reply, &error_code);
+ rd_kafka_buf_read_str(reply, &error_msg);
+ rd_kafka_buf_read_i8(reply, &res_type);
+ rd_kafka_buf_read_str(reply, &kres_name);
+ RD_KAFKAP_STR_DUPA(&res_name, &kres_name);
+
+ if (error_code) {
+ if (RD_KAFKAP_STR_IS_NULL(&error_msg) ||
+ RD_KAFKAP_STR_LEN(&error_msg) == 0)
+ errstr = (char *)rd_kafka_err2str(error_code);
+ else
+ RD_KAFKAP_STR_DUPA(&errstr, &error_msg);
+ }
+
+ config = rd_kafka_ConfigResource_new(res_type, res_name);
+ if (!config) {
+ rd_kafka_log(rko_req->rko_rk, LOG_ERR,
+ "ADMIN", "AlterConfigs returned "
+ "unsupported ConfigResource #%d with "
+ "type %d and name \"%s\": ignoring",
+ i, res_type, res_name);
+ continue;
+ }
+
+ config->err = error_code;
+ if (errstr)
+ config->errstr = rd_strdup(errstr);
+
+ /* As a convenience to the application we insert result
+ * in the same order as they were requested. The broker
+ * does not maintain ordering unfortunately. */
+ skel.restype = config->restype;
+ skel.name = config->name;
+ orig_pos = rd_list_index(&rko_req->rko_u.admin_request.args,
+ &skel, rd_kafka_ConfigResource_cmp);
+ if (orig_pos == -1) {
+ rd_kafka_ConfigResource_destroy(config);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned ConfigResource %d,%s "
+ "that was not "
+ "included in the original request",
+ res_type, res_name);
+ }
+
+ if (rd_list_elem(&rko_result->rko_u.admin_result.results,
+ orig_pos) != NULL) {
+ rd_kafka_ConfigResource_destroy(config);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned ConfigResource %d,%s "
+ "multiple times",
+ res_type, res_name);
+ }
+
+ rd_list_set(&rko_result->rko_u.admin_result.results, orig_pos,
+ config);
+ }
+
+ *rko_resultp = rko_result;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ err_parse:
+ if (rko_result)
+ rd_kafka_op_destroy(rko_result);
+
+ rd_snprintf(errstr, errstr_size,
+ "AlterConfigs response protocol parse failure: %s",
+ rd_kafka_err2str(err));
+
+ return err;
+}
+
+
+
+
+void rd_kafka_AlterConfigs (rd_kafka_t *rk,
+ rd_kafka_ConfigResource_t **configs,
+ size_t config_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu) {
+ rd_kafka_op_t *rko;
+ size_t i;
+ rd_kafka_resp_err_t err;
+ char errstr[256];
+ static const struct rd_kafka_admin_worker_cbs cbs = {
+ rd_kafka_AlterConfigsRequest,
+ rd_kafka_AlterConfigsResponse_parse,
+ };
+
+ rko = rd_kafka_admin_request_op_new(
+ rk,
+ RD_KAFKA_OP_ALTERCONFIGS,
+ RD_KAFKA_EVENT_ALTERCONFIGS_RESULT,
+ &cbs, options, rkqu);
+
+ rd_list_init(&rko->rko_u.admin_request.args, (int)config_cnt,
+ rd_kafka_ConfigResource_free);
+
+ for (i = 0 ; i < config_cnt ; i++)
+ rd_list_add(&rko->rko_u.admin_request.args,
+ rd_kafka_ConfigResource_copy(configs[i]));
+
+ /* If there's a BROKER resource in the list we need to
+ * speak directly to that broker rather than the controller.
+ *
+ * Multiple BROKER resources are not allowed.
+ */
+ err = rd_kafka_ConfigResource_get_single_broker_id(
+ &rko->rko_u.admin_request.args,
+ &rko->rko_u.admin_request.broker_id,
+ errstr, sizeof(errstr));
+ if (err) {
+ rd_kafka_admin_result_fail(rko, err, "%s", errstr);
+ rd_kafka_admin_common_worker_destroy(rk, rko);
+ return;
+ }
+
+ rd_kafka_q_enq(rk->rk_ops, rko);
+}
+
+
+const rd_kafka_ConfigResource_t **
+rd_kafka_AlterConfigs_result_resources (
+ const rd_kafka_AlterConfigs_result_t *result,
+ size_t *cntp) {
+ return rd_kafka_admin_result_ret_resources(
+ (const rd_kafka_op_t *)result, cntp);
+}
+
+/**@}*/
+
+
+
+
+/**
+ * @name DescribeConfigs
+ * @{
+ *
+ *
+ *
+ */
+
+
+/**
+ * @brief Parse DescribeConfigsResponse and create ADMIN_RESULT op.
+ */
+static rd_kafka_resp_err_t
+rd_kafka_DescribeConfigsResponse_parse (rd_kafka_op_t *rko_req,
+ rd_kafka_op_t **rko_resultp,
+ rd_kafka_buf_t *reply,
+ char *errstr, size_t errstr_size) {
+ const int log_decode_errors = LOG_ERR;
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ rd_kafka_broker_t *rkb = reply->rkbuf_rkb;
+ rd_kafka_t *rk = rkb->rkb_rk;
+ rd_kafka_op_t *rko_result = NULL;
+ int32_t res_cnt;
+ int i;
+ int32_t Throttle_Time;
+ rd_kafka_ConfigResource_t *config = NULL;
+ rd_kafka_ConfigEntry_t *entry = NULL;
+
+ rd_kafka_buf_read_i32(reply, &Throttle_Time);
+ rd_kafka_op_throttle_time(rkb, rk->rk_rep, Throttle_Time);
+
+ /* #resources */
+ rd_kafka_buf_read_i32(reply, &res_cnt);
+
+ if (res_cnt > rd_list_cnt(&rko_req->rko_u.admin_request.args))
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Received %"PRId32" ConfigResources in response "
+ "when only %d were requested", res_cnt,
+ rd_list_cnt(&rko_req->rko_u.admin_request.args));
+
+ rko_result = rd_kafka_admin_result_new(rko_req);
+
+ rd_list_init(&rko_result->rko_u.admin_result.results, res_cnt,
+ rd_kafka_ConfigResource_free);
+
+ for (i = 0 ; i < (int)res_cnt ; i++) {
+ int16_t error_code;
+ rd_kafkap_str_t error_msg;
+ int8_t res_type;
+ rd_kafkap_str_t kres_name;
+ char *res_name;
+ char *errstr = NULL;
+ rd_kafka_ConfigResource_t skel;
+ int orig_pos;
+ int32_t entry_cnt;
+ int ci;
+
+ rd_kafka_buf_read_i16(reply, &error_code);
+ rd_kafka_buf_read_str(reply, &error_msg);
+ rd_kafka_buf_read_i8(reply, &res_type);
+ rd_kafka_buf_read_str(reply, &kres_name);
+ RD_KAFKAP_STR_DUPA(&res_name, &kres_name);
+
+ if (error_code) {
+ if (RD_KAFKAP_STR_IS_NULL(&error_msg) ||
+ RD_KAFKAP_STR_LEN(&error_msg) == 0)
+ errstr = (char *)rd_kafka_err2str(error_code);
+ else
+ RD_KAFKAP_STR_DUPA(&errstr, &error_msg);
+ }
+
+ config = rd_kafka_ConfigResource_new(res_type, res_name);
+ if (!config) {
+ rd_kafka_log(rko_req->rko_rk, LOG_ERR,
+ "ADMIN", "DescribeConfigs returned "
+ "unsupported ConfigResource #%d with "
+ "type %d and name \"%s\": ignoring",
+ i, res_type, res_name);
+ continue;
+ }
+
+ config->err = error_code;
+ if (errstr)
+ config->errstr = rd_strdup(errstr);
+
+ /* #config_entries */
+ rd_kafka_buf_read_i32(reply, &entry_cnt);
+
+ for (ci = 0 ; ci < (int)entry_cnt ; ci++) {
+ rd_kafkap_str_t config_name, config_value;
+ int32_t syn_cnt;
+ int si;
+
+ rd_kafka_buf_read_str(reply, &config_name);
+ rd_kafka_buf_read_str(reply, &config_value);
+
+ entry = rd_kafka_ConfigEntry_new0(
+ config_name.str,
+ RD_KAFKAP_STR_LEN(&config_name),
+ config_value.str,
+ RD_KAFKAP_STR_LEN(&config_value));
+
+ rd_kafka_buf_read_bool(reply, &entry->a.is_readonly);
+
+ /* ApiVersion 0 has is_default field, while
+ * ApiVersion 1 has source field.
+ * Convert between the two so they look the same
+ * to the caller. */
+ if (rd_kafka_buf_ApiVersion(reply) == 0) {
+ rd_kafka_buf_read_bool(reply,
+ &entry->a.is_default);
+ if (entry->a.is_default)
+ entry->a.source =
+ RD_KAFKA_CONFIG_SOURCE_DEFAULT_CONFIG;
+ } else {
+ int8_t config_source;
+ rd_kafka_buf_read_i8(reply, &config_source);
+ entry->a.source = config_source;
+
+ if (entry->a.source ==
+ RD_KAFKA_CONFIG_SOURCE_DEFAULT_CONFIG)
+ entry->a.is_default = 1;
+
+ }
+
+ rd_kafka_buf_read_bool(reply, &entry->a.is_sensitive);
+
+
+ if (rd_kafka_buf_ApiVersion(reply) == 1) {
+ /* #config_synonyms (ApiVersion 1) */
+ rd_kafka_buf_read_i32(reply, &syn_cnt);
+
+ if (syn_cnt > 100000)
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned %"PRId32
+ " config synonyms for "
+ "ConfigResource %d,%s: "
+ "limit is 100000",
+ syn_cnt,
+ config->restype,
+ config->name);
+
+ if (syn_cnt > 0)
+ rd_list_grow(&entry->synonyms, syn_cnt);
+
+ } else {
+ /* No synonyms in ApiVersion 0 */
+ syn_cnt = 0;
+ }
+
+
+
+ /* Read synonyms (ApiVersion 1) */
+ for (si = 0 ; si < (int)syn_cnt ; si++) {
+ rd_kafkap_str_t syn_name, syn_value;
+ int8_t syn_source;
+ rd_kafka_ConfigEntry_t *syn_entry;
+
+ rd_kafka_buf_read_str(reply, &syn_name);
+ rd_kafka_buf_read_str(reply, &syn_value);
+ rd_kafka_buf_read_i8(reply, &syn_source);
+
+ syn_entry = rd_kafka_ConfigEntry_new0(
+ syn_name.str,
+ RD_KAFKAP_STR_LEN(&syn_name),
+ syn_value.str,
+ RD_KAFKAP_STR_LEN(&syn_value));
+ if (!syn_entry)
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned invalid "
+ "synonym #%d "
+ "for ConfigEntry #%d (%s) "
+ "and ConfigResource %d,%s: "
+ "syn_name.len %d, "
+ "syn_value.len %d",
+ si, ci, entry->kv->name,
+ config->restype, config->name,
+ (int)syn_name.len,
+ (int)syn_value.len);
+
+ syn_entry->a.source = syn_source;
+ syn_entry->a.is_synonym = 1;
+
+ rd_list_add(&entry->synonyms, syn_entry);
+ }
+
+ rd_kafka_ConfigResource_add_ConfigEntry(
+ config, entry);
+ entry = NULL;
+ }
+
+ /* As a convenience to the application we insert result
+ * in the same order as they were requested. The broker
+ * does not maintain ordering unfortunately. */
+ skel.restype = config->restype;
+ skel.name = config->name;
+ orig_pos = rd_list_index(&rko_req->rko_u.admin_request.args,
+ &skel, rd_kafka_ConfigResource_cmp);
+ if (orig_pos == -1) {
+ rd_kafka_ConfigResource_destroy(config);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned ConfigResource %d,%s "
+ "that was not "
+ "included in the original request",
+ res_type, res_name);
+ }
+
+ if (rd_list_elem(&rko_result->rko_u.admin_result.results,
+ orig_pos) != NULL) {
+ rd_kafka_ConfigResource_destroy(config);
+ rd_kafka_buf_parse_fail(
+ reply,
+ "Broker returned ConfigResource %d,%s "
+ "multiple times",
+ res_type, res_name);
+ }
+
+ rd_list_set(&rko_result->rko_u.admin_result.results, orig_pos,
+ config);
+ config = NULL;
+ }
+
+ *rko_resultp = rko_result;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ err_parse:
+ if (entry)
+ rd_kafka_ConfigEntry_destroy(entry);
+ if (config)
+ rd_kafka_ConfigResource_destroy(config);
+
+ if (rko_result)
+ rd_kafka_op_destroy(rko_result);
+
+ rd_snprintf(errstr, errstr_size,
+ "DescribeConfigs response protocol parse failure: %s",
+ rd_kafka_err2str(err));
+
+ return err;
+}
+
+
+
+void rd_kafka_DescribeConfigs (rd_kafka_t *rk,
+ rd_kafka_ConfigResource_t **configs,
+ size_t config_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu) {
+ rd_kafka_op_t *rko;
+ size_t i;
+ rd_kafka_resp_err_t err;
+ char errstr[256];
+ static const struct rd_kafka_admin_worker_cbs cbs = {
+ rd_kafka_DescribeConfigsRequest,
+ rd_kafka_DescribeConfigsResponse_parse,
+ };
+
+ rko = rd_kafka_admin_request_op_new(
+ rk,
+ RD_KAFKA_OP_DESCRIBECONFIGS,
+ RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT,
+ &cbs, options, rkqu);
+
+ rd_list_init(&rko->rko_u.admin_request.args, (int)config_cnt,
+ rd_kafka_ConfigResource_free);
+
+ for (i = 0 ; i < config_cnt ; i++)
+ rd_list_add(&rko->rko_u.admin_request.args,
+ rd_kafka_ConfigResource_copy(configs[i]));
+
+ /* If there's a BROKER resource in the list we need to
+ * speak directly to that broker rather than the controller.
+ *
+ * Multiple BROKER resources are not allowed.
+ */
+ err = rd_kafka_ConfigResource_get_single_broker_id(
+ &rko->rko_u.admin_request.args,
+ &rko->rko_u.admin_request.broker_id,
+ errstr, sizeof(errstr));
+ if (err) {
+ rd_kafka_admin_result_fail(rko, err, "%s", errstr);
+ rd_kafka_admin_common_worker_destroy(rk, rko);
+ return;
+ }
+
+ rd_kafka_q_enq(rk->rk_ops, rko);
+}
+
+
+
+
+const rd_kafka_ConfigResource_t **
+rd_kafka_DescribeConfigs_result_resources (
+ const rd_kafka_DescribeConfigs_result_t *result,
+ size_t *cntp) {
+ return rd_kafka_admin_result_ret_resources(
+ (const rd_kafka_op_t *)result, cntp);
+}
+
+/**@}*/
diff -Nru librdkafka-0.11.3/src/rdkafka_admin.h librdkafka-0.11.6/src/rdkafka_admin.h
--- librdkafka-0.11.3/src/rdkafka_admin.h 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_admin.h 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,266 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RDKAFKA_ADMIN_H_
+#define _RDKAFKA_ADMIN_H_
+
+
+#include "rdstring.h"
+#include "rdkafka_confval.h"
+
+
+
+/**
+ * @brief Common AdminOptions type used for all admin APIs.
+ *
+ * @remark Visit AdminOptions_use() when you change this struct
+ * to make sure it is copied properly.
+ */
+struct rd_kafka_AdminOptions_s {
+ rd_kafka_admin_op_t for_api; /**< Limit allowed options to
+ * this API (optional) */
+
+ /* Generic */
+ rd_kafka_confval_t request_timeout;/**< I32: Full request timeout,
+ * includes looking up leader
+ * broker,
+ * waiting for req/response,
+ * etc. */
+ rd_ts_t abs_timeout; /**< Absolute timeout calculated
+ * from .timeout */
+
+ /* Specific for one or more APIs */
+ rd_kafka_confval_t operation_timeout; /**< I32: Timeout on broker.
+ * Valid for:
+ * CreateTopics
+ * DeleteTopics
+ */
+ rd_kafka_confval_t validate_only; /**< BOOL: Only validate (on broker),
+ * but don't perform action.
+ * Valid for:
+ * CreateTopics
+ * CreatePartitions
+ * AlterConfigs
+ */
+
+ rd_kafka_confval_t incremental; /**< BOOL: Incremental rather than
+ * absolute application
+ * of config.
+ * Valid for:
+ * AlterConfigs
+ */
+
+ rd_kafka_confval_t broker; /**< INT: Explicitly override
+ * broker id to send
+ * requests to.
+ * Valid for:
+ * all
+ */
+
+ rd_kafka_confval_t opaque; /**< PTR: Application opaque.
+ * Valid for all. */
+};
+
+
+
+
+
+/**
+ * @name CreateTopics
+ * @{
+ */
+
+/**
+ * @brief NewTopic type, used with CreateTopics.
+ */
+struct rd_kafka_NewTopic_s {
+ /* Required */
+ char *topic; /**< Topic to be created */
+ int num_partitions; /**< Number of partitions to create */
+ int replication_factor; /**< Replication factor */
+
+ /* Optional */
+ rd_list_t replicas; /**< Type (rd_list_t (int32_t)):
+ * Array of replica lists indexed by
+ * partition, size num_partitions. */
+ rd_list_t config; /**< Type (rd_kafka_ConfigEntry_t *):
+ * List of configuration entries */
+};
+
+/**@}*/
+
+
+/**
+ * @name DeleteTopics
+ * @{
+ */
+
+/**
+ * @brief DeleteTopics result
+ */
+struct rd_kafka_DeleteTopics_result_s {
+ rd_list_t topics; /**< Type (rd_kafka_topic_result_t *) */
+};
+
+struct rd_kafka_DeleteTopic_s {
+ char *topic; /**< Points to data */
+ char data[1]; /**< The topic name is allocated along with
+ * the struct here. */
+};
+
+/**@}*/
+
+
+
+/**
+ * @name CreatePartitions
+ * @{
+ */
+
+
+/**
+ * @brief CreatePartitions result
+ */
+struct rd_kafka_CreatePartitions_result_s {
+ rd_list_t topics; /**< Type (rd_kafka_topic_result_t *) */
+};
+
+struct rd_kafka_NewPartitions_s {
+ char *topic; /**< Points to data */
+ size_t total_cnt; /**< New total partition count */
+
+ /* Optional */
+ rd_list_t replicas; /**< Type (rd_list_t (int32_t)):
+ * Array of replica lists indexed by
+ * new partition relative index.
+ * Size is dynamic since we don't
+ * know how many partitions are actually
+ * being added by total_cnt */
+
+ char data[1]; /**< The topic name is allocated along with
+ * the struct here. */
+};
+
+/**@}*/
+
+
+
+/**
+ * @name ConfigEntry
+ * @{
+ */
+
+/* KIP-248 */
+typedef enum rd_kafka_AlterOperation_t {
+ RD_KAFKA_ALTER_OP_ADD = 0,
+ RD_KAFKA_ALTER_OP_SET = 1,
+ RD_KAFKA_ALTER_OP_DELETE = 2,
+} rd_kafka_AlterOperation_t;
+
+struct rd_kafka_ConfigEntry_s {
+ rd_strtup_t *kv; /**< Name/Value pair */
+
+ /* Response */
+
+ /* Attributes: this is a struct for easy copying */
+ struct {
+ rd_kafka_AlterOperation_t operation; /**< Operation */
+ rd_kafka_ConfigSource_t source; /**< Config source */
+ rd_bool_t is_readonly; /**< Value is read-only (on broker) */
+ rd_bool_t is_default; /**< Value is at its default */
+ rd_bool_t is_sensitive; /**< Value is sensitive */
+ rd_bool_t is_synonym; /**< Value is synonym */
+ } a;
+
+ rd_list_t synonyms; /**< Type (rd_kafka_configEntry *) */
+};
+
+/**
+ * @brief A cluster ConfigResource constisting of:
+ * - resource type (BROKER, TOPIC)
+ * - configuration property name
+ * - configuration property value
+ *
+ * https://cwiki.apache.org/confluence/display/KAFKA/KIP-133%3A+Describe+and+Alter+Configs+Admin+APIs
+ */
+struct rd_kafka_ConfigResource_s {
+ rd_kafka_ResourceType_t restype; /**< Resource type */
+ char *name; /**< Resource name, points to .data*/
+ rd_list_t config; /**< Type (rd_kafka_ConfigEntry_t *):
+ * List of config props */
+
+ /* Response */
+ rd_kafka_resp_err_t err; /**< Response error code */
+ char *errstr; /**< Response error string */
+
+ char data[1]; /**< The name is allocated along with
+ * the struct here. */
+};
+
+
+
+
+/**@}*/
+
+/**
+ * @name AlterConfigs
+ * @{
+ */
+
+
+
+
+struct rd_kafka_AlterConfigs_result_s {
+ rd_list_t resources; /**< Type (rd_kafka_ConfigResource_t *) */
+};
+
+struct rd_kafka_ConfigResource_result_s {
+ rd_list_t resources; /**< Type (struct rd_kafka_ConfigResource *):
+ * List of config resources, sans config
+ * but with response error values. */
+};
+
+/**@}*/
+
+
+
+/**
+ * @name DescribeConfigs
+ * @{
+ */
+
+struct rd_kafka_DescribeConfigs_result_s {
+ rd_list_t configs; /**< Type (rd_kafka_ConfigResource_t *) */
+};
+
+/**@}*/
+
+
+/**@}*/
+
+#endif /* _RDKAFKA_ADMIN_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_assignor.h librdkafka-0.11.6/src/rdkafka_assignor.h
--- librdkafka-0.11.3/src/rdkafka_assignor.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_assignor.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,9 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
-
-
+#ifndef _RDKAFKA_ASSIGNOR_H_
+#define _RDKAFKA_ASSIGNOR_H_
@@ -157,3 +156,4 @@
char *errstr, size_t errstr_size,
void *opaque);
+#endif /* _RDKAFKA_ASSIGNOR_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_aux.c librdkafka-0.11.6/src/rdkafka_aux.c
--- librdkafka-0.11.3/src/rdkafka_aux.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_aux.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,99 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "rdkafka_int.h"
+#include "rdkafka_aux.h"
+
+rd_kafka_resp_err_t
+rd_kafka_topic_result_error (const rd_kafka_topic_result_t *topicres) {
+ return topicres->err;
+}
+
+const char *
+rd_kafka_topic_result_error_string (const rd_kafka_topic_result_t *topicres) {
+ return topicres->errstr;
+}
+
+const char *
+rd_kafka_topic_result_name (const rd_kafka_topic_result_t *topicres) {
+ return topicres->topic;
+}
+
+/**
+ * @brief Create new topic_result (single allocation).
+ *
+ * @param topic Topic string, if topic_size is != -1 it does not have to
+ * be nul-terminated.
+ * @param topic_size Size of topic, or -1 to perform automatic strlen()
+ * @param err Error code
+ * @param errstr Optional error string.
+ *
+ * All input arguments are copied.
+ */
+
+rd_kafka_topic_result_t *
+rd_kafka_topic_result_new (const char *topic, ssize_t topic_size,
+ rd_kafka_resp_err_t err,
+ const char *errstr) {
+ size_t tlen = topic_size != -1 ? (size_t)topic_size : strlen(topic);
+ size_t elen = errstr ? strlen(errstr) + 1 : 0;
+ rd_kafka_topic_result_t *terr;
+
+ terr = rd_malloc(sizeof(*terr) + tlen + 1 + elen);
+
+ terr->err = err;
+
+ terr->topic = terr->data;
+ memcpy(terr->topic, topic, tlen);
+ terr->topic[tlen] = '\0';
+
+ if (errstr) {
+ terr->errstr = terr->topic + tlen + 1;
+ memcpy(terr->errstr, errstr, elen);
+ } else {
+ terr->errstr = NULL;
+ }
+
+ return terr;
+}
+
+
+/**
+ * @brief Destroy topic_result
+ */
+void rd_kafka_topic_result_destroy (rd_kafka_topic_result_t *terr) {
+ rd_free(terr);
+}
+
+/**
+ * @brief Destroy-variant suitable for rd_list free_cb use.
+ */
+void rd_kafka_topic_result_free (void *ptr) {
+ rd_kafka_topic_result_destroy((rd_kafka_topic_result_t *)ptr);
+}
diff -Nru librdkafka-0.11.3/src/rdkafka_aux.h librdkafka-0.11.6/src/rdkafka_aux.h
--- librdkafka-0.11.3/src/rdkafka_aux.h 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_aux.h 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,64 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RDKAFKA_AUX_H_
+#define _RDKAFKA_AUX_H_
+
+/**
+ * @name Auxiliary types
+ */
+
+#include "rdkafka_conf.h"
+
+
+
+/**
+ * @brief Topic [ + Error code + Error string ]
+ *
+ * @remark Public type.
+ * @remark Single allocation.
+ */
+struct rd_kafka_topic_result_s {
+ char *topic; /**< Points to data */
+ rd_kafka_resp_err_t err; /**< Error code */
+ char *errstr; /**< Points to data after topic, unless NULL */
+ char data[1]; /**< topic followed by errstr */
+};
+
+void rd_kafka_topic_result_destroy (rd_kafka_topic_result_t *terr);
+void rd_kafka_topic_result_free (void *ptr);
+
+rd_kafka_topic_result_t *
+rd_kafka_topic_result_new (const char *topic, ssize_t topic_size,
+ rd_kafka_resp_err_t err,
+ const char *errstr);
+
+
+/**@}*/
+
+#endif /* _RDKAFKA_AUX_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_background.c librdkafka-0.11.6/src/rdkafka_background.c
--- librdkafka-0.11.3/src/rdkafka_background.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_background.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,143 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * Background queue thread and event handling.
+ *
+ * See rdkafka.h's rd_kafka_conf_set_background_event_cb() for details.
+ */
+
+#include "rd.h"
+#include "rdkafka_int.h"
+#include "rdkafka_event.h"
+
+/**
+ * @brief Call the registered background_event_cb.
+ * @locality rdkafka background queue thread
+ */
+static RD_INLINE void
+rd_kafka_call_background_event_cb (rd_kafka_t *rk, rd_kafka_op_t *rko) {
+ rd_assert(!rk->rk_background.calling);
+ rk->rk_background.calling = 1;
+
+ rk->rk_conf.background_event_cb(rk, rko, rk->rk_conf.opaque);
+
+ rk->rk_background.calling = 0;
+}
+
+
+/**
+ * @brief Background queue handler.
+ *
+ * Triggers the background_event_cb for all event:able ops,
+ * for non-event:able ops:
+ * - call op callback if set, else
+ * - log and discard the op. This is a user error, forwarding non-event
+ * APIs to the background queue.
+ */
+static rd_kafka_op_res_t
+rd_kafka_background_queue_serve (rd_kafka_t *rk,
+ rd_kafka_q_t *rkq,
+ rd_kafka_op_t *rko,
+ rd_kafka_q_cb_type_t cb_type,
+ void *opaque) {
+ rd_kafka_op_res_t res;
+
+ /*
+ * Dispatch Event:able ops to background_event_cb()
+ */
+ if (likely(rd_kafka_event_setup(rk, rko))) {
+ rd_kafka_call_background_event_cb(rk, rko);
+ /* Event must be destroyed by application. */
+ return RD_KAFKA_OP_RES_HANDLED;
+ }
+
+ /*
+ * Handle non-event:able ops through the standard poll_cb that
+ * will trigger type-specific callbacks (and return OP_RES_HANDLED)
+ * or do no handling and return OP_RES_PASS
+ */
+ res = rd_kafka_poll_cb(rk, rkq, rko, RD_KAFKA_Q_CB_CALLBACK, opaque);
+ if (res == RD_KAFKA_OP_RES_HANDLED)
+ return res;
+
+ /* Op was not handled, log and destroy it. */
+ rd_kafka_log(rk, LOG_NOTICE, "BGQUEUE",
+ "No support for handling "
+ "non-event op %s in background queue: discarding",
+ rd_kafka_op2str(rko->rko_type));
+ rd_kafka_op_destroy(rko);
+
+ /* Signal yield to q_serve() (implies that the op was handled). */
+ if (res == RD_KAFKA_OP_RES_YIELD)
+ return res;
+
+ /* Indicate that the op was handled. */
+ return RD_KAFKA_OP_RES_HANDLED;
+}
+
+
+/**
+ * @brief Main loop for background queue thread.
+ */
+int rd_kafka_background_thread_main (void *arg) {
+ rd_kafka_t *rk = arg;
+
+ rd_kafka_set_thread_name("background");
+ rd_kafka_set_thread_sysname("rdk:bg");
+
+ (void)rd_atomic32_add(&rd_kafka_thread_cnt_curr, 1);
+
+ /* Acquire lock (which was held by thread creator during creation)
+ * to synchronise state. */
+ rd_kafka_wrlock(rk);
+ rd_kafka_wrunlock(rk);
+
+ while (likely(!rd_kafka_terminating(rk))) {
+ rd_kafka_q_serve(rk->rk_background.q, 10*1000, 0,
+ RD_KAFKA_Q_CB_RETURN,
+ rd_kafka_background_queue_serve, NULL);
+ }
+
+ /* Inform the user that they terminated the client before
+ * all outstanding events were handled. */
+ if (rd_kafka_q_len(rk->rk_background.q) > 0)
+ rd_kafka_log(rk, LOG_INFO, "BGQUEUE",
+ "Purging %d unserved events from background queue",
+ rd_kafka_q_len(rk->rk_background.q));
+ rd_kafka_q_disable(rk->rk_background.q);
+ rd_kafka_q_purge(rk->rk_background.q);
+
+ rd_kafka_dbg(rk, GENERIC, "BGQUEUE",
+ "Background queue thread exiting");
+
+ rd_atomic32_sub(&rd_kafka_thread_cnt_curr, 1);
+
+ return 0;
+}
+
diff -Nru librdkafka-0.11.3/src/rdkafka_broker.c librdkafka-0.11.6/src/rdkafka_broker.c
--- librdkafka-0.11.3/src/rdkafka_broker.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_broker.c 2018-10-10 06:54:38.000000000 +0000
@@ -265,7 +265,7 @@
* is bound to fail once on older brokers. */
if (rd_atomic32_add(&rkb->rkb_rk->rk_broker_down_cnt, 1) ==
rd_atomic32_get(&rkb->rkb_rk->rk_broker_cnt) &&
- !rd_atomic32_get(&rkb->rkb_rk->rk_terminate))
+ !rd_kafka_terminating(rkb->rkb_rk))
rd_kafka_op_err(rkb->rkb_rk,
RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN,
"%i/%i brokers are down",
@@ -365,7 +365,7 @@
*/
if (fmt &&
!(errno_save == EINTR &&
- rd_atomic32_get(&rkb->rkb_rk->rk_terminate)) &&
+ rd_kafka_terminating(rkb->rkb_rk)) &&
!(err == RD_KAFKA_RESP_ERR__TRANSPORT &&
rkb->rkb_state == RD_KAFKA_BROKER_STATE_APIVERSION_QUERY)) {
int of;
@@ -380,12 +380,25 @@
sizeof(rkb->rkb_err.msg)-of, fmt, ap);
va_end(ap);
+ /* Append time since last state change
+ * to help debug connection issues */
+ of = strlen(rkb->rkb_err.msg);
+ if (of + 30 < (int)sizeof(rkb->rkb_err.msg))
+ rd_snprintf(rkb->rkb_err.msg+of,
+ sizeof(rkb->rkb_err.msg)-of,
+ " (after %"PRId64"ms in state %s)",
+ (rd_clock() - rkb->rkb_ts_state)/1000,
+ rd_kafka_broker_state_names[rkb->
+ rkb_state]);
+
if (level >= LOG_DEBUG)
rd_kafka_dbg(rkb->rkb_rk, BROKER, "FAIL",
"%s", rkb->rkb_err.msg);
else {
- /* Don't log if an error callback is registered */
- if (!rkb->rkb_rk->rk_conf.error_cb)
+ /* Don't log if an error callback is registered,
+ * or the error event is enabled. */
+ if (!(rkb->rkb_rk->rk_conf.enabled_events &
+ RD_KAFKA_EVENT_ERROR))
rd_kafka_log(rkb->rkb_rk, level, "FAIL",
"%s", rkb->rkb_err.msg);
/* Send ERR op back to application for processing. */
@@ -419,11 +432,13 @@
rd_kafka_bufq_concat(&tmpq, &rkb->rkb_outbufs);
rd_atomic32_init(&rkb->rkb_blocking_request_cnt, 0);
- /* Purge the buffers (might get re-enqueued in case of retries) */
+ /* Purge the in-flight buffers (might get re-enqueued in case
+ * of retries). */
rd_kafka_bufq_purge(rkb, &tmpq_waitresp, err);
- /* Put the outbufs back on queue */
- rd_kafka_bufq_concat(&rkb->rkb_outbufs, &tmpq);
+ /* Purge the waiting-in-output-queue buffers,
+ * might also get re-enqueued. */
+ rd_kafka_bufq_purge(rkb, &tmpq, err);
/* Update bufq for connection reset:
* - Purge connection-setup requests from outbufs since they will be
@@ -464,6 +479,47 @@
+/**
+ * @brief Handle broker connection close.
+ *
+ * @locality broker thread
+ */
+void rd_kafka_broker_conn_closed (rd_kafka_broker_t *rkb,
+ rd_kafka_resp_err_t err,
+ const char *errstr) {
+ int log_level = LOG_ERR;
+
+ if (!rkb->rkb_rk->rk_conf.log_connection_close) {
+ /* Silence all connection closes */
+ log_level = LOG_DEBUG;
+
+ } else {
+ /* Silence close logs for connections that are idle,
+ * it is most likely the broker's idle connection
+ * reaper kicking in.
+ *
+ * Indications there might be an error and not an
+ * idle disconnect:
+ * - If the connection age is low a disconnect
+ * typically indicates a failure, such as protocol mismatch.
+ * - If the connection hasn't been idle long enough.
+ * - There are outstanding requests, or requests enqueued.
+ */
+ rd_ts_t now = rd_clock();
+ rd_ts_t minidle =
+ RD_MAX(60*1000/*60s*/,
+ rkb->rkb_rk->rk_conf.socket_timeout_ms) * 1000;
+
+ if (rkb->rkb_ts_state + minidle < now &&
+ rkb->rkb_ts_tx_last + minidle < now &&
+ rd_kafka_bufq_cnt(&rkb->rkb_waitresps) == 0 &&
+ rd_kafka_bufq_cnt(&rkb->rkb_outbufs) == 0)
+ log_level = LOG_DEBUG;
+ }
+
+ rd_kafka_broker_fail(rkb, log_level, err, "%s", errstr);
+}
+
/**
@@ -519,41 +575,45 @@
* Locality: Broker thread
*/
static void rd_kafka_broker_timeout_scan (rd_kafka_broker_t *rkb, rd_ts_t now) {
- int req_cnt, retry_cnt, q_cnt;
+ int inflight_cnt, retry_cnt, outq_cnt;
+ int partial_cnt = 0;
rd_kafka_assert(rkb->rkb_rk, thrd_is_current(rkb->rkb_thread));
- /* Outstanding requests waiting for response */
- req_cnt = rd_kafka_broker_bufq_timeout_scan(
- rkb, 1, &rkb->rkb_waitresps, NULL,
- RD_KAFKA_RESP_ERR__TIMED_OUT, now);
+ /* In-flight requests waiting for response */
+ inflight_cnt = rd_kafka_broker_bufq_timeout_scan(
+ rkb, 1, &rkb->rkb_waitresps, NULL,
+ RD_KAFKA_RESP_ERR__TIMED_OUT, now);
/* Requests in retry queue */
retry_cnt = rd_kafka_broker_bufq_timeout_scan(
rkb, 0, &rkb->rkb_retrybufs, NULL,
- RD_KAFKA_RESP_ERR__TIMED_OUT, now);
- /* Requests in local queue not sent yet. */
- q_cnt = rd_kafka_broker_bufq_timeout_scan(
- rkb, 0, &rkb->rkb_outbufs, &req_cnt,
- RD_KAFKA_RESP_ERR__TIMED_OUT, now);
-
- if (req_cnt + retry_cnt + q_cnt > 0) {
- rd_rkb_dbg(rkb, MSG|RD_KAFKA_DBG_BROKER,
- "REQTMOUT", "Timed out %i+%i+%i requests",
- req_cnt, retry_cnt, q_cnt);
-
- /* Fail the broker if socket.max.fails is configured and
- * now exceeded. */
- rkb->rkb_req_timeouts += req_cnt + q_cnt;
- rd_atomic64_add(&rkb->rkb_c.req_timeouts, req_cnt + q_cnt);
-
- /* If this was an in-flight request that timed out, or
- * the other queues has reached the socket.max.fails threshold,
- * we need to take down the connection. */
- if ((req_cnt > 0 ||
- (rkb->rkb_rk->rk_conf.socket_max_fails &&
- rkb->rkb_req_timeouts >=
- rkb->rkb_rk->rk_conf.socket_max_fails)) &&
- rkb->rkb_state >= RD_KAFKA_BROKER_STATE_UP) {
+ RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE, now);
+ /* Requests in local queue not sent yet.
+ * partial_cnt is included in outq_cnt and denotes a request
+ * that has been partially transmitted. */
+ outq_cnt = rd_kafka_broker_bufq_timeout_scan(
+ rkb, 0, &rkb->rkb_outbufs, &partial_cnt,
+ RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE, now);
+
+ if (inflight_cnt + retry_cnt + outq_cnt + partial_cnt > 0) {
+ rd_rkb_log(rkb, LOG_WARNING, "REQTMOUT",
+ "Timed out %i in-flight, %i retry-queued, "
+ "%i out-queue, %i partially-sent requests",
+ inflight_cnt, retry_cnt, outq_cnt, partial_cnt);
+
+ rkb->rkb_req_timeouts += inflight_cnt + outq_cnt;
+ rd_atomic64_add(&rkb->rkb_c.req_timeouts,
+ inflight_cnt + outq_cnt);
+
+ /* If this was a partially sent request that timed out, or the
+ * number of timed out requests have reached the
+ * socket.max.fails threshold, we need to take down the
+ * connection. */
+ if (partial_cnt > 0 ||
+ (rkb->rkb_rk->rk_conf.socket_max_fails &&
+ rkb->rkb_req_timeouts >=
+ rkb->rkb_rk->rk_conf.socket_max_fails &&
+ rkb->rkb_state >= RD_KAFKA_BROKER_STATE_UP)) {
char rttinfo[32];
/* Print average RTT (if avail) to help diagnose. */
rd_avg_calc(&rkb->rkb_avg_rtt, now);
@@ -566,7 +626,7 @@
rttinfo[0] = 0;
errno = ETIMEDOUT;
rd_kafka_broker_fail(rkb, LOG_ERR,
- RD_KAFKA_RESP_ERR__MSG_TIMED_OUT,
+ RD_KAFKA_RESP_ERR__TIMED_OUT,
"%i request(s) timed out: "
"disconnect%s",
rkb->rkb_req_timeouts, rttinfo);
@@ -604,11 +664,17 @@
static int rd_kafka_broker_resolve (rd_kafka_broker_t *rkb) {
const char *errstr;
+ int save_idx = 0;
if (rkb->rkb_rsal &&
rkb->rkb_ts_rsal_last + (rkb->rkb_rk->rk_conf.broker_addr_ttl*1000)
< rd_clock()) {
/* Address list has expired. */
+
+ /* Save the address index to make sure we still round-robin
+ * if we get the same address list back */
+ save_idx = rkb->rkb_rsal->rsal_curr;
+
rd_sockaddr_list_destroy(rkb->rkb_rsal);
rkb->rkb_rsal = NULL;
}
@@ -634,6 +700,9 @@
return -1;
} else {
rkb->rkb_ts_rsal_last = rd_clock();
+ /* Continue at previous round-robin position */
+ if (rkb->rkb_rsal->rsal_cnt > save_idx)
+ rkb->rkb_rsal->rsal_curr = save_idx;
}
}
@@ -643,14 +712,16 @@
static void rd_kafka_broker_buf_enq0 (rd_kafka_broker_t *rkb,
rd_kafka_buf_t *rkbuf, int at_head) {
+ rd_ts_t now;
+
rd_kafka_assert(rkb->rkb_rk, thrd_is_current(rkb->rkb_thread));
- rkbuf->rkbuf_ts_enq = rd_clock();
+ now = rd_clock();
+ rkbuf->rkbuf_ts_enq = now;
+ rkbuf->rkbuf_flags &= ~RD_KAFKA_OP_F_SENT;
- /* Set timeout if not already set */
- if (!rkbuf->rkbuf_ts_timeout)
- rkbuf->rkbuf_ts_timeout = rkbuf->rkbuf_ts_enq +
- rkb->rkb_rk->rk_conf.socket_timeout_ms * 1000;
+ /* Calculate request attempt timeout */
+ rd_kafka_buf_calc_timeout(rkb->rkb_rk, rkbuf, now);
if (unlikely(at_head)) {
/* Insert message at head of queue */
@@ -680,8 +751,8 @@
}
(void)rd_atomic32_add(&rkb->rkb_outbufs.rkbq_cnt, 1);
- (void)rd_atomic32_add(&rkb->rkb_outbufs.rkbq_msg_cnt,
- rd_atomic32_get(&rkbuf->rkbuf_msgq.rkmq_msg_cnt));
+ (void)rd_atomic32_add(&rkb->rkb_outbufs.rkbq_msg_cnt,
+ rkbuf->rkbuf_msgq.rkmq_msg_cnt);
}
@@ -790,7 +861,7 @@
/**
* @returns the current broker state change version.
- * Pass this value to fugure rd_kafka_brokers_wait_state_change() calls
+ * Pass this value to future rd_kafka_brokers_wait_state_change() calls
* to avoid the race condition where a state-change happens between
* an initial call to some API that fails and the sub-sequent
* .._wait_state_change() call.
@@ -837,17 +908,71 @@
/**
+ * @brief Same as rd_kafka_brokers_wait_state_change() but will trigger
+ * the wakeup asynchronously through the provided \p eonce.
+ *
+ * If the eonce was added to the wait list its reference count
+ * will have been updated, this reference is later removed by
+ * rd_kafka_broker_state_change_trigger_eonce() by calling trigger().
+ *
+ * @returns 1 if the \p eonce was added to the wait-broker-state-changes list,
+ * or 0 if the \p stored_version is outdated in which case the
+ * caller should redo the broker lookup.
+ */
+int rd_kafka_brokers_wait_state_change_async (rd_kafka_t *rk,
+ int stored_version,
+ rd_kafka_enq_once_t *eonce) {
+ int r = 1;
+ mtx_lock(&rk->rk_broker_state_change_lock);
+
+ if (stored_version != rk->rk_broker_state_change_version)
+ r = 0;
+ else {
+ rd_kafka_enq_once_add_source(eonce, "wait broker state change");
+ rd_list_add(&rk->rk_broker_state_change_waiters, eonce);
+ }
+
+ mtx_unlock(&rk->rk_broker_state_change_lock);
+ return r;
+}
+
+
+/**
+ * @brief eonce trigger callback for rd_list_apply() call in
+ * rd_kafka_brokers_broadcast_state_change()
+ */
+static int
+rd_kafka_broker_state_change_trigger_eonce (void *elem, void *opaque) {
+ rd_kafka_enq_once_t *eonce = elem;
+ rd_kafka_enq_once_trigger(eonce, RD_KAFKA_RESP_ERR_NO_ERROR,
+ "broker state change");
+ return 0; /* remove eonce from list */
+}
+
+
+/**
* @brief Broadcast broker state change to listeners, if any.
*
* @locality any thread
*/
void rd_kafka_brokers_broadcast_state_change (rd_kafka_t *rk) {
- rd_kafka_dbg(rk, GENERIC, "BROADCAST",
- "Broadcasting state change");
- mtx_lock(&rk->rk_broker_state_change_lock);
- rk->rk_broker_state_change_version++;
- cnd_broadcast(&rk->rk_broker_state_change_cnd);
- mtx_unlock(&rk->rk_broker_state_change_lock);
+
+ rd_kafka_dbg(rk, GENERIC, "BROADCAST",
+ "Broadcasting state change");
+
+ mtx_lock(&rk->rk_broker_state_change_lock);
+
+ /* Bump version */
+ rk->rk_broker_state_change_version++;
+
+ /* Trigger waiters */
+ rd_list_apply(&rk->rk_broker_state_change_waiters,
+ rd_kafka_broker_state_change_trigger_eonce, NULL);
+
+ /* Broadcast to listeners */
+ cnd_broadcast(&rk->rk_broker_state_change_cnd);
+
+ mtx_unlock(&rk->rk_broker_state_change_lock);
}
@@ -974,6 +1099,133 @@
+/**
+ * @returns the broker handle fork \p broker_id using cached metadata
+ * information (if available) in state == \p state,
+ * with refcount increaesd.
+ *
+ * Otherwise enqueues the \p eonce on the wait-state-change queue
+ * which will be triggered on broker state changes.
+ * It may also be triggered erroneously, so the caller
+ * should call rd_kafka_broker_get_async() again when
+ * the eonce is triggered.
+ *
+ * @locks none
+ * @locality any thread
+ */
+rd_kafka_broker_t *
+rd_kafka_broker_get_async (rd_kafka_t *rk, int32_t broker_id, int state,
+ rd_kafka_enq_once_t *eonce) {
+ int version;
+ do {
+ rd_kafka_broker_t *rkb;
+
+ version = rd_kafka_brokers_get_state_version(rk);
+
+ rd_kafka_rdlock(rk);
+ rkb = rd_kafka_broker_find_by_nodeid0(rk, broker_id, state);
+ rd_kafka_rdunlock(rk);
+
+ if (rkb)
+ return rkb;
+
+ } while (!rd_kafka_brokers_wait_state_change_async(rk, version, eonce));
+
+ return NULL; /* eonce added to wait list */
+}
+
+
+/**
+ * @returns the current controller using cached metadata information,
+ * and only if the broker's state == \p state.
+ * The reference count is increased for the returned broker.
+ *
+ * @locks none
+ * @locality any thread
+ */
+
+static rd_kafka_broker_t *rd_kafka_broker_controller_nowait (rd_kafka_t *rk,
+ int state) {
+ rd_kafka_broker_t *rkb;
+
+ rd_kafka_rdlock(rk);
+
+ if (rk->rk_controllerid == -1) {
+ rd_kafka_rdunlock(rk);
+ rd_kafka_metadata_refresh_brokers(rk, NULL,
+ "lookup controller");
+ return NULL;
+ }
+
+ rkb = rd_kafka_broker_find_by_nodeid0(rk, rk->rk_controllerid, state);
+
+ rd_kafka_rdunlock(rk);
+
+ return rkb;
+}
+
+
+/**
+ * @returns the current controller using cached metadata information if
+ * available in state == \p state, with refcount increaesd.
+ *
+ * Otherwise enqueues the \p eonce on the wait-controller queue
+ * which will be triggered on controller updates or broker state
+ * changes. It may also be triggered erroneously, so the caller
+ * should call rd_kafka_broker_controller_async() again when
+ * the eonce is triggered.
+ *
+ * @locks none
+ * @locality any thread
+ */
+rd_kafka_broker_t *
+rd_kafka_broker_controller_async (rd_kafka_t *rk, int state,
+ rd_kafka_enq_once_t *eonce) {
+ int version;
+ do {
+ rd_kafka_broker_t *rkb;
+
+ version = rd_kafka_brokers_get_state_version(rk);
+
+ rkb = rd_kafka_broker_controller_nowait(rk, state);
+ if (rkb)
+ return rkb;
+
+ } while (!rd_kafka_brokers_wait_state_change_async(rk, version, eonce));
+
+ return NULL; /* eonce added to wait list */
+}
+
+
+/**
+ * @returns the current controller using cached metadata information,
+ * blocking up to \p abs_timeout for the controller to be known
+ * and to reach state == \p state. The reference count is increased
+ * for the returned broker.
+ *
+ * @locks none
+ * @locality any thread
+ */
+rd_kafka_broker_t *rd_kafka_broker_controller (rd_kafka_t *rk, int state,
+ rd_ts_t abs_timeout) {
+
+ while (1) {
+ int version = rd_kafka_brokers_get_state_version(rk);
+ rd_kafka_broker_t *rkb;
+ int remains_ms;
+
+ rkb = rd_kafka_broker_controller_nowait(rk, state);
+ if (rkb)
+ return rkb;
+
+ remains_ms = rd_timeout_remains(abs_timeout);
+ if (rd_timeout_expired(remains_ms))
+ return NULL;
+
+ rd_kafka_brokers_wait_state_change(rk, version, remains_ms);
+ }
+}
+
@@ -1039,6 +1291,9 @@
rkbuf->rkbuf_totlen, rkbuf->rkbuf_reshdr.CorrId,
(float)req->rkbuf_ts_sent / 1000.0f);
+ /* Copy request's header to response object's reqhdr for convenience. */
+ rkbuf->rkbuf_reqhdr = req->rkbuf_reqhdr;
+
/* Set up response reader slice starting past the response header */
rd_slice_init(&rkbuf->rkbuf_reader, &rkbuf->rkbuf_buf,
RD_KAFKAP_RESHDR_SIZE,
@@ -1171,11 +1426,11 @@
err_parse:
err = rkbuf->rkbuf_err;
err:
- rd_kafka_broker_fail(rkb,
- !rkb->rkb_rk->rk_conf.log_connection_close &&
- !strcmp(errstr, "Disconnected") ?
- LOG_DEBUG : LOG_ERR, err,
- "Receive failed: %s", errstr);
+ if (!strcmp(errstr, "Disconnected"))
+ rd_kafka_broker_conn_closed(rkb, err, errstr);
+ else
+ rd_kafka_broker_fail(rkb, LOG_ERR, err,
+ "Receive failed: %s", errstr);
return -1;
}
@@ -1544,6 +1799,15 @@
rd_kafka_broker_feature_enable(rkb, RD_KAFKA_FEATURE_APIVERSION);
}
+ if (!(rkb->rkb_features & RD_KAFKA_FEATURE_APIVERSION)) {
+ /* Use configured broker.version.fallback to
+ * figure out API versions.
+ * In case broker.version.fallback indicates a version
+ * that supports ApiVersionRequest it will update
+ * rkb_features to have FEATURE_APIVERSION set which will
+ * trigger an ApiVersionRequest below. */
+ rd_kafka_broker_set_api_versions(rkb, NULL, 0);
+ }
if (rkb->rkb_features & RD_KAFKA_FEATURE_APIVERSION) {
/* Query broker for supported API versions.
@@ -1559,11 +1823,6 @@
rd_kafka_broker_handle_ApiVersion, NULL,
1 /*Flash message: prepend to transmit queue*/);
} else {
-
- /* Use configured broker.version.fallback to
- * figure out API versions */
- rd_kafka_broker_set_api_versions(rkb, NULL, 0);
-
/* Authenticate if necessary */
rd_kafka_broker_connect_auth(rkb);
}
@@ -1625,6 +1884,7 @@
(rkbuf = TAILQ_FIRST(&rkb->rkb_outbufs.rkbq_bufs))) {
ssize_t r;
size_t pre_of = rd_slice_offset(&rkbuf->rkbuf_reader);
+ rd_ts_t now;
/* Check for broker support */
if (unlikely(!rd_kafka_broker_request_supported(rkb, rkbuf))) {
@@ -1681,6 +1941,9 @@
if ((r = rd_kafka_broker_send(rkb, &rkbuf->rkbuf_reader)) == -1)
return -1;
+ now = rd_clock();
+ rkb->rkb_ts_tx_last = now;
+
/* Partial send? Continue next time. */
if (rd_slice_remains(&rkbuf->rkbuf_reader) > 0) {
rd_rkb_dbg(rkb, PROTOCOL, "SEND",
@@ -1705,11 +1968,20 @@
rd_slice_size(&rkbuf->rkbuf_reader),
pre_of, rkbuf->rkbuf_corrid);
+ /* Notify transport layer of full request sent */
+ if (likely(rkb->rkb_transport != NULL))
+ rd_kafka_transport_request_sent(rkb, rkbuf);
+
/* Entire buffer sent, unlink from outbuf */
rd_kafka_bufq_deq(&rkb->rkb_outbufs, rkbuf);
+ rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_SENT;
/* Store time for RTT calculation */
- rkbuf->rkbuf_ts_sent = rd_clock();
+ rkbuf->rkbuf_ts_sent = now;
+
+ /* Add to outbuf_latency averager */
+ rd_avg_add(&rkb->rkb_avg_outbuf_latency,
+ rkbuf->rkbuf_ts_sent - rkbuf->rkbuf_ts_enq);
if (rkbuf->rkbuf_flags & RD_KAFKA_OP_F_BLOCKING &&
rd_atomic32_add(&rkb->rkb_blocking_request_cnt, 1) == 1)
@@ -1752,16 +2024,25 @@
}
rd_rkb_dbg(rkb, PROTOCOL, "RETRY",
- "Retrying %sRequest (v%hd, %"PRIusz" bytes, retry %d/%d)",
+ "Retrying %sRequest (v%hd, %"PRIusz" bytes, retry %d/%d, "
+ "prev CorrId %"PRId32") in %dms",
rd_kafka_ApiKey2str(rkbuf->rkbuf_reqhdr.ApiKey),
rkbuf->rkbuf_reqhdr.ApiVersion,
rd_slice_size(&rkbuf->rkbuf_reader),
- rkbuf->rkbuf_retries, rkb->rkb_rk->rk_conf.max_retries);
+ rkbuf->rkbuf_retries, rkb->rkb_rk->rk_conf.max_retries,
+ rkbuf->rkbuf_corrid,
+ rkb->rkb_rk->rk_conf.retry_backoff_ms);
rd_atomic64_add(&rkb->rkb_c.tx_retries, 1);
rkbuf->rkbuf_ts_retry = rd_clock() +
(rkb->rkb_rk->rk_conf.retry_backoff_ms * 1000);
+ /* Precaution: time out the request if it hasn't moved from the
+ * retry queue within the retry interval (such as when the broker is
+ * down). */
+ // FIXME: implememt this properly.
+ rkbuf->rkbuf_ts_timeout = rkbuf->rkbuf_ts_retry + (5*1000*1000);
+
/* Reset send offset */
rd_slice_seek(&rkbuf->rkbuf_reader, 0);
rkbuf->rkbuf_corrid = 0;
@@ -1777,6 +2058,7 @@
static void rd_kafka_broker_retry_bufs_move (rd_kafka_broker_t *rkb) {
rd_ts_t now = rd_clock();
rd_kafka_buf_t *rkbuf;
+ int cnt = 0;
while ((rkbuf = TAILQ_FIRST(&rkb->rkb_retrybufs.rkbq_bufs))) {
if (rkbuf->rkbuf_ts_retry > now)
@@ -1785,7 +2067,12 @@
rd_kafka_bufq_deq(&rkb->rkb_retrybufs, rkbuf);
rd_kafka_broker_buf_enq0(rkb, rkbuf, 0/*tail*/);
+ cnt++;
}
+
+ if (cnt > 0)
+ rd_rkb_dbg(rkb, BROKER, "RETRY",
+ "Moved %d retry buffer(s) to output queue", cnt);
}
@@ -1800,7 +2087,7 @@
return;
/* Call on_acknowledgement() interceptors */
- rd_kafka_interceptors_on_acknowledgement_queue(rk, rkmq);
+ rd_kafka_interceptors_on_acknowledgement_queue(rk, rkmq, err);
if ((rk->rk_conf.enabled_events & RD_KAFKA_EVENT_DR) &&
(!rk->rk_conf.dr_err_only || err)) {
@@ -1940,6 +2227,10 @@
rkb->rkb_nodeid = rko->rko_u.node.nodeid;
+ /* Update system thread name */
+ rd_kafka_set_thread_sysname("rdk:broker%"PRId32,
+ rkb->rkb_nodeid);
+
/* Update broker_by_id sorted list */
if (old_nodeid == -1)
rd_list_add(&rkb->rkb_rk->rk_broker_by_id, rkb);
@@ -2068,6 +2359,9 @@
rktp->rktp_msgq_wakeup_fd = rkb->rkb_toppar_wakeup_fd;
rd_kafka_broker_keep(rkb);
+ if (rkb->rkb_rk->rk_type == RD_KAFKA_PRODUCER)
+ rd_kafka_broker_active_toppar_add(rkb, rktp);
+
rd_kafka_broker_destroy(rktp->rktp_next_leader);
rktp->rktp_next_leader = NULL;
@@ -2115,12 +2409,15 @@
rd_kafka_broker_name(rktp->rktp_next_leader) :
"(none)", rktp);
- /* Prepend xmitq(broker-local) messages to the msgq(global).
- * There is no msgq_prepend() so we append msgq to xmitq
- * and then move the queue altogether back over to msgq. */
- rd_kafka_msgq_concat(&rktp->rktp_xmit_msgq,
- &rktp->rktp_msgq);
- rd_kafka_msgq_move(&rktp->rktp_msgq, &rktp->rktp_xmit_msgq);
+ /* Insert xmitq(broker-local) messages to the msgq(global)
+ * at their sorted position to maintain ordering. */
+ rd_kafka_msgq_insert_msgq(&rktp->rktp_msgq,
+ &rktp->rktp_xmit_msgq,
+ rktp->rktp_rkt->rkt_conf.
+ msg_order_cmp);
+
+ if (rkb->rkb_rk->rk_type == RD_KAFKA_PRODUCER)
+ rd_kafka_broker_active_toppar_del(rkb, rktp);
rd_kafka_broker_lock(rkb);
TAILQ_REMOVE(&rkb->rkb_toppars, rktp, rktp_rkblink);
@@ -2167,14 +2464,25 @@
rkb->rkb_blocking_max_ms = 1; /* Speed up termination*/
rd_rkb_dbg(rkb, BROKER, "TERM",
"Received TERMINATE op in state %s: "
- "%d refcnts, %d toppar(s), %d fetch toppar(s), "
+ "%d refcnts, %d toppar(s), %d active toppar(s), "
"%d outbufs, %d waitresps, %d retrybufs",
rd_kafka_broker_state_names[rkb->rkb_state],
rd_refcnt_get(&rkb->rkb_refcnt),
- rkb->rkb_toppar_cnt, rkb->rkb_fetch_toppar_cnt,
+ rkb->rkb_toppar_cnt, rkb->rkb_active_toppar_cnt,
(int)rd_kafka_bufq_cnt(&rkb->rkb_outbufs),
(int)rd_kafka_bufq_cnt(&rkb->rkb_waitresps),
(int)rd_kafka_bufq_cnt(&rkb->rkb_retrybufs));
+ /* Expedite termination by bringing down the broker
+ * and trigger a state change.
+ * This makes sure any eonce dependent on state changes
+ * are triggered. */
+ rd_kafka_broker_fail(rkb, LOG_DEBUG,
+ RD_KAFKA_RESP_ERR__DESTROY,
+ "Client is terminating");
+ ret = 0;
+ break;
+
+ case RD_KAFKA_OP_WAKEUP:
ret = 0;
break;
@@ -2190,6 +2498,22 @@
}
+
+/**
+ * @brief Serve broker ops.
+ * @returns the number of ops served
+ */
+static int rd_kafka_broker_ops_serve (rd_kafka_broker_t *rkb, int timeout_ms) {
+ rd_kafka_op_t *rko;
+ int cnt = 0;
+
+ while ((rko = rd_kafka_q_pop(rkb->rkb_ops, timeout_ms, 0)) &&
+ (cnt++, rd_kafka_broker_op_serve(rkb, rko)))
+ timeout_ms = RD_POLL_NOWAIT;
+
+ return cnt;
+}
+
/**
* @brief Serve broker ops and IOs.
*
@@ -2200,29 +2524,27 @@
*/
static void rd_kafka_broker_serve (rd_kafka_broker_t *rkb,
rd_ts_t abs_timeout) {
- rd_kafka_op_t *rko;
rd_ts_t now;
int initial_state = rkb->rkb_state;
int remains_ms = rd_timeout_remains(abs_timeout);
/* Serve broker ops */
- while ((rko = rd_kafka_q_pop(rkb->rkb_ops,
- !rkb->rkb_transport ?
- remains_ms : RD_POLL_NOWAIT,
- 0))
- && rd_kafka_broker_op_serve(rkb, rko))
+ if (rd_kafka_broker_ops_serve(rkb,
+ !rkb->rkb_transport ?
+ remains_ms : RD_POLL_NOWAIT))
remains_ms = RD_POLL_NOWAIT;
- /* If the broker state changed in op_serve() we minimize
- * the IO timeout since our caller might want to exit out of
- * its loop on state change. */
if (likely(rkb->rkb_transport != NULL)) {
int blocking_max_ms;
- if ((int)rkb->rkb_state != initial_state)
+ /* If the broker state changed in op_serve() we minimize
+ * the IO timeout since our caller might want to exit out of
+ * its loop on state change. */
+ if (unlikely((int)rkb->rkb_state != initial_state))
blocking_max_ms = 0;
else {
- int remains_ms = rd_timeout_remains(abs_timeout);
+ if (remains_ms == RD_POLL_NOWAIT)
+ remains_ms = rd_timeout_remains(abs_timeout);
if (remains_ms == RD_POLL_INFINITE ||
remains_ms > rkb->rkb_blocking_max_ms)
remains_ms = rkb->rkb_blocking_max_ms;
@@ -2300,74 +2622,129 @@
/**
+ * @brief Scan toppar's xmit queue for message timeouts.
+ * @locality broker thread
+ * @locks none
+ */
+static void rd_kafka_broker_toppar_msgq_scan (rd_kafka_broker_t *rkb,
+ rd_kafka_toppar_t *rktp,
+ rd_ts_t now) {
+ rd_kafka_msgq_t timedout = RD_KAFKA_MSGQ_INITIALIZER(timedout);
+
+ if (rd_kafka_msgq_age_scan(&rktp->rktp_xmit_msgq, &timedout, now)) {
+ /* Trigger delivery report for timed out messages */
+ rd_kafka_dr_msgq(rktp->rktp_rkt, &timedout,
+ RD_KAFKA_RESP_ERR__MSG_TIMED_OUT);
+ }
+}
+
+/**
* @brief Serve a toppar for producing.
*
* @param next_wakeup will be updated to when the next wake-up/attempt is
* desired, only lower (sooner) values will be set.
*
- * Locks: toppar_lock(rktp) MUST be held.
- * Returns the number of messages produced.
+ * @returns the number of messages produced.
+ *
+ * @locks toppar_lock(rktp) MUST be held.
+ * @locality broker thread
*/
static int rd_kafka_toppar_producer_serve (rd_kafka_broker_t *rkb,
rd_kafka_toppar_t *rktp,
- int do_timeout_scan,
rd_ts_t now,
- rd_ts_t *next_wakeup) {
+ rd_ts_t *next_wakeup,
+ int do_timeout_scan) {
int cnt = 0;
int r;
+ rd_kafka_msg_t *rkm;
+ int move_cnt = 0;
- rd_rkb_dbg(rkb, QUEUE, "TOPPAR",
- "%.*s [%"PRId32"] %i+%i msgs",
- RD_KAFKAP_STR_PR(rktp->rktp_rkt->
- rkt_topic),
- rktp->rktp_partition,
- rd_atomic32_get(&rktp->rktp_msgq.rkmq_msg_cnt),
- rd_atomic32_get(&rktp->rktp_xmit_msgq.
- rkmq_msg_cnt));
+ /* By limiting the number of not-yet-sent buffers (rkb_outbufs) we
+ * provide a backpressure mechanism to the producer loop
+ * which allows larger message batches to accumulate and thus
+ * increase throughput.
+ * This comes at no latency cost since there are already
+ * buffers enqueued waiting for transmission.
+ *
+ * The !do_timeout_scan condition is an optimization to
+ * avoid having to acquire the lock in the typical case
+ * (do_timeout_scan==0). */
+ if (unlikely(!do_timeout_scan &&
+ rd_atomic32_get(&rkb->rkb_outbufs.rkbq_cnt) >=
+ rkb->rkb_rk->rk_conf.queue_backpressure_thres))
+ return 0;
+
+ rd_kafka_toppar_lock(rktp);
- if (rd_atomic32_get(&rktp->rktp_msgq.rkmq_msg_cnt) > 0)
- rd_kafka_msgq_concat(&rktp->rktp_xmit_msgq, &rktp->rktp_msgq);
+ if (unlikely(rktp->rktp_leader != rkb)) {
+ /* Currently migrating away from this
+ * broker. */
+ rd_kafka_toppar_unlock(rktp);
+ return 0;
+ }
- /* Timeout scan */
if (unlikely(do_timeout_scan)) {
- rd_kafka_msgq_t timedout = RD_KAFKA_MSGQ_INITIALIZER(timedout);
+ /* Scan xmit queue for msg timeouts */
+ rd_kafka_broker_toppar_msgq_scan(rkb, rktp, now);
+ }
- if (rd_kafka_msgq_age_scan(&rktp->rktp_xmit_msgq,
- &timedout, now)) {
- /* Trigger delivery report for timed out messages */
- rd_kafka_dr_msgq(rktp->rktp_rkt, &timedout,
- RD_KAFKA_RESP_ERR__MSG_TIMED_OUT);
- }
+ if (unlikely(RD_KAFKA_TOPPAR_IS_PAUSED(rktp))) {
+ /* Partition is paused */
+ rd_kafka_toppar_unlock(rktp);
+ return 0;
}
- r = rd_atomic32_get(&rktp->rktp_xmit_msgq.rkmq_msg_cnt);
+
+
+ /* Move messages from locked partition produce queue
+ * to broker-local xmit queue. */
+ if ((move_cnt = rktp->rktp_msgq.rkmq_msg_cnt) > 0)
+ rd_kafka_msgq_insert_msgq(&rktp->rktp_xmit_msgq,
+ &rktp->rktp_msgq,
+ rktp->rktp_rkt->rkt_conf.
+ msg_order_cmp);
+ rd_kafka_toppar_unlock(rktp);
+
+ r = rktp->rktp_xmit_msgq.rkmq_msg_cnt;
if (r == 0)
return 0;
+ rd_rkb_dbg(rkb, QUEUE, "TOPPAR",
+ "%.*s [%"PRId32"] %d message(s) in "
+ "xmit queue (%d added from partition queue)",
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
+ rktp->rktp_partition,
+ r, move_cnt);
+
+ rkm = TAILQ_FIRST(&rktp->rktp_xmit_msgq.rkmq_msgs);
+ rd_dassert(rkm != NULL);
+
/* Attempt to fill the batch size, but limit
* our waiting to queue.buffering.max.ms
* and batch.num.messages. */
if (r < rkb->rkb_rk->rk_conf.batch_num_messages) {
- rd_kafka_msg_t *rkm_oldest;
rd_ts_t wait_max;
- rkm_oldest = TAILQ_FIRST(&rktp->rktp_xmit_msgq.rkmq_msgs);
- if (unlikely(!rkm_oldest))
- return 0;
-
- /* Calculate maximum wait-time to
- * honour queue.buffering.max.ms contract. */
- wait_max = rd_kafka_msg_enq_time(rkm_oldest) +
+ /* Calculate maximum wait-time to honour
+ * queue.buffering.max.ms contract. */
+ wait_max = rd_kafka_msg_enq_time(rkm) +
(rkb->rkb_rk->rk_conf.buffering_max_ms * 1000);
+
if (wait_max > now) {
- if (wait_max < *next_wakeup)
- *next_wakeup = wait_max;
/* Wait for more messages or queue.buffering.max.ms
* to expire. */
+ *next_wakeup = wait_max;
return 0;
}
}
+ /* Honour retry.backoff.ms. */
+ if (unlikely(rkm->rkm_u.producer.ts_backoff > now)) {
+ *next_wakeup = rkm->rkm_u.producer.ts_backoff;
+ /* Wait for backoff to expire */
+ return 0;
+ }
+
/* Send Produce requests for this toppar */
while (1) {
r = rd_kafka_ProduceRequest(rkb, rktp);
@@ -2377,10 +2754,55 @@
break;
}
+ /* If there are messages still in the queue, make the next
+ * wakeup immediate. */
+ if (rd_kafka_msgq_len(&rktp->rktp_xmit_msgq) > 0)
+ *next_wakeup = now;
+
return cnt;
}
+
+/**
+ * @brief Produce from all toppars assigned to this broker.
+ * @returns the total number of messages produced.
+ */
+static int rd_kafka_broker_produce_toppars (rd_kafka_broker_t *rkb,
+ rd_ts_t now,
+ rd_ts_t *next_wakeup,
+ int do_timeout_scan) {
+ rd_kafka_toppar_t *rktp;
+ int cnt = 0;
+ rd_ts_t ret_next_wakeup = *next_wakeup;
+
+ /* Round-robin serve each toppar. */
+ rktp = rkb->rkb_active_toppar_next;
+ if (unlikely(!rktp))
+ return 0;
+
+ do {
+ rd_ts_t this_next_wakeup = ret_next_wakeup;
+
+ /* Try producing toppar */
+ cnt += rd_kafka_toppar_producer_serve(
+ rkb, rktp, now, &this_next_wakeup,
+ do_timeout_scan);
+
+ if (this_next_wakeup < ret_next_wakeup)
+ ret_next_wakeup = this_next_wakeup;
+
+ } while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->
+ rkb_active_toppars,
+ rktp, rktp_activelink)) !=
+ rkb->rkb_active_toppar_next);
+
+ *next_wakeup = ret_next_wakeup;
+
+
+ return cnt;
+}
+
/**
* Producer serving
*/
@@ -2395,11 +2817,9 @@
while (!rd_kafka_broker_terminating(rkb) &&
rkb->rkb_state == RD_KAFKA_BROKER_STATE_UP) {
- rd_kafka_toppar_t *rktp;
- int cnt;
rd_ts_t now;
rd_ts_t next_wakeup;
- int do_timeout_scan = 0;
+ int do_timeout_scan;
rd_kafka_broker_unlock(rkb);
@@ -2407,36 +2827,11 @@
next_wakeup = now + (rkb->rkb_rk->rk_conf.
socket_blocking_max_ms * 1000);
- if (rd_interval(&timeout_scan, 1000*1000, now) >= 0)
- do_timeout_scan = 1;
-
- do {
- cnt = 0;
+ do_timeout_scan = rd_interval(&timeout_scan, 1000*1000,
+ now) >= 0;
- /* Serve each toppar */
- TAILQ_FOREACH(rktp, &rkb->rkb_toppars, rktp_rkblink) {
- /* Serve toppar op queue */
- rd_kafka_toppar_lock(rktp);
- if (unlikely(rktp->rktp_leader != rkb)) {
- /* Currently migrating away from this
- * broker. */
- rd_kafka_toppar_unlock(rktp);
- continue;
- }
- if (unlikely(RD_KAFKA_TOPPAR_IS_PAUSED(rktp))) {
- /* Partition is paused */
- rd_kafka_toppar_unlock(rktp);
- continue;
- }
- /* Try producing toppar */
- cnt += rd_kafka_toppar_producer_serve(
- rkb, rktp, do_timeout_scan, now,
- &next_wakeup);
-
- rd_kafka_toppar_unlock(rktp);
- }
-
- } while (cnt);
+ rd_kafka_broker_produce_toppars(rkb, now, &next_wakeup,
+ do_timeout_scan);
/* Check and move retry buffers */
if (unlikely(rd_atomic32_get(&rkb->rkb_retrybufs.rkbq_cnt) > 0))
@@ -2477,6 +2872,11 @@
rd_kafka_toppar_t *rktp,
rd_kafka_resp_err_t err) {
int backoff_ms = rkb->rkb_rk->rk_conf.fetch_error_backoff_ms;
+
+ /* Don't back off on reaching end of partition */
+ if (err == RD_KAFKA_RESP_ERR__PARTITION_EOF)
+ return;
+
rktp->rktp_ts_fetch_backoff = rd_clock() + (backoff_ms * 1000);
rd_rkb_dbg(rkb, FETCH, "BACKOFF",
"%s [%"PRId32"]: Fetch backoff for %dms: %s",
@@ -2607,22 +3007,24 @@
rd_kafka_toppar_unlock(rktp);
/* Check if this Fetch is for an outdated fetch version,
- * if so ignore it. */
+ * or the original rktp was removed and a new one
+ * created (due to partition count decreasing and
+ * then increasing again, which can happen in
+ * desynchronized clusters): if so ignore it. */
tver_skel.s_rktp = s_rktp;
tver = rd_list_find(request->rkbuf_rktp_vers,
&tver_skel,
rd_kafka_toppar_ver_cmp);
- rd_kafka_assert(NULL, tver &&
- rd_kafka_toppar_s2i(tver->s_rktp) ==
- rktp);
- if (tver->version < fetch_version) {
- rd_rkb_dbg(rkb, MSG, "DROP",
- "%s [%"PRId32"]: "
- "dropping outdated fetch response "
- "(v%d < %d)",
- rktp->rktp_rkt->rkt_topic->str,
- rktp->rktp_partition,
- tver->version, fetch_version);
+ rd_kafka_assert(NULL, tver);
+ if (rd_kafka_toppar_s2i(tver->s_rktp) != rktp ||
+ tver->version < fetch_version) {
+ rd_rkb_dbg(rkb, MSG, "DROP",
+ "%s [%"PRId32"]: "
+ "dropping outdated fetch response "
+ "(v%d < %d or old rktp)",
+ rktp->rktp_rkt->rkt_topic->str,
+ rktp->rktp_partition,
+ tver->version, fetch_version);
rd_atomic64_add(&rktp->rktp_c. rx_ver_drops, 1);
rd_kafka_toppar_destroy(s_rktp); /* from get */
rd_kafka_buf_skip(rkbuf, hdr.MessageSetSize);
@@ -2868,7 +3270,7 @@
* when allocating and assume each partition is on its own topic
*/
- if (unlikely(rkb->rkb_fetch_toppar_cnt == 0))
+ if (unlikely(rkb->rkb_active_toppar_cnt == 0))
return 0;
rkbuf = rd_kafka_buf_new_request(
@@ -2876,7 +3278,7 @@
/* ReplicaId+MaxWaitTime+MinBytes+TopicCnt */
4+4+4+4+
/* N x PartCnt+Partition+FetchOffset+MaxBytes+?TopicNameLen?*/
- (rkb->rkb_fetch_toppar_cnt * (4+4+8+4+40)));
+ (rkb->rkb_active_toppar_cnt * (4+4+8+4+40)));
if (rkb->rkb_features & RD_KAFKA_FEATURE_MSGVER2)
rd_kafka_buf_ApiVersion_set(rkbuf, 4,
@@ -2900,7 +3302,7 @@
if (rd_kafka_buf_ApiVersion(rkbuf) == 4) {
/* MaxBytes */
rd_kafka_buf_write_i32(rkbuf,
- rkb->rkb_rk->rk_conf.recv_max_msg_size);
+ rkb->rkb_rk->rk_conf.fetch_max_bytes);
/* IsolationLevel */
rd_kafka_buf_write_i8(rkbuf, RD_KAFKAP_READ_UNCOMMITTED);
}
@@ -2915,10 +3317,10 @@
0, (void *)rd_kafka_toppar_ver_destroy);
rd_list_prealloc_elems(rkbuf->rkbuf_rktp_vers,
sizeof(struct rd_kafka_toppar_ver),
- rkb->rkb_fetch_toppar_cnt);
+ rkb->rkb_active_toppar_cnt, 0);
/* Round-robin start of the list. */
- rktp = rkb->rkb_fetch_toppar_next;
+ rktp = rkb->rkb_active_toppar_next;
do {
struct rd_kafka_toppar_ver *tver;
@@ -2962,20 +3364,19 @@
tver->version = rktp->rktp_fetch_version;
cnt++;
- } while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_fetch_toppars,
- rktp, rktp_fetchlink)) !=
- rkb->rkb_fetch_toppar_next);
+ } while ((rktp = CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars,
+ rktp, rktp_activelink)) !=
+ rkb->rkb_active_toppar_next);
/* Update next toppar to fetch in round-robin list. */
- rd_kafka_broker_fetch_toppar_next(rkb,
- rktp ?
- CIRCLEQ_LOOP_NEXT(&rkb->
- rkb_fetch_toppars,
- rktp, rktp_fetchlink):
- NULL);
+ rd_kafka_broker_active_toppar_next(
+ rkb,
+ rktp ?
+ CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars,
+ rktp, rktp_activelink) : NULL);
rd_rkb_dbg(rkb, FETCH, "FETCH", "Fetch %i/%i/%i toppar(s)",
- cnt, rkb->rkb_fetch_toppar_cnt, rkb->rkb_toppar_cnt);
+ cnt, rkb->rkb_active_toppar_cnt, rkb->rkb_toppar_cnt);
if (!cnt) {
rd_kafka_buf_destroy(rkbuf);
return cnt;
@@ -2991,10 +3392,11 @@
/* Update TopicArrayCnt */
rd_kafka_buf_update_i32(rkbuf, of_TopicArrayCnt, TopicArrayCnt);
- /* Use configured timeout */
- rkbuf->rkbuf_ts_timeout = now +
- ((rkb->rkb_rk->rk_conf.socket_timeout_ms +
- rkb->rkb_rk->rk_conf.fetch_wait_max_ms) * 1000);
+ /* Use configured timeout */
+ rd_kafka_buf_set_timeout(rkbuf,
+ rkb->rkb_rk->rk_conf.socket_timeout_ms +
+ rkb->rkb_rk->rk_conf.fetch_wait_max_ms,
+ now);
/* Sort toppar versions for quicker lookups in Fetch response. */
rd_list_sort(rkbuf->rkbuf_rktp_vers, rd_kafka_toppar_ver_cmp);
@@ -3072,8 +3474,8 @@
rd_kafka_broker_t *rkb = arg;
rd_kafka_t *rk = rkb->rkb_rk;
- rd_snprintf(rd_kafka_thread_name, sizeof(rd_kafka_thread_name),
- "%s", rkb->rkb_name);
+ rd_kafka_set_thread_name("%s", rkb->rkb_name);
+ rd_kafka_set_thread_sysname("rdk:broker%"PRId32, rkb->rkb_nodeid);
(void)rd_atomic32_add(&rd_kafka_thread_cnt_curr, 1);
@@ -3176,10 +3578,6 @@
rd_kafka_broker_lock(rkb);
rd_kafka_broker_set_state(rkb, RD_KAFKA_BROKER_STATE_UP);
rd_kafka_broker_unlock(rkb);
- } else {
- /* Connection torn down, sleep a short while to
- * avoid busy-looping on protocol errors */
- rd_usleep(100*1000/*100ms*/, &rk->rk_terminate);
}
break;
}
@@ -3197,9 +3595,20 @@
rkb, 0, &rkb->rkb_retrybufs, NULL,
RD_KAFKA_RESP_ERR__DESTROY, 0);
rd_rkb_dbg(rkb, BROKER, "TERMINATE",
- "Handle is terminating: "
- "failed %d request(s) in "
- "retry+outbuf", r);
+ "Handle is terminating in state %s: "
+ "%d refcnts (%p), %d toppar(s), "
+ "%d active toppar(s), "
+ "%d outbufs, %d waitresps, %d retrybufs: "
+ "failed %d request(s) in retry+outbuf",
+ rd_kafka_broker_state_names[rkb->rkb_state],
+ rd_refcnt_get(&rkb->rkb_refcnt),
+ &rkb->rkb_refcnt,
+ rkb->rkb_toppar_cnt,
+ rkb->rkb_active_toppar_cnt,
+ (int)rd_kafka_bufq_cnt(&rkb->rkb_outbufs),
+ (int)rd_kafka_bufq_cnt(&rkb->rkb_waitresps),
+ (int)rd_kafka_bufq_cnt(&rkb->rkb_retrybufs),
+ r);
}
}
@@ -3213,6 +3622,16 @@
}
rd_kafka_broker_fail(rkb, LOG_DEBUG, RD_KAFKA_RESP_ERR__DESTROY, NULL);
+
+ /* Disable and drain ops queue.
+ * Simply purging the ops queue risks leaving dangling references
+ * for ops such as PARTITION_JOIN/PARTITION_LEAVE where the broker
+ * reference is not maintained in the rko (but in rktp_next_leader).
+ * #1596 */
+ rd_kafka_q_disable(rkb->rkb_ops);
+ while (rd_kafka_broker_ops_serve(rkb, RD_POLL_NOWAIT))
+ ;
+
rd_kafka_broker_destroy(rkb);
#if WITH_SSL
@@ -3268,6 +3687,7 @@
rd_kafka_q_destroy_owner(rkb->rkb_ops);
rd_avg_destroy(&rkb->rkb_avg_int_latency);
+ rd_avg_destroy(&rkb->rkb_avg_outbuf_latency);
rd_avg_destroy(&rkb->rkb_avg_rtt);
rd_avg_destroy(&rkb->rkb_avg_throttle);
@@ -3314,8 +3734,8 @@
const char *name, uint16_t port,
int32_t nodeid) {
rd_kafka_broker_t *rkb;
-#ifndef _MSC_VER
int r;
+#ifndef _MSC_VER
sigset_t newset, oldset;
#endif
@@ -3337,15 +3757,20 @@
mtx_init(&rkb->rkb_logname_lock, mtx_plain);
rkb->rkb_logname = rd_strdup(rkb->rkb_name);
TAILQ_INIT(&rkb->rkb_toppars);
- CIRCLEQ_INIT(&rkb->rkb_fetch_toppars);
+ CIRCLEQ_INIT(&rkb->rkb_active_toppars);
rd_kafka_bufq_init(&rkb->rkb_outbufs);
rd_kafka_bufq_init(&rkb->rkb_waitresps);
rd_kafka_bufq_init(&rkb->rkb_retrybufs);
rkb->rkb_ops = rd_kafka_q_new(rk);
rd_interval_init(&rkb->rkb_connect_intvl);
- rd_avg_init(&rkb->rkb_avg_int_latency, RD_AVG_GAUGE);
- rd_avg_init(&rkb->rkb_avg_rtt, RD_AVG_GAUGE);
- rd_avg_init(&rkb->rkb_avg_throttle, RD_AVG_GAUGE);
+ rd_avg_init(&rkb->rkb_avg_int_latency, RD_AVG_GAUGE, 0, 100*1000, 2,
+ rk->rk_conf.stats_interval_ms ? 1 : 0);
+ rd_avg_init(&rkb->rkb_avg_outbuf_latency, RD_AVG_GAUGE, 0, 100*1000, 2,
+ rk->rk_conf.stats_interval_ms ? 1 : 0);
+ rd_avg_init(&rkb->rkb_avg_rtt, RD_AVG_GAUGE, 0, 500*1000, 2,
+ rk->rk_conf.stats_interval_ms ? 1 : 0);
+ rd_avg_init(&rkb->rkb_avg_throttle, RD_AVG_GAUGE, 0, 5000*1000, 2,
+ rk->rk_conf.stats_interval_ms ? 1 : 0);
rd_refcnt_init(&rkb->rkb_refcnt, 0);
rd_kafka_broker_keep(rkb); /* rk_broker's refcount */
@@ -3393,7 +3818,6 @@
rkb->rkb_wakeup_fd[1] = -1;
rkb->rkb_toppar_wakeup_fd = -1;
-#ifndef _MSC_VER /* pipes cant be mixed with WSAPoll on Win32 */
if ((r = rd_pipe_nonblocking(rkb->rkb_wakeup_fd)) == -1) {
rd_rkb_log(rkb, LOG_ERR, "WAKEUPFD",
"Failed to setup broker queue wake-up fds: "
@@ -3425,7 +3849,6 @@
rd_kafka_q_io_event_enable(rkb->rkb_ops, rkb->rkb_wakeup_fd[1],
&onebyte, sizeof(onebyte));
}
-#endif
/* Lock broker's lock here to synchronise state, i.e., hold off
* the broker thread until we've finalized the rkb. */
@@ -3536,7 +3959,7 @@
TAILQ_FOREACH(rkb, &rk->rk_brokers, rkb_link) {
rd_kafka_broker_lock(rkb);
- if (!rd_atomic32_get(&rk->rk_terminate) &&
+ if (!rd_kafka_terminating(rk) &&
rkb->rkb_proto == proto &&
!strcmp(rkb->rkb_nodename, nodename)) {
rd_kafka_broker_keep(rkb);
@@ -3731,7 +4154,7 @@
rd_kafka_mk_nodename(nodename, sizeof(nodename), mdb->host, mdb->port);
rd_kafka_wrlock(rk);
- if (unlikely(rd_atomic32_get(&rk->rk_terminate))) {
+ if (unlikely(rd_kafka_terminating(rk))) {
/* Dont update metadata while terminating, do this
* after acquiring lock for proper synchronisation */
rd_kafka_wrunlock(rk);
@@ -3806,6 +4229,78 @@
}
+/**
+ * @brief Add toppar to broker's active list list.
+ *
+ * For consumer this means the fetch list.
+ * For producers this is all partitions assigned to this broker.
+ *
+ * @locality broker thread
+ * @locks none
+ */
+void rd_kafka_broker_active_toppar_add (rd_kafka_broker_t *rkb,
+ rd_kafka_toppar_t *rktp) {
+ int is_consumer = rkb->rkb_rk->rk_type == RD_KAFKA_CONSUMER;
+
+ if (is_consumer && rktp->rktp_fetch)
+ return; /* Already added */
+
+ CIRCLEQ_INSERT_TAIL(&rkb->rkb_active_toppars, rktp, rktp_activelink);
+ rkb->rkb_active_toppar_cnt++;
+
+ if (is_consumer)
+ rktp->rktp_fetch = 1;
+
+ if (unlikely(rkb->rkb_active_toppar_cnt == 1))
+ rd_kafka_broker_active_toppar_next(rkb, rktp);
+
+ rd_rkb_dbg(rkb, TOPIC, "FETCHADD",
+ "Added %.*s [%"PRId32"] to %s list (%d entries, opv %d)",
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
+ rktp->rktp_partition,
+ is_consumer ? "fetch" : "active",
+ rkb->rkb_active_toppar_cnt, rktp->rktp_fetch_version);
+}
+
+
+/**
+ * @brief Remove toppar from active list.
+ *
+ * Locality: broker thread
+ * Locks: none
+ */
+void rd_kafka_broker_active_toppar_del (rd_kafka_broker_t *rkb,
+ rd_kafka_toppar_t *rktp) {
+ int is_consumer = rkb->rkb_rk->rk_type == RD_KAFKA_CONSUMER;
+
+ if (is_consumer && !rktp->rktp_fetch)
+ return; /* Not added */
+
+ CIRCLEQ_REMOVE(&rkb->rkb_active_toppars, rktp, rktp_activelink);
+ rd_kafka_assert(NULL, rkb->rkb_active_toppar_cnt > 0);
+ rkb->rkb_active_toppar_cnt--;
+
+ if (is_consumer)
+ rktp->rktp_fetch = 0;
+
+ if (rkb->rkb_active_toppar_next == rktp) {
+ /* Update next pointer */
+ rd_kafka_broker_active_toppar_next(
+ rkb, CIRCLEQ_LOOP_NEXT(&rkb->rkb_active_toppars,
+ rktp, rktp_activelink));
+ }
+
+ rd_rkb_dbg(rkb, TOPIC, "FETCHADD",
+ "Removed %.*s [%"PRId32"] from %s list "
+ "(%d entries, opv %d)",
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
+ rktp->rktp_partition,
+ is_consumer ? "fetch" : "active",
+ rkb->rkb_active_toppar_cnt, rktp->rktp_fetch_version);
+
+}
+
+
void rd_kafka_brokers_init (void) {
}
diff -Nru librdkafka-0.11.3/src/rdkafka_broker.h librdkafka-0.11.6/src/rdkafka_broker.h
--- librdkafka-0.11.3/src/rdkafka_broker.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_broker.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_BROKER_H_
+#define _RDKAFKA_BROKER_H_
#include "rdkafka_feature.h"
@@ -64,13 +65,18 @@
TAILQ_HEAD(, rd_kafka_toppar_s) rkb_toppars;
int rkb_toppar_cnt;
- /* Underflowed toppars that are eligible for fetching. */
- CIRCLEQ_HEAD(, rd_kafka_toppar_s) rkb_fetch_toppars;
- int rkb_fetch_toppar_cnt;
- rd_kafka_toppar_t *rkb_fetch_toppar_next; /* Next 'first' toppar
- * in fetch list.
- * This is used for
- * round-robin. */
+ /* Active toppars that are eligible for:
+ * - (consumer) fetching due to underflow
+ * - (producer) producing
+ *
+ * The circleq provides round-robin scheduling for both cases.
+ */
+ CIRCLEQ_HEAD(, rd_kafka_toppar_s) rkb_active_toppars;
+ int rkb_active_toppar_cnt;
+ rd_kafka_toppar_t *rkb_active_toppar_next; /* Next 'first' toppar
+ * in fetch list.
+ * This is used for
+ * round-robin. */
rd_kafka_cgrp_t *rkb_cgrp;
@@ -143,6 +149,9 @@
int rkb_req_timeouts; /* Current value */
+ rd_ts_t rkb_ts_tx_last; /**< Timestamp of last
+ * transmitted requested */
+
rd_ts_t rkb_ts_metadata_poll; /* Next metadata poll time */
int rkb_metadata_fast_poll_cnt; /* Perform fast
* metadata polls. */
@@ -162,6 +171,10 @@
rd_kafka_bufq_t rkb_retrybufs;
rd_avg_t rkb_avg_int_latency;/* Current internal latency period*/
+ rd_avg_t rkb_avg_outbuf_latency; /**< Current latency
+ * between buf_enq0
+ * and writing to socket
+ */
rd_avg_t rkb_avg_rtt; /* Current RTT period */
rd_avg_t rkb_avg_throttle; /* Current throttle period */
@@ -268,6 +281,16 @@
rd_kafka_broker_t *rd_kafka_broker_prefer (rd_kafka_t *rk, int32_t broker_id, int state);
+rd_kafka_broker_t *
+rd_kafka_broker_get_async (rd_kafka_t *rk, int32_t broker_id, int state,
+ rd_kafka_enq_once_t *eonce);
+
+rd_kafka_broker_t *rd_kafka_broker_controller (rd_kafka_t *rk, int state,
+ rd_ts_t abs_timeout);
+rd_kafka_broker_t *
+rd_kafka_broker_controller_async (rd_kafka_t *rk, int state,
+ rd_kafka_enq_once_t *eonce);
+
int rd_kafka_brokers_add0 (rd_kafka_t *rk, const char *brokerlist);
void rd_kafka_broker_set_state (rd_kafka_broker_t *rkb, int state);
@@ -275,6 +298,10 @@
int level, rd_kafka_resp_err_t err,
const char *fmt, ...);
+void rd_kafka_broker_conn_closed (rd_kafka_broker_t *rkb,
+ rd_kafka_resp_err_t err,
+ const char *errstr);
+
void rd_kafka_broker_destroy_final (rd_kafka_broker_t *rkb);
#define rd_kafka_broker_destroy(rkb) \
@@ -325,4 +352,34 @@
int rd_kafka_brokers_get_state_version (rd_kafka_t *rk);
int rd_kafka_brokers_wait_state_change (rd_kafka_t *rk, int stored_version,
int timeout_ms);
+int rd_kafka_brokers_wait_state_change_async (rd_kafka_t *rk,
+ int stored_version,
+ rd_kafka_enq_once_t *eonce);
void rd_kafka_brokers_broadcast_state_change (rd_kafka_t *rk);
+
+
+
+/**
+ * Updates the current toppar active round-robin next pointer.
+ */
+static RD_INLINE RD_UNUSED
+void rd_kafka_broker_active_toppar_next (rd_kafka_broker_t *rkb,
+ rd_kafka_toppar_t *sugg_next) {
+ if (CIRCLEQ_EMPTY(&rkb->rkb_active_toppars) ||
+ (void *)sugg_next == CIRCLEQ_ENDC(&rkb->rkb_active_toppars))
+ rkb->rkb_active_toppar_next = NULL;
+ else if (sugg_next)
+ rkb->rkb_active_toppar_next = sugg_next;
+ else
+ rkb->rkb_active_toppar_next =
+ CIRCLEQ_FIRST(&rkb->rkb_active_toppars);
+}
+
+
+void rd_kafka_broker_active_toppar_add (rd_kafka_broker_t *rkb,
+ rd_kafka_toppar_t *rktp);
+
+void rd_kafka_broker_active_toppar_del (rd_kafka_broker_t *rkb,
+ rd_kafka_toppar_t *rktp);
+
+#endif /* _RDKAFKA_BROKER_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_buf.c librdkafka-0.11.6/src/rdkafka_buf.c
--- librdkafka-0.11.3/src/rdkafka_buf.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_buf.c 2018-10-10 06:54:38.000000000 +0000
@@ -130,6 +130,8 @@
rkbuf->rkbuf_rkb = rkb;
rd_kafka_broker_keep(rkb);
+ rkbuf->rkbuf_rel_timeout = rkb->rkb_rk->rk_conf.socket_timeout_ms;
+
rkbuf->rkbuf_reqhdr.ApiKey = ApiKey;
/* Write request header, will be updated later. */
@@ -186,16 +188,16 @@
void rd_kafka_bufq_enq (rd_kafka_bufq_t *rkbufq, rd_kafka_buf_t *rkbuf) {
TAILQ_INSERT_TAIL(&rkbufq->rkbq_bufs, rkbuf, rkbuf_link);
(void)rd_atomic32_add(&rkbufq->rkbq_cnt, 1);
- (void)rd_atomic32_add(&rkbufq->rkbq_msg_cnt,
- rd_atomic32_get(&rkbuf->rkbuf_msgq.rkmq_msg_cnt));
+ (void)rd_atomic32_add(&rkbufq->rkbq_msg_cnt,
+ rkbuf->rkbuf_msgq.rkmq_msg_cnt);
}
void rd_kafka_bufq_deq (rd_kafka_bufq_t *rkbufq, rd_kafka_buf_t *rkbuf) {
TAILQ_REMOVE(&rkbufq->rkbq_bufs, rkbuf, rkbuf_link);
rd_kafka_assert(NULL, rd_atomic32_get(&rkbufq->rkbq_cnt) > 0);
(void)rd_atomic32_sub(&rkbufq->rkbq_cnt, 1);
- (void)rd_atomic32_sub(&rkbufq->rkbq_msg_cnt,
- rd_atomic32_get(&rkbuf->rkbuf_msgq.rkmq_msg_cnt));
+ (void)rd_atomic32_sub(&rkbufq->rkbq_msg_cnt,
+ rkbuf->rkbuf_msgq.rkmq_msg_cnt);
}
void rd_kafka_bufq_init(rd_kafka_bufq_t *rkbufq) {
@@ -249,6 +251,7 @@
void rd_kafka_bufq_connection_reset (rd_kafka_broker_t *rkb,
rd_kafka_bufq_t *rkbufq) {
rd_kafka_buf_t *rkbuf, *tmp;
+ rd_ts_t now = rd_clock();
rd_kafka_assert(rkb->rkb_rk, thrd_is_current(rkb->rkb_thread));
@@ -269,6 +272,8 @@
default:
/* Reset buffer send position */
rd_slice_seek(&rkbuf->rkbuf_reader, 0);
+ /* Reset timeout */
+ rd_kafka_buf_calc_timeout(rkb->rkb_rk, rkbuf, now);
break;
}
}
@@ -306,22 +311,52 @@
/**
- * Retry failed request, depending on the error.
+ * @brief Calculate the effective timeout for a request attempt
+ */
+void rd_kafka_buf_calc_timeout (const rd_kafka_t *rk, rd_kafka_buf_t *rkbuf,
+ rd_ts_t now) {
+ if (likely(rkbuf->rkbuf_rel_timeout)) {
+ /* Default:
+ * Relative timeout, set request timeout to
+ * to now + rel timeout. */
+ rkbuf->rkbuf_ts_timeout = now + rkbuf->rkbuf_rel_timeout * 1000;
+ } else {
+ /* Use absolute timeout, limited by socket.timeout.ms */
+ rd_ts_t sock_timeout = now +
+ rk->rk_conf.socket_timeout_ms * 1000;
+
+ rkbuf->rkbuf_ts_timeout =
+ RD_MIN(sock_timeout, rkbuf->rkbuf_abs_timeout);
+ }
+}
+
+/**
+ * Retry failed request, if permitted.
* @remark \p rkb may be NULL
+ * @remark the retry count is only increased for actually transmitted buffers,
+ * if there is a failure while the buffers lingers in the output queue
+ * (rkb_outbufs) then the retry counter is not increased.
* Returns 1 if the request was scheduled for retry, else 0.
*/
int rd_kafka_buf_retry (rd_kafka_broker_t *rkb, rd_kafka_buf_t *rkbuf) {
+ int incr_retry = rd_kafka_buf_was_sent(rkbuf) ? 1 : 0;
if (unlikely(!rkb ||
rkb->rkb_source == RD_KAFKA_INTERNAL ||
rd_kafka_terminating(rkb->rkb_rk) ||
- rkbuf->rkbuf_retries + 1 >
+ rkbuf->rkbuf_retries + incr_retry >
rkb->rkb_rk->rk_conf.max_retries))
return 0;
+ /* Absolute timeout, check for expiry. */
+ if (rkbuf->rkbuf_abs_timeout &&
+ rkbuf->rkbuf_abs_timeout < rd_clock())
+ return 0; /* Expired */
+
/* Try again */
rkbuf->rkbuf_ts_sent = 0;
- rkbuf->rkbuf_retries++;
+ rkbuf->rkbuf_ts_timeout = 0; /* Will be updated in calc_timeout() */
+ rkbuf->rkbuf_retries += incr_retry;
rd_kafka_buf_keep(rkbuf);
rd_kafka_broker_buf_retry(rkb, rkbuf);
return 1;
@@ -375,22 +410,14 @@
* \p response may be NULL.
*
* Will decrease refcount for both response and request, eventually.
+ *
+ * The decision to retry, and the call to buf_retry(), is delegated
+ * to the buffer's response callback.
*/
void rd_kafka_buf_callback (rd_kafka_t *rk,
rd_kafka_broker_t *rkb, rd_kafka_resp_err_t err,
rd_kafka_buf_t *response, rd_kafka_buf_t *request){
- /* Decide if the request should be retried.
- * This is always done in the originating broker thread. */
- if (unlikely(err && err != RD_KAFKA_RESP_ERR__DESTROY &&
- rd_kafka_buf_retry(rkb, request))) {
- /* refcount for retry was increased in buf_retry() so we can
- * let go of this caller's refcounts. */
- rd_kafka_buf_destroy(request);
- if (response)
- rd_kafka_buf_destroy(response);
- return;
- }
if (err != RD_KAFKA_RESP_ERR__DESTROY && request->rkbuf_replyq.q) {
rd_kafka_op_t *rko = rd_kafka_op_new(RD_KAFKA_OP_RECV_BUF);
diff -Nru librdkafka-0.11.3/src/rdkafka_buf.h librdkafka-0.11.6/src/rdkafka_buf.h
--- librdkafka-0.11.3/src/rdkafka_buf.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_buf.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_BUF_H_
+#define _RDKAFKA_BUF_H_
#include "rdkafka_int.h"
#include "rdcrc32.h"
@@ -176,6 +177,35 @@
goto err_parse; \
} while (0)
+/**
+ * @name Fail buffer reading due to buffer underflow.
+ */
+#define rd_kafka_buf_underflow_fail(rkbuf,wantedlen,...) do { \
+ if (log_decode_errors > 0) { \
+ rd_kafka_assert(NULL, rkbuf->rkbuf_rkb); \
+ char __tmpstr[256]; \
+ rd_snprintf(__tmpstr, sizeof(__tmpstr), \
+ ": " __VA_ARGS__); \
+ if (strlen(__tmpstr) == 2) __tmpstr[0] = '\0'; \
+ rd_rkb_log(rkbuf->rkbuf_rkb, log_decode_errors, \
+ "PROTOUFLOW", \
+ "Protocol read buffer underflow " \
+ "at %"PRIusz"/%"PRIusz" (%s:%i): " \
+ "expected %"PRIusz" bytes > " \
+ "%"PRIusz" remaining bytes (%s)%s", \
+ rd_slice_offset(&rkbuf->rkbuf_reader), \
+ rd_slice_size(&rkbuf->rkbuf_reader), \
+ __FUNCTION__, __LINE__, \
+ wantedlen, \
+ rd_slice_remains(&rkbuf->rkbuf_reader), \
+ rkbuf->rkbuf_uflow_mitigation ? \
+ rkbuf->rkbuf_uflow_mitigation : \
+ "incorrect broker.version.fallback?", \
+ __tmpstr); \
+ } \
+ (rkbuf)->rkbuf_err = RD_KAFKA_RESP_ERR__UNDERFLOW; \
+ goto err_parse; \
+ } while (0)
/**
@@ -190,13 +220,7 @@
#define rd_kafka_buf_check_len(rkbuf,len) do { \
size_t __len0 = (size_t)(len); \
if (unlikely(__len0 > rd_kafka_buf_read_remain(rkbuf))) { \
- rd_kafka_buf_parse_fail( \
- rkbuf, \
- "expected %"PRIusz" bytes > %"PRIusz \
- " remaining bytes", \
- __len0, rd_kafka_buf_read_remain(rkbuf)); \
- (rkbuf)->rkbuf_err = RD_KAFKA_RESP_ERR__BAD_MSG; \
- goto err_parse; \
+ rd_kafka_buf_underflow_fail(rkbuf, __len0); \
} \
} while (0)
@@ -277,7 +301,7 @@
#define rd_kafka_buf_read_i16(rkbuf,dstptr) do { \
int16_t _v; \
rd_kafka_buf_read(rkbuf, &_v, sizeof(_v)); \
- *(dstptr) = be16toh(_v); \
+ *(dstptr) = (int16_t)be16toh(_v); \
} while (0)
@@ -291,6 +315,13 @@
#define rd_kafka_buf_peek_i8(rkbuf,of,dst) rd_kafka_buf_peek(rkbuf,of,dst,1)
+#define rd_kafka_buf_read_bool(rkbuf, dstptr) do { \
+ int8_t _v; \
+ rd_bool_t *_dst = dstptr; \
+ rd_kafka_buf_read(rkbuf, &_v, 1); \
+ *_dst = (rd_bool_t)_v; \
+ } while (0)
+
/**
* @brief Read varint and store in int64_t \p dst
@@ -299,9 +330,8 @@
int64_t _v; \
size_t _r = rd_varint_dec_slice(&(rkbuf)->rkbuf_reader, &_v); \
if (unlikely(RD_UVARINT_UNDERFLOW(_r))) \
- rd_kafka_buf_parse_fail(rkbuf, \
- "varint parsing failed: " \
- "buffer underflow"); \
+ rd_kafka_buf_underflow_fail(rkbuf, (size_t)0, \
+ "varint parsing failed");\
*(dst) = _v; \
} while (0)
@@ -311,7 +341,7 @@
int _klen; \
rd_kafka_buf_read_i16a(rkbuf, (kstr)->len); \
_klen = RD_KAFKAP_STR_LEN(kstr); \
- if (RD_KAFKAP_STR_LEN0(_klen) == 0) \
+ if (RD_KAFKAP_STR_IS_NULL(kstr)) \
(kstr)->str = NULL; \
else if (!((kstr)->str = \
rd_slice_ensure_contig(&rkbuf->rkbuf_reader, \
@@ -386,9 +416,8 @@
size_t _r = rd_varint_dec_slice(&(rkbuf)->rkbuf_reader, \
&_len2); \
if (unlikely(RD_UVARINT_UNDERFLOW(_r))) \
- rd_kafka_buf_parse_fail(rkbuf, \
- "varint parsing failed: " \
- "buffer underflow"); \
+ rd_kafka_buf_underflow_fail(rkbuf, (size_t)0, \
+ "varint parsing failed"); \
(kbytes)->len = (int32_t)_len2; \
if (RD_KAFKAP_BYTES_IS_NULL(kbytes)) { \
(kbytes)->data = NULL; \
@@ -397,7 +426,7 @@
(kbytes)->data = ""; \
else if (!((kbytes)->data = \
rd_slice_ensure_contig(&(rkbuf)->rkbuf_reader, \
- _len2))) \
+ (size_t)_len2))) \
rd_kafka_buf_check_len(rkbuf, _len2); \
} while (0)
@@ -442,7 +471,12 @@
struct rd_kafkap_reqhdr rkbuf_reqhdr; /* Request header.
* These fields are encoded
* and written to output buffer
- * on buffer finalization. */
+ * on buffer finalization.
+ * Note:
+ * The request's
+ * reqhdr is copied to the
+ * response's reqhdr as a
+ * convenience. */
struct rd_kafkap_reshdr rkbuf_reshdr; /* Response header.
* Decoded fields are copied
* here from the buffer
@@ -474,7 +508,47 @@
rd_ts_t rkbuf_ts_enq;
rd_ts_t rkbuf_ts_sent; /* Initially: Absolute time of transmission,
* after response: RTT. */
- rd_ts_t rkbuf_ts_timeout;
+
+ /* Request timeouts:
+ * rkbuf_ts_timeout is the effective absolute request timeout used
+ * by the timeout scanner to see if a request has timed out.
+ * It is set when a request is enqueued on the broker transmit
+ * queue based on the relative or absolute timeout:
+ *
+ * rkbuf_rel_timeout is the per-request-transmit relative timeout,
+ * this value is reused for each sub-sequent retry of a request.
+ *
+ * rkbuf_abs_timeout is the absolute request timeout, spanning
+ * all retries.
+ * This value is effectively limited by socket.timeout.ms for
+ * each transmission, but the absolute timeout for a request's
+ * lifetime is the absolute value.
+ *
+ * Use rd_kafka_buf_set_timeout() to set a relative timeout
+ * that will be reused on retry,
+ * or rd_kafka_buf_set_abs_timeout() to set a fixed absolute timeout
+ * for the case where the caller knows the request will be
+ * semantically outdated when that absolute time expires, such as for
+ * session.timeout.ms-based requests.
+ *
+ * The decision to retry a request is delegated to the rkbuf_cb
+ * response callback, which should use rd_kafka_err_action()
+ * and check the return actions for RD_KAFKA_ERR_ACTION_RETRY to be set
+ * and then call rd_kafka_buf_retry().
+ * rd_kafka_buf_retry() will enqueue the request on the rkb_retrybufs
+ * queue with a backoff time of retry.backoff.ms.
+ * The rkb_retrybufs queue is served by the broker thread's timeout
+ * scanner.
+ * @warning rkb_retrybufs is NOT purged on broker down.
+ */
+ rd_ts_t rkbuf_ts_timeout; /* Request timeout (absolute time). */
+ rd_ts_t rkbuf_abs_timeout;/* Absolute timeout for request, including
+ * retries.
+ * Mutually exclusive with rkbuf_rel_timeout*/
+ int rkbuf_rel_timeout;/* Relative timeout (ms), used for retries.
+ * Defaults to socket.timeout.ms.
+ * Mutually exclusive with rkbuf_abs_timeout*/
+
int64_t rkbuf_offset; /* Used by OffsetCommit */
@@ -503,9 +577,23 @@
} Metadata;
} rkbuf_u;
+
+ const char *rkbuf_uflow_mitigation; /**< Buffer read underflow
+ * human readable mitigation
+ * string (const memory).
+ * This is used to hint the
+ * user why the underflow
+ * might have occurred, which
+ * depends on request type. */
};
+/**
+ * @returns true if buffer has been sent on wire, else 0.
+ */
+#define rd_kafka_buf_was_sent(rkbuf) \
+ ((rkbuf)->rkbuf_flags & RD_KAFKA_OP_F_SENT)
+
typedef struct rd_kafka_bufq_s {
TAILQ_HEAD(, rd_kafka_buf_s) rkbq_bufs;
rd_atomic32_t rkbq_cnt;
@@ -514,6 +602,47 @@
#define rd_kafka_bufq_cnt(rkbq) rd_atomic32_get(&(rkbq)->rkbq_cnt)
+/**
+ * @brief Set buffer's request timeout to relative \p timeout_ms measured
+ * from the time the buffer is sent on the underlying socket.
+ *
+ * @param now Reuse current time from existing rd_clock() var, else 0.
+ *
+ * The relative timeout value is reused upon request retry.
+ */
+static RD_INLINE void
+rd_kafka_buf_set_timeout (rd_kafka_buf_t *rkbuf, int timeout_ms, rd_ts_t now) {
+ if (!now)
+ now = rd_clock();
+ rkbuf->rkbuf_rel_timeout = timeout_ms;
+ rkbuf->rkbuf_abs_timeout = 0;
+}
+
+
+/**
+ * @brief Calculate the effective timeout for a request attempt
+ */
+void rd_kafka_buf_calc_timeout (const rd_kafka_t *rk, rd_kafka_buf_t *rkbuf,
+ rd_ts_t now);
+
+
+/**
+ * @brief Set buffer's request timeout to relative \p timeout_ms measured
+ * from \p now.
+ *
+ * @param now Reuse current time from existing rd_clock() var, else 0.
+ *
+ * The remaining time is used as timeout for request retries.
+ */
+static RD_INLINE void
+rd_kafka_buf_set_abs_timeout (rd_kafka_buf_t *rkbuf, int timeout_ms,
+ rd_ts_t now) {
+ if (!now)
+ now = rd_clock();
+ rkbuf->rkbuf_rel_timeout = 0;
+ rkbuf->rkbuf_abs_timeout = now + (timeout_ms * 1000);
+}
+
#define rd_kafka_buf_keep(rkbuf) rd_refcnt_add(&(rkbuf)->rkbuf_refcnt)
#define rd_kafka_buf_destroy(rkbuf) \
@@ -699,6 +828,20 @@
/**
+ * @brief Write varint-encoded signed value to buffer.
+ */
+static RD_INLINE size_t
+rd_kafka_buf_write_varint (rd_kafka_buf_t *rkbuf, int64_t v) {
+ char varint[RD_UVARINT_ENC_SIZEOF(v)];
+ size_t sz;
+
+ sz = rd_uvarint_enc_i64(varint, sizeof(varint), v);
+
+ return rd_kafka_buf_write(rkbuf, varint, sz);
+}
+
+
+/**
* Write (copy) Kafka string to buffer.
*/
static RD_INLINE size_t rd_kafka_buf_write_kstr (rd_kafka_buf_t *rkbuf,
@@ -817,3 +960,5 @@
return rkbuf && rkbuf->rkbuf_replyq.version &&
rkbuf->rkbuf_replyq.version < version;
}
+
+#endif /* _RDKAFKA_BUF_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka.c librdkafka-0.11.6/src/rdkafka.c
--- librdkafka-0.11.3/src/rdkafka.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka.c 2018-10-10 06:54:38.000000000 +0000
@@ -90,11 +90,37 @@
}
/**
- * Current thread's name (TLS)
+ * Current thread's log name (TLS)
*/
-char RD_TLS rd_kafka_thread_name[64] = "app";
+static char RD_TLS rd_kafka_thread_name[64] = "app";
+void rd_kafka_set_thread_name (const char *fmt, ...) {
+ va_list ap;
+ va_start(ap, fmt);
+ rd_vsnprintf(rd_kafka_thread_name, sizeof(rd_kafka_thread_name),
+ fmt, ap);
+ va_end(ap);
+}
+
+/**
+ * @brief Current thread's system name (TLS)
+ *
+ * Note the name must be 15 characters or less, because it is passed to
+ * pthread_setname_np on Linux which imposes this limit.
+ */
+static char RD_TLS rd_kafka_thread_sysname[16] = "app";
+
+void rd_kafka_set_thread_sysname (const char *fmt, ...) {
+ va_list ap;
+
+ va_start(ap, fmt);
+ rd_vsnprintf(rd_kafka_thread_sysname, sizeof(rd_kafka_thread_sysname),
+ fmt, ap);
+ va_end(ap);
+
+ thrd_setname(rd_kafka_thread_sysname);
+}
static void rd_kafka_global_init (void) {
#if ENABLE_SHAREDPTR_DEBUG
@@ -381,6 +407,14 @@
"Local: Value deserialization error"),
_ERR_DESC(RD_KAFKA_RESP_ERR__PARTIAL,
"Local: Partial response"),
+ _ERR_DESC(RD_KAFKA_RESP_ERR__READ_ONLY,
+ "Local: Read-only object"),
+ _ERR_DESC(RD_KAFKA_RESP_ERR__NOENT,
+ "Local: No such entry"),
+ _ERR_DESC(RD_KAFKA_RESP_ERR__UNDERFLOW,
+ "Local: Read underflow"),
+ _ERR_DESC(RD_KAFKA_RESP_ERR__INVALID_TYPE,
+ "Local: Invalid type"),
_ERR_DESC(RD_KAFKA_RESP_ERR_UNKNOWN,
"Unknown broker error"),
@@ -590,7 +624,7 @@
*/
void rd_kafka_destroy_final (rd_kafka_t *rk) {
- rd_kafka_assert(rk, rd_atomic32_get(&rk->rk_terminate) != 0);
+ rd_kafka_assert(rk, rd_kafka_terminating(rk));
/* Synchronize state */
rd_kafka_wrlock(rk);
@@ -657,35 +691,72 @@
}
-static void rd_kafka_destroy_app (rd_kafka_t *rk, int blocking) {
+static void rd_kafka_destroy_app (rd_kafka_t *rk, int flags) {
thrd_t thrd;
#ifndef _MSC_VER
int term_sig = rk->rk_conf.term_sig;
#endif
- rd_kafka_dbg(rk, ALL, "DESTROY", "Terminating instance");
+ char flags_str[256];
+ static const char *rd_kafka_destroy_flags_names[] = {
+ "Terminate",
+ "DestroyCalled",
+ "Immediate",
+ "NoConsumerClose",
+ NULL
+ };
+
+ /* _F_IMMEDIATE also sets .._NO_CONSUMER_CLOSE */
+ if (flags & RD_KAFKA_DESTROY_F_IMMEDIATE)
+ flags |= RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE;
+
+ rd_flags2str(flags_str, sizeof(flags_str),
+ rd_kafka_destroy_flags_names, flags);
+ rd_kafka_dbg(rk, ALL, "DESTROY", "Terminating instance "
+ "(destroy flags %s (0x%x))",
+ flags ? flags_str : "none", flags);
+
+ /* Make sure destroy is not called from a librdkafka thread
+ * since this will most likely cause a deadlock.
+ * FIXME: include broker threads (for log_cb) */
+ if (thrd_is_current(rk->rk_thread) ||
+ thrd_is_current(rk->rk_background.thread)) {
+ rd_kafka_log(rk, LOG_EMERG, "BGQUEUE",
+ "Application bug: "
+ "rd_kafka_destroy() called from "
+ "librdkafka owned thread");
+ rd_kafka_assert(NULL,
+ !*"Application bug: "
+ "calling rd_kafka_destroy() from "
+ "librdkafka owned thread is prohibited");
+ }
+
+ /* Before signaling for general termination, set the destroy
+ * flags to hint cgrp how to shut down. */
+ rd_atomic32_set(&rk->rk_terminate,
+ flags|RD_KAFKA_DESTROY_F_DESTROY_CALLED);
/* The legacy/simple consumer lacks an API to close down the consumer*/
if (rk->rk_cgrp) {
rd_kafka_dbg(rk, GENERIC, "TERMINATE",
- "Closing consumer group");
+ "Terminating consumer group handler");
rd_kafka_consumer_close(rk);
}
+ /* With the consumer closed, terminate the rest of librdkafka. */
+ rd_atomic32_set(&rk->rk_terminate, flags|RD_KAFKA_DESTROY_F_TERMINATE);
+
rd_kafka_dbg(rk, GENERIC, "TERMINATE", "Interrupting timers");
rd_kafka_wrlock(rk);
thrd = rk->rk_thread;
- rd_atomic32_add(&rk->rk_terminate, 1);
rd_kafka_timers_interrupt(&rk->rk_timers);
rd_kafka_wrunlock(rk);
rd_kafka_dbg(rk, GENERIC, "TERMINATE",
- "Sending TERMINATE to main background thread");
+ "Sending TERMINATE to internal main thread");
/* Send op to trigger queue/io wake-up.
* The op itself is (likely) ignored by the receiver. */
rd_kafka_q_enq(rk->rk_ops, rd_kafka_op_new(RD_KAFKA_OP_TERMINATE));
- rd_kafka_brokers_broadcast_state_change(rk);
-
#ifndef _MSC_VER
/* Interrupt main kafka thread to speed up termination. */
if (term_sig) {
@@ -695,14 +766,17 @@
}
#endif
- if (!blocking)
+ if (rd_kafka_destroy_flags_check(rk, RD_KAFKA_DESTROY_F_IMMEDIATE))
return; /* FIXME: thread resource leak */
rd_kafka_dbg(rk, GENERIC, "TERMINATE",
- "Joining main background thread");
+ "Joining internal main thread");
if (thrd_join(thrd, NULL) != thrd_success)
- rd_kafka_assert(NULL, !*"failed to join main thread");
+ rd_kafka_log(rk, LOG_ERR, "DESTROY",
+ "Failed to join internal main thread: %s "
+ "(was process forked?)",
+ rd_strerror(errno));
rd_kafka_destroy_final(rk);
}
@@ -711,7 +785,11 @@
/* NOTE: Must only be called by application.
* librdkafka itself must use rd_kafka_destroy0(). */
void rd_kafka_destroy (rd_kafka_t *rk) {
- rd_kafka_destroy_app(rk, 1);
+ rd_kafka_destroy_app(rk, 0);
+}
+
+void rd_kafka_destroy_flags (rd_kafka_t *rk, int flags) {
+ rd_kafka_destroy_app(rk, flags);
}
@@ -729,6 +807,23 @@
rd_kafka_dbg(rk, ALL, "DESTROY", "Destroy internal");
+ /* Trigger any state-change waiters (which should check the
+ * terminate flag whenever they wake up). */
+ rd_kafka_brokers_broadcast_state_change(rk);
+
+ if (rk->rk_background.thread) {
+ /* Send op to trigger queue/io wake-up.
+ * The op itself is (likely) ignored by the receiver. */
+ rd_kafka_q_enq(rk->rk_background.q,
+ rd_kafka_op_new(RD_KAFKA_OP_TERMINATE));
+
+ rd_kafka_dbg(rk, ALL, "DESTROY",
+ "Waiting for background queue thread "
+ "to terminate");
+ thrd_join(rk->rk_background.thread, NULL);
+ rd_kafka_q_destroy_owner(rk->rk_background.q);
+ }
+
/* Call on_destroy() interceptors */
rd_kafka_interceptors_on_destroy(rk);
@@ -780,6 +875,11 @@
rd_kafka_wrunlock(rk);
+ mtx_lock(&rk->rk_broker_state_change_lock);
+ /* Purge broker state change waiters */
+ rd_list_destroy(&rk->rk_broker_state_change_waiters);
+ mtx_unlock(&rk->rk_broker_state_change_lock);
+
rd_kafka_dbg(rk, GENERIC, "TERMINATE",
"Purging reply queue");
@@ -820,31 +920,96 @@
rd_list_destroy(&wait_thrds);
}
+/**
+ * @brief Buffer state for stats emitter
+ */
+struct _stats_emit {
+ char *buf; /* Pointer to allocated buffer */
+ size_t size; /* Current allocated size of buf */
+ size_t of; /* Current write-offset in buf */
+};
+
+
+/* Stats buffer printf. Requires a (struct _stats_emit *)st variable in the
+ * current scope. */
+#define _st_printf(...) do { \
+ ssize_t _r; \
+ ssize_t _rem = st->size - st->of; \
+ _r = rd_snprintf(st->buf+st->of, _rem, __VA_ARGS__); \
+ if (_r >= _rem) { \
+ st->size *= 2; \
+ _rem = st->size - st->of; \
+ st->buf = rd_realloc(st->buf, st->size); \
+ _r = rd_snprintf(st->buf+st->of, _rem, __VA_ARGS__); \
+ } \
+ st->of += _r; \
+ } while (0)
+
+struct _stats_total {
+ int64_t tx; /**< broker.tx */
+ int64_t tx_bytes; /**< broker.tx_bytes */
+ int64_t rx; /**< broker.rx */
+ int64_t rx_bytes; /**< broker.rx_bytes */
+ int64_t txmsgs; /**< partition.txmsgs */
+ int64_t txmsg_bytes; /**< partition.txbytes */
+ int64_t rxmsgs; /**< partition.rxmsgs */
+ int64_t rxmsg_bytes; /**< partition.rxbytes */
+};
+
+
-/* Stats buffer printf */
-#define _st_printf(...) do { \
- ssize_t r; \
- ssize_t rem = size-of; \
- r = rd_snprintf(buf+of, rem, __VA_ARGS__); \
- if (r >= rem) { \
- size *= 2; \
- rem = size-of; \
- buf = rd_realloc(buf, size); \
- r = rd_snprintf(buf+of, rem, __VA_ARGS__); \
- } \
- of += r; \
- } while (0)
+/**
+ * @brief Rollover and emit an average window.
+ */
+static RD_INLINE void rd_kafka_stats_emit_avg (struct _stats_emit *st,
+ const char *name,
+ rd_avg_t *src_avg) {
+ rd_avg_t avg;
+
+ rd_avg_rollover(&avg, src_avg);
+ _st_printf(
+ "\"%s\": {"
+ " \"min\":%"PRId64","
+ " \"max\":%"PRId64","
+ " \"avg\":%"PRId64","
+ " \"sum\":%"PRId64","
+ " \"stddev\": %"PRId64","
+ " \"p50\": %"PRId64","
+ " \"p75\": %"PRId64","
+ " \"p90\": %"PRId64","
+ " \"p95\": %"PRId64","
+ " \"p99\": %"PRId64","
+ " \"p99_99\": %"PRId64","
+ " \"outofrange\": %"PRId64","
+ " \"hdrsize\": %"PRId32","
+ " \"cnt\":%i "
+ "}, ",
+ name,
+ avg.ra_v.minv,
+ avg.ra_v.maxv,
+ avg.ra_v.avg,
+ avg.ra_v.sum,
+ (int64_t)avg.ra_hist.stddev,
+ avg.ra_hist.p50,
+ avg.ra_hist.p75,
+ avg.ra_hist.p90,
+ avg.ra_hist.p95,
+ avg.ra_hist.p99,
+ avg.ra_hist.p99_99,
+ avg.ra_hist.oor,
+ avg.ra_hist.hdrsize,
+ avg.ra_v.cnt);
+ rd_avg_destroy(&avg);
+}
/**
* Emit stats for toppar
*/
-static RD_INLINE void rd_kafka_stats_emit_toppar (char **bufp, size_t *sizep,
- size_t *ofp,
- rd_kafka_toppar_t *rktp,
- int first) {
- char *buf = *bufp;
- size_t size = *sizep;
- size_t of = *ofp;
+static RD_INLINE void rd_kafka_stats_emit_toppar (struct _stats_emit *st,
+ struct _stats_total *total,
+ rd_kafka_toppar_t *rktp,
+ int first) {
+ rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk;
int64_t consumer_lag = -1;
struct offset_stats offs;
int32_t leader_nodeid = -1;
@@ -860,13 +1025,20 @@
/* Grab a copy of the latest finalized offset stats */
offs = rktp->rktp_offsets_fin;
+ /* Calculate consumer_lag by using the highest offset
+ * of app_offset (the last message passed to application + 1)
+ * or the committed_offset (the last message committed by this or
+ * another consumer).
+ * Using app_offset allows consumer_lag to be up to date even if
+ * offsets are not (yet) committed.
+ */
if (rktp->rktp_hi_offset != RD_KAFKA_OFFSET_INVALID &&
- rktp->rktp_app_offset >= 0) {
- if (unlikely(rktp->rktp_app_offset > rktp->rktp_hi_offset))
+ (rktp->rktp_app_offset >= 0 || rktp->rktp_committed_offset >= 0)) {
+ consumer_lag = rktp->rktp_hi_offset -
+ RD_MAX(rktp->rktp_app_offset,
+ rktp->rktp_committed_offset);
+ if (unlikely(consumer_lag) < 0)
consumer_lag = 0;
- else
- consumer_lag = rktp->rktp_hi_offset -
- rktp->rktp_app_offset;
}
_st_printf("%s\"%"PRId32"\": { "
@@ -875,9 +1047,9 @@
"\"desired\":%s, "
"\"unknown\":%s, "
"\"msgq_cnt\":%i, "
- "\"msgq_bytes\":%"PRIu64", "
+ "\"msgq_bytes\":%"PRIusz", "
"\"xmit_msgq_cnt\":%i, "
- "\"xmit_msgq_bytes\":%"PRIu64", "
+ "\"xmit_msgq_bytes\":%"PRIusz", "
"\"fetchq_cnt\":%i, "
"\"fetchq_size\":%"PRIu64", "
"\"fetch_state\":\"%s\", "
@@ -893,6 +1065,8 @@
"\"consumer_lag\":%"PRId64", "
"\"txmsgs\":%"PRIu64", "
"\"txbytes\":%"PRIu64", "
+ "\"rxmsgs\":%"PRIu64", "
+ "\"rxbytes\":%"PRIu64", "
"\"msgs\": %"PRIu64", "
"\"rx_ver_drops\": %"PRIu64" "
"} ",
@@ -902,10 +1076,11 @@
leader_nodeid,
(rktp->rktp_flags&RD_KAFKA_TOPPAR_F_DESIRED)?"true":"false",
(rktp->rktp_flags&RD_KAFKA_TOPPAR_F_UNKNOWN)?"true":"false",
- rd_atomic32_get(&rktp->rktp_msgq.rkmq_msg_cnt),
- rd_atomic64_get(&rktp->rktp_msgq.rkmq_msg_bytes),
- rd_atomic32_get(&rktp->rktp_xmit_msgq.rkmq_msg_cnt),
- rd_atomic64_get(&rktp->rktp_xmit_msgq.rkmq_msg_bytes),
+ rd_kafka_msgq_len(&rktp->rktp_msgq),
+ rd_kafka_msgq_size(&rktp->rktp_msgq),
+ /* FIXME: xmit_msgq is local to the broker thread. */
+ 0,
+ (size_t)0,
rd_kafka_q_len(rktp->rktp_fetchq),
rd_kafka_q_size(rktp->rktp_fetchq),
rd_kafka_fetch_states[rktp->rktp_fetch_state],
@@ -920,24 +1095,28 @@
rktp->rktp_hi_offset,
consumer_lag,
rd_atomic64_get(&rktp->rktp_c.tx_msgs),
- rd_atomic64_get(&rktp->rktp_c.tx_bytes),
- rd_atomic64_get(&rktp->rktp_c.msgs),
+ rd_atomic64_get(&rktp->rktp_c.tx_msg_bytes),
+ rd_atomic64_get(&rktp->rktp_c.rx_msgs),
+ rd_atomic64_get(&rktp->rktp_c.rx_msg_bytes),
+ rk->rk_type == RD_KAFKA_PRODUCER ?
+ rd_atomic64_get(&rktp->rktp_c.producer_enq_msgs) :
+ rd_atomic64_get(&rktp->rktp_c.rx_msgs), /* legacy, same as rx_msgs */
rd_atomic64_get(&rktp->rktp_c.rx_ver_drops));
- rd_kafka_toppar_unlock(rktp);
+ if (total) {
+ total->txmsgs += rd_atomic64_get(&rktp->rktp_c.tx_msgs);
+ total->txmsg_bytes += rd_atomic64_get(&rktp->rktp_c.tx_msg_bytes);
+ total->rxmsgs += rd_atomic64_get(&rktp->rktp_c.rx_msgs);
+ total->rxmsg_bytes += rd_atomic64_get(&rktp->rktp_c.rx_msg_bytes);
+ }
- *bufp = buf;
- *sizep = size;
- *ofp = of;
+ rd_kafka_toppar_unlock(rktp);
}
/**
* Emit all statistics
*/
static void rd_kafka_stats_emit_all (rd_kafka_t *rk) {
- char *buf;
- size_t size = 1024*10;
- size_t of = 0;
rd_kafka_broker_t *rkb;
rd_kafka_itopic_t *rkt;
shptr_rd_kafka_toppar_t *s_rktp;
@@ -945,8 +1124,11 @@
rd_kafka_op_t *rko;
unsigned int tot_cnt;
size_t tot_size;
+ struct _stats_emit stx = { .size = 1024*10 };
+ struct _stats_emit *st = &stx;
+ struct _stats_total total = {0};
- buf = rd_malloc(size);
+ st->buf = rd_malloc(st->size);
rd_kafka_curr_msgs_get(rk, &tot_cnt, &tot_size);
@@ -955,6 +1137,7 @@
now = rd_clock();
_st_printf("{ "
"\"name\": \"%s\", "
+ "\"client_id\": \"%s\", "
"\"type\": \"%s\", "
"\"ts\":%"PRId64", "
"\"time\":%lli, "
@@ -967,6 +1150,7 @@
"\"metadata_cache_cnt\":%i, "
"\"brokers\":{ "/*open brokers*/,
rk->rk_name,
+ rk->rk_conf.client_id_str,
rd_kafka_type2str(rk->rk_type),
now,
(signed long long)time(NULL),
@@ -978,13 +1162,9 @@
TAILQ_FOREACH(rkb, &rk->rk_brokers, rkb_link) {
- rd_avg_t rtt, throttle, int_latency;
rd_kafka_toppar_t *rktp;
rd_kafka_broker_lock(rkb);
- rd_avg_rollover(&int_latency, &rkb->rkb_avg_int_latency);
- rd_avg_rollover(&rtt, &rkb->rkb_avg_rtt);
- rd_avg_rollover(&throttle, &rkb->rkb_avg_throttle);
_st_printf("%s\"%s\": { "/*open broker*/
"\"name\":\"%s\", "
"\"nodeid\":%"PRId32", "
@@ -1006,29 +1186,7 @@
"\"rxpartial\":%"PRIu64", "
"\"zbuf_grow\":%"PRIu64", "
"\"buf_grow\":%"PRIu64", "
- "\"wakeups\":%"PRIu64", "
- "\"int_latency\": {"
- " \"min\":%"PRId64","
- " \"max\":%"PRId64","
- " \"avg\":%"PRId64","
- " \"sum\":%"PRId64","
- " \"cnt\":%i "
- "}, "
- "\"rtt\": {"
- " \"min\":%"PRId64","
- " \"max\":%"PRId64","
- " \"avg\":%"PRId64","
- " \"sum\":%"PRId64","
- " \"cnt\":%i "
- "}, "
- "\"throttle\": {"
- " \"min\":%"PRId64","
- " \"max\":%"PRId64","
- " \"avg\":%"PRId64","
- " \"sum\":%"PRId64","
- " \"cnt\":%i "
- "}, "
- "\"toppars\":{ "/*open toppars*/,
+ "\"wakeups\":%"PRIu64", ",
rkb == TAILQ_FIRST(&rk->rk_brokers) ? "" : ", ",
rkb->rkb_name,
rkb->rkb_name,
@@ -1051,22 +1209,21 @@
rd_atomic64_get(&rkb->rkb_c.rx_partial),
rd_atomic64_get(&rkb->rkb_c.zbuf_grow),
rd_atomic64_get(&rkb->rkb_c.buf_grow),
- rd_atomic64_get(&rkb->rkb_c.wakeups),
- int_latency.ra_v.minv,
- int_latency.ra_v.maxv,
- int_latency.ra_v.avg,
- int_latency.ra_v.sum,
- int_latency.ra_v.cnt,
- rtt.ra_v.minv,
- rtt.ra_v.maxv,
- rtt.ra_v.avg,
- rtt.ra_v.sum,
- rtt.ra_v.cnt,
- throttle.ra_v.minv,
- throttle.ra_v.maxv,
- throttle.ra_v.avg,
- throttle.ra_v.sum,
- throttle.ra_v.cnt);
+ rd_atomic64_get(&rkb->rkb_c.wakeups));
+
+ total.tx += rd_atomic64_get(&rkb->rkb_c.tx);
+ total.tx_bytes += rd_atomic64_get(&rkb->rkb_c.tx_bytes);
+ total.rx += rd_atomic64_get(&rkb->rkb_c.rx);
+ total.rx_bytes += rd_atomic64_get(&rkb->rkb_c.rx_bytes);
+
+ rd_kafka_stats_emit_avg(st, "int_latency",
+ &rkb->rkb_avg_int_latency);
+ rd_kafka_stats_emit_avg(st, "outbuf_latency",
+ &rkb->rkb_avg_outbuf_latency);
+ rd_kafka_stats_emit_avg(st, "rtt", &rkb->rkb_avg_rtt);
+ rd_kafka_stats_emit_avg(st, "throttle", &rkb->rkb_avg_throttle);
+
+ _st_printf("\"toppars\":{ "/*open toppars*/);
TAILQ_FOREACH(rktp, &rkb->rkb_toppars, rktp_rkblink) {
_st_printf("%s\"%.*s-%"PRId32"\": { "
@@ -1095,30 +1252,40 @@
rd_kafka_topic_rdlock(rkt);
_st_printf("%s\"%.*s\": { "
"\"topic\":\"%.*s\", "
- "\"metadata_age\":%"PRId64", "
- "\"partitions\":{ " /*open partitions*/,
+ "\"metadata_age\":%"PRId64", ",
rkt==TAILQ_FIRST(&rk->rk_topics)?"":", ",
RD_KAFKAP_STR_PR(rkt->rkt_topic),
RD_KAFKAP_STR_PR(rkt->rkt_topic),
rkt->rkt_ts_metadata ?
(rd_clock() - rkt->rkt_ts_metadata)/1000 : 0);
- for (i = 0 ; i < rkt->rkt_partition_cnt ; i++)
- rd_kafka_stats_emit_toppar(&buf, &size, &of,
- rd_kafka_toppar_s2i(rkt->rkt_p[i]),
- i == 0);
+ rd_kafka_stats_emit_avg(st, "batchsize",
+ &rkt->rkt_avg_batchsize);
+ rd_kafka_stats_emit_avg(st, "batchcnt",
+ &rkt->rkt_avg_batchcnt);
+
+ _st_printf("\"partitions\":{ " /*open partitions*/);
+
+ for (i = 0 ; i < rkt->rkt_partition_cnt ; i++)
+ rd_kafka_stats_emit_toppar(
+ st, &total,
+ rd_kafka_toppar_s2i(rkt->rkt_p[i]),
+ i == 0);
RD_LIST_FOREACH(s_rktp, &rkt->rkt_desp, j)
- rd_kafka_stats_emit_toppar(&buf, &size, &of,
- rd_kafka_toppar_s2i(s_rktp),
- i+j == 0);
+ rd_kafka_stats_emit_toppar(
+ st, &total,
+ rd_kafka_toppar_s2i(s_rktp),
+ i+j == 0);
i += j;
- if (rkt->rkt_ua)
- rd_kafka_stats_emit_toppar(&buf, &size, &of,
- rd_kafka_toppar_s2i(rkt->rkt_ua),
- i++ == 0);
+ if (rkt->rkt_ua)
+ rd_kafka_stats_emit_toppar(
+ st, NULL,
+ rd_kafka_toppar_s2i(rkt->rkt_ua),
+ i++ == 0);
+
rd_kafka_topic_rdunlock(rkt);
_st_printf("} "/*close partitions*/
@@ -1140,14 +1307,33 @@
}
rd_kafka_rdunlock(rk);
+ /* Total counters */
+ _st_printf(", "
+ "\"tx\":%"PRId64", "
+ "\"tx_bytes\":%"PRId64", "
+ "\"rx\":%"PRId64", "
+ "\"rx_bytes\":%"PRId64", "
+ "\"txmsgs\":%"PRId64", "
+ "\"txmsg_bytes\":%"PRId64", "
+ "\"rxmsgs\":%"PRId64", "
+ "\"rxmsg_bytes\":%"PRId64,
+ total.tx,
+ total.tx_bytes,
+ total.rx,
+ total.rx_bytes,
+ total.txmsgs,
+ total.txmsg_bytes,
+ total.rxmsgs,
+ total.rxmsg_bytes);
+
_st_printf("}"/*close object*/);
/* Enqueue op for application */
rko = rd_kafka_op_new(RD_KAFKA_OP_STATS);
rd_kafka_op_set_prio(rko, RD_KAFKA_PRIO_HIGH);
- rko->rko_u.stats.json = buf;
- rko->rko_u.stats.json_len = of;
+ rko->rko_u.stats.json = st->buf;
+ rko->rko_u.stats.json_len = st->of;
rd_kafka_q_enq(rk->rk_rep, rko);
}
@@ -1188,6 +1374,9 @@
}
+
+
+
/**
* Main loop for Kafka handler thread.
*/
@@ -1197,7 +1386,8 @@
rd_kafka_timer_t tmr_stats_emit = RD_ZERO_INIT;
rd_kafka_timer_t tmr_metadata_refresh = RD_ZERO_INIT;
- rd_snprintf(rd_kafka_thread_name, sizeof(rd_kafka_thread_name), "main");
+ rd_kafka_set_thread_name("main");
+ rd_kafka_set_thread_sysname("rdk:main");
(void)rd_atomic32_add(&rd_kafka_thread_cnt_curr, 1);
@@ -1208,9 +1398,10 @@
rd_kafka_timer_start(&rk->rk_timers, &tmr_topic_scan, 1000000,
rd_kafka_topic_scan_tmr_cb, NULL);
- rd_kafka_timer_start(&rk->rk_timers, &tmr_stats_emit,
- rk->rk_conf.stats_interval_ms * 1000ll,
- rd_kafka_stats_emit_tmr_cb, NULL);
+ if (rk->rk_conf.stats_interval_ms)
+ rd_kafka_timer_start(&rk->rk_timers, &tmr_stats_emit,
+ rk->rk_conf.stats_interval_ms * 1000ll,
+ rd_kafka_stats_emit_tmr_cb, NULL);
if (rk->rk_conf.metadata_refresh_interval_ms > 0)
rd_kafka_timer_start(&rk->rk_timers, &tmr_metadata_refresh,
rk->rk_conf.metadata_refresh_interval_ms *
@@ -1233,11 +1424,15 @@
rd_kafka_timers_run(&rk->rk_timers, RD_POLL_NOWAIT);
}
+ rd_kafka_dbg(rk, GENERIC, "TERMINATE",
+ "Internal main thread terminating");
+
rd_kafka_q_disable(rk->rk_ops);
rd_kafka_q_purge(rk->rk_ops);
rd_kafka_timer_stop(&rk->rk_timers, &tmr_topic_scan, 1);
- rd_kafka_timer_stop(&rk->rk_timers, &tmr_stats_emit, 1);
+ if (rk->rk_conf.stats_interval_ms)
+ rd_kafka_timer_stop(&rk->rk_timers, &tmr_stats_emit, 1);
rd_kafka_timer_stop(&rk->rk_timers, &tmr_metadata_refresh, 1);
/* Synchronise state */
@@ -1247,7 +1442,7 @@
rd_kafka_destroy_internal(rk);
rd_kafka_dbg(rk, GENERIC, "TERMINATE",
- "Main background thread exiting");
+ "Internal main thread termination done");
rd_atomic32_sub(&rd_kafka_thread_cnt_curr, 1);
@@ -1308,6 +1503,30 @@
return NULL;
}
+#if WITH_SSL
+ if (conf->ssl.keystore_location && !conf->ssl.keystore_password) {
+ rd_snprintf(errstr, errstr_size,
+ "Mandatory config property 'ssl.keystore.password' not set (mandatory because 'ssl.keystore.location' is set)");
+ if (!app_conf)
+ rd_kafka_conf_destroy(conf);
+ rd_kafka_set_last_error(RD_KAFKA_RESP_ERR__INVALID_ARG, EINVAL);
+ return NULL;
+ }
+#endif
+
+ if (type == RD_KAFKA_CONSUMER) {
+ /* Automatically adjust `fetch.max.bytes` to be >=
+ * `message.max.bytes`. */
+ conf->fetch_max_bytes = RD_MAX(conf->fetch_max_bytes,
+ conf->max_msg_size);
+
+ /* Automatically adjust 'receive.message.max.bytes' to
+ * be 512 bytes larger than 'fetch.max.bytes' to have enough
+ * room for protocol framing (including topic name). */
+ conf->recv_max_msg_size = RD_MAX(conf->recv_max_msg_size,
+ conf->fetch_max_bytes + 512);
+ }
+
if (conf->metadata_max_age_ms == -1) {
if (conf->metadata_refresh_interval_ms > 0)
conf->metadata_max_age_ms =
@@ -1342,6 +1561,8 @@
cnd_init(&rk->rk_broker_state_change_cnd);
mtx_init(&rk->rk_broker_state_change_lock, mtx_plain);
+ rd_list_init(&rk->rk_broker_state_change_waiters, 8,
+ rd_kafka_enq_once_trigger_destroy);
rk->rk_rep = rd_kafka_q_new(rk);
rk->rk_ops = rd_kafka_q_new(rk);
@@ -1365,6 +1586,13 @@
rk->rk_conf.enabled_events |= RD_KAFKA_EVENT_REBALANCE;
if (rk->rk_conf.offset_commit_cb)
rk->rk_conf.enabled_events |= RD_KAFKA_EVENT_OFFSET_COMMIT;
+ if (rk->rk_conf.error_cb)
+ rk->rk_conf.enabled_events |= RD_KAFKA_EVENT_ERROR;
+
+ rk->rk_controllerid = -1;
+
+ /* Admin client defaults */
+ rk->rk_conf.admin.request_timeout_ms = rk->rk_conf.socket_timeout_ms;
/* Convenience Kafka protocol null bytes */
rk->rk_null_bytes = rd_kafkap_bytes_new(NULL, 0);
@@ -1443,7 +1671,7 @@
#ifndef _MSC_VER
- /* Block all signals in newly created thread.
+ /* Block all signals in newly created threads.
* To avoid race condition we block all signals in the calling
* thread, which the new thread will inherit its sigmask from,
* and then restore the original sigmask of the calling thread when
@@ -1459,6 +1687,41 @@
pthread_sigmask(SIG_SETMASK, &newset, &oldset);
#endif
+ /* Create background thread and queue if background_event_cb()
+ * has been configured.
+ * Do this before creating the main thread since after
+ * the main thread is created it is no longer trivial to error
+ * out from rd_kafka_new(). */
+ if (rk->rk_conf.background_event_cb) {
+ /* Hold off background thread until thrd_create() is done. */
+ rd_kafka_wrlock(rk);
+
+ rk->rk_background.q = rd_kafka_q_new(rk);
+
+ if ((thrd_create(&rk->rk_background.thread,
+ rd_kafka_background_thread_main, rk)) !=
+ thrd_success) {
+ ret_err = RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE;
+ ret_errno = errno;
+ if (errstr)
+ rd_snprintf(errstr, errstr_size,
+ "Failed to create background "
+ "thread: %s (%i)",
+ rd_strerror(errno), errno);
+ rd_kafka_wrunlock(rk);
+
+#ifndef _MSC_VER
+ /* Restore sigmask of caller */
+ pthread_sigmask(SIG_SETMASK, &oldset, NULL);
+#endif
+ goto fail;
+ }
+
+ rd_kafka_wrunlock(rk);
+ }
+
+
+
/* Lock handle here to synchronise state, i.e., hold off
* the thread until we've finalized the handle. */
rd_kafka_wrlock(rk);
@@ -1482,6 +1745,10 @@
rd_kafka_wrunlock(rk);
+ /*
+ * @warning `goto fail` is prohibited past this point
+ */
+
rk->rk_eos.PID = -1;
rk->rk_eos.TransactionalId = rd_kafkap_str_new(NULL, 0);
@@ -1512,6 +1779,13 @@
rk->rk_initialized = 1;
+ rd_kafka_dbg(rk, ALL, "INIT",
+ "librdkafka v%s (0x%x) %s initialized "
+ "(builtin.features 0x%x, debug 0x%x)",
+ rd_kafka_version_str(), rd_kafka_version(),
+ rk->rk_name,
+ rk->rk_conf.builtin_features, rk->rk_conf.debug);
+
return rk;
fail:
@@ -1519,6 +1793,16 @@
* Error out and clean up
*/
+ /*
+ * Tell background thread to terminate and wait for it to return.
+ */
+ rd_atomic32_set(&rk->rk_terminate, RD_KAFKA_DESTROY_F_TERMINATE);
+
+ if (rk->rk_background.thread) {
+ thrd_join(rk->rk_background.thread, NULL);
+ rd_kafka_q_destroy_owner(rk->rk_background.q);
+ }
+
/* If on_new() interceptors have been called we also need
* to allow interceptor clean-up by calling on_destroy() */
rd_kafka_interceptors_on_destroy(rk);
@@ -1535,7 +1819,6 @@
memset(&rk->rk_conf, 0, sizeof(rk->rk_conf));
}
- rd_atomic32_add(&rk->rk_terminate, 1);
rd_kafka_destroy_internal(rk);
rd_kafka_destroy_final(rk);
@@ -1743,6 +2026,7 @@
rd_kafka_toppar_t *rktp;
rd_kafka_q_t *tmpq = NULL;
rd_kafka_resp_err_t err;
+ rd_kafka_replyq_t replyq = RD_KAFKA_NO_REPLYQ;
/* FIXME: simple consumer check */
@@ -1757,12 +2041,13 @@
}
rd_kafka_topic_rdunlock(rkt);
- if (timeout_ms)
+ if (timeout_ms) {
tmpq = rd_kafka_q_new(rkt->rkt_rk);
+ replyq = RD_KAFKA_REPLYQ(tmpq, 0);
+ }
rktp = rd_kafka_toppar_s2i(s_rktp);
- if ((err = rd_kafka_toppar_op_seek(rktp, offset,
- RD_KAFKA_REPLYQ(tmpq, 0)))) {
+ if ((err = rd_kafka_toppar_op_seek(rktp, offset, replyq))) {
if (tmpq)
rd_kafka_q_destroy_owner(tmpq);
rd_kafka_toppar_destroy(s_rktp);
@@ -2071,6 +2356,8 @@
if (!(rkcg = rd_kafka_cgrp_get(rk)))
return RD_KAFKA_RESP_ERR__UNKNOWN_GROUP;
+ rd_kafka_dbg(rk, CONSUMER, "CLOSE", "Closing consumer");
+
/* Redirect cgrp queue to our temporary queue to make sure
* all posted ops (e.g., rebalance callbacks) are served by
* this function. */
@@ -2079,25 +2366,42 @@
rd_kafka_cgrp_terminate(rkcg, RD_KAFKA_REPLYQ(rkq, 0)); /* async */
- while ((rko = rd_kafka_q_pop(rkq, RD_POLL_INFINITE, 0))) {
- rd_kafka_op_res_t res;
- if ((rko->rko_type & ~RD_KAFKA_OP_FLAGMASK) ==
- RD_KAFKA_OP_TERMINATE) {
- err = rko->rko_err;
- rd_kafka_op_destroy(rko);
- break;
+ /* Disable the queue if termination is immediate or the user
+ * does not want the blocking consumer_close() behaviour, this will
+ * cause any ops posted for this queue (such as rebalance) to
+ * be destroyed.
+ */
+ if (rd_kafka_destroy_flags_no_consumer_close(rk)) {
+ rd_kafka_dbg(rk, CONSUMER, "CLOSE",
+ "Disabling and purging temporary queue to quench "
+ "close events");
+ rd_kafka_q_disable(rkq);
+ /* Purge ops already enqueued */
+ rd_kafka_q_purge(rkq);
+ } else {
+ rd_kafka_dbg(rk, CONSUMER, "CLOSE",
+ "Waiting for close events");
+ while ((rko = rd_kafka_q_pop(rkq, RD_POLL_INFINITE, 0))) {
+ rd_kafka_op_res_t res;
+ if ((rko->rko_type & ~RD_KAFKA_OP_FLAGMASK) ==
+ RD_KAFKA_OP_TERMINATE) {
+ err = rko->rko_err;
+ rd_kafka_op_destroy(rko);
+ break;
+ }
+ res = rd_kafka_poll_cb(rk, rkq, rko,
+ RD_KAFKA_Q_CB_RETURN, NULL);
+ if (res == RD_KAFKA_OP_RES_PASS)
+ rd_kafka_op_destroy(rko);
+ /* Ignore YIELD, we need to finish */
}
- res = rd_kafka_poll_cb(rk, rkq, rko,
- RD_KAFKA_Q_CB_RETURN, NULL);
- if (res == RD_KAFKA_OP_RES_PASS)
- rd_kafka_op_destroy(rko);
- /* Ignore YIELD, we need to finish */
}
rd_kafka_q_fwd_set(rkcg->rkcg_q, NULL);
rd_kafka_q_destroy_owner(rkq);
+ rd_kafka_dbg(rk, CONSUMER, "CLOSE", "Consumer closed");
return err;
}
@@ -2511,23 +2815,27 @@
/**
- * rd_kafka_poll() (and similar) op callback handler.
- * Will either call registered callback depending on cb_type and op type
- * or return op to application, if applicable (e.g., fetch message).
+ * @brief rd_kafka_poll() (and similar) op callback handler.
+ * Will either call registered callback depending on cb_type and op type
+ * or return op to application, if applicable (e.g., fetch message).
*
- * Returns 1 if op was handled, else 0.
+ * @returns RD_KAFKA_OP_RES_HANDLED if op was handled, else one of the
+ * other res types (such as OP_RES_PASS).
*
- * Locality: application thread
+ * @locality application thread
*/
rd_kafka_op_res_t
rd_kafka_poll_cb (rd_kafka_t *rk, rd_kafka_q_t *rkq, rd_kafka_op_t *rko,
rd_kafka_q_cb_type_t cb_type, void *opaque) {
rd_kafka_msg_t *rkm;
+ rd_kafka_op_res_t res = RD_KAFKA_OP_RES_HANDLED;
- /* Return-as-event requested, see if op can be converted to event,
- * otherwise fall through and trigger callbacks. */
- if (cb_type == RD_KAFKA_Q_CB_EVENT && rd_kafka_event_setup(rk, rko))
- return 0; /* Return as event */
+ /* Special handling for events based on cb_type */
+ if (cb_type == RD_KAFKA_Q_CB_EVENT &&
+ rd_kafka_event_setup(rk, rko)) {
+ /* Return-as-event requested. */
+ return RD_KAFKA_OP_RES_PASS; /* Return as event */
+ }
switch ((int)rko->rko_type)
{
@@ -2592,13 +2900,25 @@
rk->rk_conf.error_cb(rk, rko->rko_err,
rko->rko_u.err.errstr,
rk->rk_conf.opaque);
- else
- rd_kafka_log(rk, LOG_ERR, "ERROR",
- "%s: %s: %s",
- rk->rk_name,
- rd_kafka_err2str(rko->rko_err),
- rko->rko_u.err.errstr);
- break;
+ else {
+ /* If error string already contains
+ * the err2str then skip including err2str in
+ * the printout */
+ if (rko->rko_u.err.errstr &&
+ strstr(rko->rko_u.err.errstr,
+ rd_kafka_err2str(rko->rko_err)))
+ rd_kafka_log(rk, LOG_ERR, "ERROR",
+ "%s: %s",
+ rk->rk_name,
+ rko->rko_u.err.errstr);
+ else
+ rd_kafka_log(rk, LOG_ERR, "ERROR",
+ "%s: %s: %s",
+ rk->rk_name,
+ rko->rko_u.err.errstr,
+ rd_kafka_err2str(rko->rko_err));
+ }
+ break;
case RD_KAFKA_OP_DR:
/* Delivery report:
@@ -2675,14 +2995,33 @@
/* nop: just a wake-up */
break;
+ case RD_KAFKA_OP_CREATETOPICS:
+ case RD_KAFKA_OP_DELETETOPICS:
+ case RD_KAFKA_OP_CREATEPARTITIONS:
+ case RD_KAFKA_OP_ALTERCONFIGS:
+ case RD_KAFKA_OP_DESCRIBECONFIGS:
+ /* Calls op_destroy() from worker callback,
+ * when the time comes. */
+ res = rd_kafka_op_call(rk, rkq, rko);
+ break;
+
+ case RD_KAFKA_OP_ADMIN_RESULT:
+ if (cb_type == RD_KAFKA_Q_CB_RETURN ||
+ cb_type == RD_KAFKA_Q_CB_FORCE_RETURN)
+ return RD_KAFKA_OP_RES_PASS; /* Don't handle here */
+
+ /* Op is silently destroyed below */
+ break;
+
default:
rd_kafka_assert(rk, !*"cant handle op type");
break;
}
- rd_kafka_op_destroy(rko);
+ if (res == RD_KAFKA_OP_RES_HANDLED)
+ rd_kafka_op_destroy(rko);
- return 1; /* op was handled */
+ return res;
}
int rd_kafka_poll (rd_kafka_t *rk, int timeout_ms) {
@@ -2723,9 +3062,10 @@
"%s xmit_msgq: %i messages\n"
"%s total: %"PRIu64" messages, %"PRIu64" bytes\n",
indent, rd_refcnt_get(&rktp->rktp_refcnt),
- indent, rd_atomic32_get(&rktp->rktp_msgq.rkmq_msg_cnt),
- indent, rd_atomic32_get(&rktp->rktp_xmit_msgq.rkmq_msg_cnt),
- indent, rd_atomic64_get(&rktp->rktp_c.tx_msgs), rd_atomic64_get(&rktp->rktp_c.tx_bytes));
+ indent, rktp->rktp_msgq.rkmq_msg_cnt,
+ indent, rktp->rktp_xmit_msgq.rkmq_msg_cnt,
+ indent, rd_atomic64_get(&rktp->rktp_c.tx_msgs),
+ rd_atomic64_get(&rktp->rktp_c.tx_msg_bytes));
}
static void rd_kafka_broker_dump (FILE *fp, rd_kafka_broker_t *rkb, int locks) {
@@ -2932,13 +3272,58 @@
}
+int32_t rd_kafka_controllerid (rd_kafka_t *rk, int timeout_ms) {
+ rd_ts_t abs_timeout = rd_timeout_init(timeout_ms);
+
+ /* ControllerId is returned in Metadata >=V1 responses and
+ * cached on the rk. If no cached value is available
+ * it means no metadata has been received yet, or we're
+ * using a lower protocol version
+ * (e.g., lack of api.version.request=true). */
+
+ while (1) {
+ int remains_ms;
+ int version;
+
+ version = rd_kafka_brokers_get_state_version(rk);
+
+ rd_kafka_rdlock(rk);
+
+ if (rk->rk_controllerid != -1) {
+ /* Cached controllerid available. */
+ rd_kafka_rdunlock(rk);
+ return rk->rk_controllerid;
+ } else if (rk->rk_ts_metadata > 0) {
+ /* Metadata received but no clusterid,
+ * this probably means the broker is too old
+ * or api.version.request=false. */
+ rd_kafka_rdunlock(rk);
+ return -1;
+ }
+
+ rd_kafka_rdunlock(rk);
+
+ /* Wait for up to timeout_ms for a metadata refresh,
+ * if permitted by application. */
+ remains_ms = rd_timeout_remains(abs_timeout);
+ if (rd_timeout_expired(remains_ms))
+ return -1;
+
+ rd_kafka_brokers_wait_state_change(rk, version, remains_ms);
+ }
+
+ return -1;
+}
+
+
void *rd_kafka_opaque (const rd_kafka_t *rk) {
return rk->rk_conf.opaque;
}
int rd_kafka_outq_len (rd_kafka_t *rk) {
- return rd_kafka_curr_msgs_cnt(rk) + rd_kafka_q_len(rk->rk_rep);
+ return rd_kafka_curr_msgs_cnt(rk) + rd_kafka_q_len(rk->rk_rep) +
+ (rk->rk_background.q ? rd_kafka_q_len(rk->rk_background.q) : 0);
}
@@ -2952,11 +3337,18 @@
return RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED;
rd_kafka_yield_thread = 0;
- while (((qlen = rd_kafka_q_len(rk->rk_rep)) > 0 ||
- (msg_cnt = rd_kafka_curr_msgs_cnt(rk)) > 0) &&
- !rd_kafka_yield_thread &&
- (tmout = rd_timeout_remains_limit(ts_end, 100))!=RD_POLL_NOWAIT)
+
+ /* First poll call is non-blocking for the case
+ * where timeout_ms==RD_POLL_NOWAIT to make sure poll is
+ * called at least once. */
+ tmout = RD_POLL_NOWAIT;
+ do {
rd_kafka_poll(rk, tmout);
+ } while (((qlen = rd_kafka_q_len(rk->rk_rep)) > 0 ||
+ (msg_cnt = rd_kafka_curr_msgs_cnt(rk)) > 0) &&
+ !rd_kafka_yield_thread &&
+ (tmout = rd_timeout_remains_limit(ts_end, 10)) !=
+ RD_POLL_NOWAIT);
return qlen + msg_cnt > 0 ? RD_KAFKA_RESP_ERR__TIMED_OUT :
RD_KAFKA_RESP_ERR_NO_ERROR;
diff -Nru librdkafka-0.11.3/src/rdkafka_cgrp.c librdkafka-0.11.6/src/rdkafka_cgrp.c
--- librdkafka-0.11.3/src/rdkafka_cgrp.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_cgrp.c 2018-10-10 06:54:38.000000000 +0000
@@ -475,6 +475,8 @@
if (ErrorCode == RD_KAFKA_RESP_ERR__DESTROY)
return;
+ /* No need for retries since the coord query is intervalled. */
+
if (ErrorCode == RD_KAFKA_RESP_ERR_GROUP_COORDINATOR_NOT_AVAILABLE)
rd_kafka_cgrp_coord_update(rkcg, -1);
else {
@@ -558,10 +560,13 @@
static void rd_kafka_cgrp_leave (rd_kafka_cgrp_t *rkcg, int ignore_response) {
rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "LEAVE",
- "Group \"%.*s\": leave",
- RD_KAFKAP_STR_PR(rkcg->rkcg_group_id));
+ "Group \"%.*s\": leave (in state %s)",
+ RD_KAFKAP_STR_PR(rkcg->rkcg_group_id),
+ rd_kafka_cgrp_state_names[rkcg->rkcg_state]);
- if (rkcg->rkcg_state == RD_KAFKA_CGRP_STATE_UP)
+ if (rkcg->rkcg_state == RD_KAFKA_CGRP_STATE_UP) {
+ rd_rkb_dbg(rkcg->rkcg_rkb, CONSUMER, "LEAVE",
+ "Leaving group");
rd_kafka_LeaveGroupRequest(rkcg->rkcg_rkb, rkcg->rkcg_group_id,
rkcg->rkcg_member_id,
ignore_response ?
@@ -569,7 +574,7 @@
RD_KAFKA_REPLYQ(rkcg->rkcg_ops, 0),
ignore_response ? NULL :
rd_kafka_handle_LeaveGroup, rkcg);
- else if (!ignore_response)
+ } else if (!ignore_response)
rd_kafka_handle_LeaveGroup(rkcg->rkcg_rk, rkcg->rkcg_rkb,
RD_KAFKA_RESP_ERR__WAIT_COORD,
NULL, NULL, rkcg);
@@ -604,7 +609,8 @@
rkcg->rkcg_assignment);
if (!(rkcg->rkcg_rk->rk_conf.enabled_events & RD_KAFKA_EVENT_REBALANCE)
- || !assignment) {
+ || !assignment
+ || rd_kafka_destroy_flags_no_consumer_close(rkcg->rkcg_rk)) {
no_delegation:
if (err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS)
rd_kafka_cgrp_assign(rkcg, assignment);
@@ -674,7 +680,7 @@
goto err;
}
- rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "ASSIGNOR",
+ rd_kafka_dbg(rkcg->rkcg_rk, CGRP|RD_KAFKA_DBG_CONSUMER, "ASSIGNOR",
"Group \"%s\": \"%s\" assignor run for %d member(s)",
rkcg->rkcg_group_id->str, protocol_name, member_cnt);
@@ -984,6 +990,9 @@
RD_KAFKA_OP_COORD_QUERY, ErrorCode);
}
+ /* No need for retries here since the join is intervalled,
+ * see rkcg_join_intvl */
+
if (ErrorCode) {
if (ErrorCode == RD_KAFKA_RESP_ERR__DESTROY)
return; /* Termination */
@@ -1158,7 +1167,7 @@
* refresh metadata if necessary. */
if (rd_kafka_cgrp_metadata_refresh(rkcg, &metadata_age,
"consumer join") == 1) {
- rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "JOIN",
+ rd_kafka_dbg(rkcg->rkcg_rk, CGRP|RD_KAFKA_DBG_CONSUMER, "JOIN",
"Group \"%.*s\": "
"postponing join until up-to-date "
"metadata is available",
@@ -1170,7 +1179,7 @@
rd_kafka_cgrp_metadata_update_check(rkcg, 0/*dont join*/);
if (rd_list_empty(rkcg->rkcg_subscribed_topics)) {
- rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "JOIN",
+ rd_kafka_dbg(rkcg->rkcg_rk, CGRP|RD_KAFKA_DBG_CONSUMER, "JOIN",
"Group \"%.*s\": "
"no matching topics based on %dms old metadata: "
"next metadata refresh in %dms",
@@ -1181,6 +1190,11 @@
return;
}
+ rd_rkb_dbg(rkcg->rkcg_rkb, CONSUMER, "JOIN",
+ "Joining group \"%.*s\" with %d subscribed topic(s)",
+ RD_KAFKAP_STR_PR(rkcg->rkcg_group_id),
+ rd_list_cnt(rkcg->rkcg_subscribed_topics));
+
rd_kafka_cgrp_set_join_state(rkcg, RD_KAFKA_CGRP_JOIN_STATE_WAIT_JOIN);
rd_kafka_JoinGroupRequest(rkcg->rkcg_rkb, rkcg->rkcg_group_id,
rkcg->rkcg_member_id,
@@ -1327,14 +1341,14 @@
/* Re-query for coordinator */
rd_kafka_cgrp_op(rkcg, NULL, RD_KAFKA_NO_REPLYQ,
RD_KAFKA_OP_COORD_QUERY, ErrorCode);
- /* Schedule a retry */
- if (ErrorCode != RD_KAFKA_RESP_ERR_NOT_COORDINATOR_FOR_GROUP) {
- rkcg->rkcg_flags |= RD_KAFKA_CGRP_F_HEARTBEAT_IN_TRANSIT;
- rd_kafka_buf_keep(request);
+ }
+
+ if (actions & RD_KAFKA_ERR_ACTION_RETRY) {
+ if (rd_kafka_buf_retry(rkb, request)) {
rkcg->rkcg_flags |= RD_KAFKA_CGRP_F_HEARTBEAT_IN_TRANSIT;
- rd_kafka_broker_buf_retry(request->rkbuf_rkb, request);
+ return;
}
- return;
+ /* FALLTHRU */
}
if (ErrorCode != 0 && ErrorCode != RD_KAFKA_RESP_ERR__DESTROY)
@@ -1529,14 +1543,19 @@
return;
}
- rd_kafka_topic_partition_list_log(rk, "OFFSETFETCH", offsets);
+ rd_kafka_topic_partition_list_log(rk, "OFFSETFETCH",
+ RD_KAFKA_DBG_TOPIC|RD_KAFKA_DBG_CGRP,
+ offsets);
/* If all partitions already had usable offsets then there
* was no request sent and thus no reply, the offsets list is
* good to go. */
- if (reply)
+ if (reply) {
err = rd_kafka_handle_OffsetFetch(rk, rkb, err,
reply, request, offsets,
1/* Update toppars */);
+ if (err == RD_KAFKA_RESP_ERR__IN_PROGRESS)
+ return; /* retrying */
+ }
if (err) {
rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "OFFSET",
"Offset fetch error: %s",
@@ -1582,9 +1601,6 @@
rkcg->rkcg_rk, rkb, RD_KAFKA_RESP_ERR__WAIT_COORD,
NULL, NULL, use_offsets);
else {
- rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "OFFSET",
- "Fetch %d offsets with v%d",
- use_offsets->cnt, rkcg->rkcg_version);
rd_kafka_OffsetFetchRequest(
rkb, 1, offsets,
RD_KAFKA_REPLYQ(rkcg->rkcg_ops, rkcg->rkcg_version),
@@ -1634,7 +1650,9 @@
rkcg->rkcg_version, line);
rd_kafka_topic_partition_list_log(rkcg->rkcg_rk,
- "FETCHSTART", assignment);
+ "FETCHSTART",
+ RD_KAFKA_DBG_TOPIC|RD_KAFKA_DBG_CGRP,
+ assignment);
if (assignment->cnt == 0)
return;
@@ -1784,12 +1802,10 @@
}
}
- if (rd_kafka_cgrp_try_terminate(rkcg))
- return errcnt; /* terminated */
-
if (rkcg->rkcg_join_state == RD_KAFKA_CGRP_JOIN_STATE_WAIT_UNASSIGN)
- rd_kafka_cgrp_check_unassign_done(rkcg,
- "OffsetCommit done");
+ rd_kafka_cgrp_check_unassign_done(rkcg, "OffsetCommit done");
+
+ rd_kafka_cgrp_try_terminate(rkcg);
return errcnt;
}
@@ -1855,6 +1871,15 @@
rd_kafka_assert(NULL, rkcg->rkcg_wait_commit_cnt > 0);
rkcg->rkcg_wait_commit_cnt--;
+ if (err == RD_KAFKA_RESP_ERR_NO_ERROR) {
+ rd_kafka_dbg(rk, CGRP, "COMMIT", "All committed, call fetch_start.");
+ if (rkcg->rkcg_wait_commit_cnt == 0 &&
+ rkcg->rkcg_assignment &&
+ RD_KAFKA_CGRP_CAN_FETCH_START(rkcg))
+ rd_kafka_cgrp_partitions_fetch_start(rkcg,
+ rkcg->rkcg_assignment, 0);
+ }
+
if (err == RD_KAFKA_RESP_ERR__DESTROY ||
(err == RD_KAFKA_RESP_ERR__NO_OFFSET &&
rko_orig->rko_u.offset_commit.silent_empty)) {
@@ -2010,6 +2035,15 @@
if (rkcg->rkcg_state != RD_KAFKA_CGRP_STATE_UP || !rkcg->rkcg_rkb ||
rkcg->rkcg_rkb->rkb_source == RD_KAFKA_INTERNAL) {
+ rd_kafka_dbg(rkcg->rkcg_rk, CONSUMER, "COMMIT",
+ "Deferring \"%s\" offset commit "
+ "for %d partition(s) in state %s: %s",
+ reason, valid_offsets,
+ rd_kafka_cgrp_state_names[rkcg->rkcg_state],
+ (!rkcg->rkcg_rkb ||
+ rkcg->rkcg_rkb->rkb_source == RD_KAFKA_INTERNAL)
+ ? "no coordinator available" : "not in state up");
+
if (rd_kafka_cgrp_defer_offset_commit(rkcg, rko, reason))
return;
@@ -2018,6 +2052,10 @@
} else {
int r;
+ rd_rkb_dbg(rkcg->rkcg_rkb, CONSUMER, "COMMIT",
+ "Committing offsets for %d partition(s): %s",
+ valid_offsets, reason);
+
/* Send OffsetCommit */
r = rd_kafka_OffsetCommitRequest(
rkcg->rkcg_rkb, rkcg, 1, offsets,
@@ -2177,14 +2215,15 @@
rd_kafka_cgrp_version_new_barrier(rkcg);
- rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "UNASSIGN",
+ rd_kafka_dbg(rkcg->rkcg_rk, CGRP|RD_KAFKA_DBG_CONSUMER, "UNASSIGN",
"Group \"%s\": unassigning %d partition(s) (v%"PRId32")",
rkcg->rkcg_group_id->str, old_assignment->cnt,
rkcg->rkcg_version);
if (rkcg->rkcg_rk->rk_conf.offset_store_method ==
RD_KAFKA_OFFSET_METHOD_BROKER &&
- rkcg->rkcg_rk->rk_conf.enable_auto_commit) {
+ rkcg->rkcg_rk->rk_conf.enable_auto_commit &&
+ !rd_kafka_destroy_flags_no_consumer_close(rkcg->rkcg_rk)) {
/* Commit all offsets for all assigned partitions to broker */
rd_kafka_cgrp_assigned_offsets_commit(rkcg, old_assignment,
"unassign");
@@ -2232,7 +2271,7 @@
rd_kafka_topic_partition_list_t *assignment) {
int i;
- rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "ASSIGN",
+ rd_kafka_dbg(rkcg->rkcg_rk, CGRP|RD_KAFKA_DBG_CONSUMER, "ASSIGN",
"Group \"%s\": new assignment of %d partition(s) "
"in join state %s",
rkcg->rkcg_group_id->str,
@@ -2368,6 +2407,13 @@
case RD_KAFKA_RESP_ERR_NOT_COORDINATOR_FOR_GROUP:
case RD_KAFKA_RESP_ERR_GROUP_COORDINATOR_NOT_AVAILABLE:
case RD_KAFKA_RESP_ERR__TRANSPORT:
+ rd_kafka_dbg(rkcg->rkcg_rk, CONSUMER, "HEARTBEAT",
+ "Heartbeat failed due to coordinator (%s) "
+ "no longer available: %s: "
+ "re-querying for coordinator",
+ rkcg->rkcg_rkb ?
+ rd_kafka_broker_name(rkcg->rkcg_rkb) : "none",
+ rd_kafka_err2str(err));
/* Remain in joined state and keep querying for coordinator */
rd_interval_expedite(&rkcg->rkcg_coord_query_intvl, 0);
break;
@@ -2376,6 +2422,13 @@
rd_kafka_cgrp_set_member_id(rkcg, "");
case RD_KAFKA_RESP_ERR_REBALANCE_IN_PROGRESS:
case RD_KAFKA_RESP_ERR_ILLEGAL_GENERATION:
+ rd_kafka_dbg(rkcg->rkcg_rk, CONSUMER, "HEARTBEAT",
+ "Heartbeat failed: %s: %s",
+ rd_kafka_err2str(err),
+ err == RD_KAFKA_RESP_ERR_UNKNOWN_MEMBER_ID ?
+ "resetting member-id" :
+ "group is rebalancing");
+
default:
/* Just revert to INIT state if join state is active. */
if (rkcg->rkcg_join_state <
@@ -2488,7 +2541,7 @@
rd_kafka_cgrp_subscribe (rd_kafka_cgrp_t *rkcg,
rd_kafka_topic_partition_list_t *rktparlist) {
- rd_kafka_dbg(rkcg->rkcg_rk, CGRP, "SUBSCRIBE",
+ rd_kafka_dbg(rkcg->rkcg_rk, CGRP|RD_KAFKA_DBG_CONSUMER, "SUBSCRIBE",
"Group \"%.*s\": subscribe to new %ssubscription "
"of %d topics (join state %s)",
RD_KAFKAP_STR_PR(rkcg->rkcg_group_id),
@@ -2569,12 +2622,20 @@
rkcg->rkcg_reply_rko = rko;
if (rkcg->rkcg_flags & RD_KAFKA_CGRP_F_SUBSCRIPTION)
- rd_kafka_cgrp_unsubscribe(rkcg, 1/*leave group*/);
+ rd_kafka_cgrp_unsubscribe(
+ rkcg,
+ /* Leave group if this is a controlled shutdown */
+ !rd_kafka_destroy_flags_no_consumer_close(
+ rkcg->rkcg_rk));
/* If there's an oustanding rebalance_cb which has not yet been
- * served by the application it will be served from consumer_close(). */
- if (!RD_KAFKA_CGRP_WAIT_REBALANCE_CB(rkcg) &&
- !(rkcg->rkcg_flags & RD_KAFKA_CGRP_F_WAIT_UNASSIGN))
+ * served by the application it will be served from consumer_close().
+ * If the instate is being terminated with NO_CONSUMER_CLOSE we
+ * trigger unassign directly to avoid stalling on rebalance callback
+ * queues that are no longer served by the application. */
+ if ((!RD_KAFKA_CGRP_WAIT_REBALANCE_CB(rkcg) &&
+ !(rkcg->rkcg_flags & RD_KAFKA_CGRP_F_WAIT_UNASSIGN)) ||
+ rd_kafka_destroy_flags_no_consumer_close(rkcg->rkcg_rk))
rd_kafka_cgrp_unassign(rkcg);
/* Try to terminate right away if all preconditions are met. */
@@ -3110,7 +3171,9 @@
*/
if (rd_kafka_cgrp_update_subscribed_topics(rkcg, tinfos) && do_join) {
/* List of subscribed topics changed, trigger rejoin. */
- rd_kafka_dbg(rkcg->rkcg_rk, CGRP|RD_KAFKA_DBG_METADATA, "REJOIN",
+ rd_kafka_dbg(rkcg->rkcg_rk,
+ CGRP|RD_KAFKA_DBG_METADATA|RD_KAFKA_DBG_CONSUMER,
+ "REJOIN",
"Group \"%.*s\": "
"subscription updated from metadata change: "
"rejoining group",
diff -Nru librdkafka-0.11.3/src/rdkafka_cgrp.h librdkafka-0.11.6/src/rdkafka_cgrp.h
--- librdkafka-0.11.3/src/rdkafka_cgrp.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_cgrp.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_CGRP_H_
+#define _RDKAFKA_CGRP_H_
#include "rdinterval.h"
@@ -273,3 +274,5 @@
const char *reason);
void rd_kafka_cgrp_metadata_update_check (rd_kafka_cgrp_t *rkcg, int do_join);
#define rd_kafka_cgrp_get(rk) ((rk)->rk_cgrp)
+
+#endif /* _RDKAFKA_CGRP_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_conf.c librdkafka-0.11.6/src/rdkafka_conf.c
--- librdkafka-0.11.3/src/rdkafka_conf.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_conf.c 2018-10-10 06:54:38.000000000 +0000
@@ -40,6 +40,10 @@
#include "rdkafka_plugin.h"
#endif
+#ifndef _MSC_VER
+#include
+#endif
+
struct rd_kafka_property {
rd_kafka_conf_scope_t scope;
const char *name;
@@ -120,6 +124,19 @@
return !strchr(val, ',') && !strchr(val, ' ');
}
+/**
+ * @brief Validate builtin partitioner string
+ */
+static RD_UNUSED int
+rd_kafka_conf_validate_partitioner (const struct rd_kafka_property *prop,
+ const char *val, int ival) {
+ return !strcmp(val, "random") ||
+ !strcmp(val, "consistent") ||
+ !strcmp(val, "consistent_random") ||
+ !strcmp(val, "murmur2") ||
+ !strcmp(val, "murmur2_random");
+}
+
/**
* librdkafka configuration property definitions.
@@ -178,12 +195,12 @@
0, 1000000000, 0xffff },
{ _RK_GLOBAL, "receive.message.max.bytes", _RK_C_INT,
_RK(recv_max_msg_size),
- "Maximum Kafka protocol response message size. "
- "This is a safety precaution to avoid memory exhaustion in case of "
- "protocol hickups. "
- "The value should be at least fetch.message.max.bytes * number of "
- "partitions consumed from + messaging overhead (e.g. 200000 bytes).",
- 1000, 1000000000, 100000000 },
+ "Maximum Kafka protocol response message size. "
+ "This serves as a safety precaution to avoid memory exhaustion in "
+ "case of protocol hickups. "
+ "This value is automatically adjusted upwards to be at least "
+ "`fetch.max.bytes` + 512 to allow for protocol overhead.",
+ 1000, INT_MAX, 100000000 },
{ _RK_GLOBAL, "max.in.flight.requests.per.connection", _RK_C_INT,
_RK(max_inflight),
"Maximum number of in-flight requests per broker connection. "
@@ -208,7 +225,7 @@
{ _RK_GLOBAL, "metadata.max.age.ms", _RK_C_INT,
_RK(metadata_max_age_ms),
"Metadata cache max age. "
- "Defaults to metadata.refresh.interval.ms * 3",
+ "Defaults to topic.metadata.refresh.interval.ms * 3",
1, 24*3600*1000, -1 },
{ _RK_GLOBAL, "topic.metadata.refresh.fast.interval.ms", _RK_C_INT,
_RK(metadata_refresh_fast_interval_ms),
@@ -232,25 +249,35 @@
"broker metadata information as if the topics did not exist." },
{ _RK_GLOBAL, "debug", _RK_C_S2F, _RK(debug),
"A comma-separated list of debug contexts to enable. "
- "Debugging the Producer: broker,topic,msg. Consumer: cgrp,topic,fetch",
+ "Detailed Producer debugging: broker,topic,msg. "
+ "Consumer: consumer,cgrp,topic,fetch",
.s2i = {
{ RD_KAFKA_DBG_GENERIC, "generic" },
{ RD_KAFKA_DBG_BROKER, "broker" },
{ RD_KAFKA_DBG_TOPIC, "topic" },
{ RD_KAFKA_DBG_METADATA, "metadata" },
+ { RD_KAFKA_DBG_FEATURE, "feature" },
{ RD_KAFKA_DBG_QUEUE, "queue" },
{ RD_KAFKA_DBG_MSG, "msg" },
{ RD_KAFKA_DBG_PROTOCOL, "protocol" },
{ RD_KAFKA_DBG_CGRP, "cgrp" },
{ RD_KAFKA_DBG_SECURITY, "security" },
{ RD_KAFKA_DBG_FETCH, "fetch" },
- { RD_KAFKA_DBG_FEATURE, "feature" },
{ RD_KAFKA_DBG_INTERCEPTOR, "interceptor" },
{ RD_KAFKA_DBG_PLUGIN, "plugin" },
- { RD_KAFKA_DBG_ALL, "all" },
+ { RD_KAFKA_DBG_CONSUMER, "consumer" },
+ { RD_KAFKA_DBG_ADMIN, "admin" },
+ { RD_KAFKA_DBG_ALL, "all" }
} },
{ _RK_GLOBAL, "socket.timeout.ms", _RK_C_INT, _RK(socket_timeout_ms),
- "Timeout for network requests.",
+ "Default timeout for network requests. "
+ "Producer: ProduceRequests will use the lesser value of "
+ "`socket.timeout.ms` and remaining `message.timeout.ms` for the "
+ "first message in the batch. "
+ "Consumer: FetchRequests will use "
+ "`fetch.wait.max.ms` + `socket.timeout.ms`. "
+ "Admin: Admin requests will use `socket.timeout.ms` or explicitly "
+ "set `rd_kafka_AdminOptions_set_operation_timeout()` value.",
10, 300*1000, 60*1000 },
{ _RK_GLOBAL, "socket.blocking.max.ms", _RK_C_INT,
_RK(socket_blocking_max_ms),
@@ -266,20 +293,27 @@
_RK(socket_rcvbuf_size),
"Broker socket receive buffer size. System default is used if 0.",
0, 100000000, 0 },
+#ifdef SO_KEEPALIVE
{ _RK_GLOBAL, "socket.keepalive.enable", _RK_C_BOOL,
_RK(socket_keepalive),
"Enable TCP keep-alives (SO_KEEPALIVE) on broker sockets",
0, 1, 0 },
+#endif
+#ifdef TCP_NODELAY
{ _RK_GLOBAL, "socket.nagle.disable", _RK_C_BOOL,
_RK(socket_nagle_disable),
- "Disable the Nagle algorithm (TCP_NODELAY).",
+ "Disable the Nagle algorithm (TCP_NODELAY) on broker sockets.",
0, 1, 0 },
+#endif
{ _RK_GLOBAL, "socket.max.fails", _RK_C_INT,
_RK(socket_max_fails),
"Disconnect from broker when this number of send failures "
"(e.g., timed out requests) is reached. Disable with 0. "
+ "WARNING: It is highly recommended to leave this setting at "
+ "its default value of 1 to avoid the client and broker to "
+ "become desynchronized in case of request timeouts. "
"NOTE: The connection is automatically re-established.",
- 0, 1000000, 3 },
+ 0, 1000000, 1 },
{ _RK_GLOBAL, "broker.address.ttl", _RK_C_INT,
_RK(broker_addr_ttl),
"How long to cache the broker address resolving "
@@ -344,6 +378,10 @@
"It might be useful to turn this off when interacting with "
"0.9 brokers with an aggressive `connection.max.idle.ms` value.",
0, 1, 1 },
+ { _RK_GLOBAL, "background_event_cb", _RK_C_PTR,
+ _RK(background_event_cb),
+ "Background queue event callback "
+ "(set with rd_kafka_conf_set_background_event_cb())" },
{ _RK_GLOBAL, "socket_cb", _RK_C_PTR,
_RK(socket_cb),
"Socket creation callback to provide race-free CLOEXEC",
@@ -411,7 +449,7 @@
{ _RK_GLOBAL, "broker.version.fallback", _RK_C_STR,
_RK(broker_version_fallback),
- "Older broker versions (<0.10.0) provides no way for a client to query "
+ "Older broker versions (before 0.10.0) provide no way for a client to query "
"for supported protocol features "
"(ApiVersionRequest, see `api.version.request`) making it impossible "
"for the client to know what features it may use. "
@@ -450,6 +488,22 @@
"protocol. See manual page for `ciphers(1)` and "
"`SSL_CTX_set_cipher_list(3)."
},
+#if OPENSSL_VERSION_NUMBER >= 0x1000200fL && !defined(LIBRESSL_VERSION_NUMBER)
+ { _RK_GLOBAL, "ssl.curves.list", _RK_C_STR,
+ _RK(ssl.curves_list),
+ "The supported-curves extension in the TLS ClientHello message specifies "
+ "the curves (standard/named, or 'explicit' GF(2^k) or GF(p)) the client "
+ "is willing to have the server use. See manual page for "
+ "`SSL_CTX_set1_curves_list(3)`. OpenSSL >= 1.0.2 required."
+ },
+ { _RK_GLOBAL, "ssl.sigalgs.list", _RK_C_STR,
+ _RK(ssl.sigalgs_list),
+ "The client uses the TLS ClientHello signature_algorithms extension "
+ "to indicate to the server which signature/hash algorithm pairs "
+ "may be used in digital signatures. See manual page for "
+ "`SSL_CTX_set1_sigalgs_list(3)`. OpenSSL >= 1.0.2 required."
+ },
+#endif
{ _RK_GLOBAL, "ssl.key.location", _RK_C_STR,
_RK(ssl.key_location),
"Path to client's private key (PEM) used for authentication."
@@ -471,16 +525,18 @@
_RK(ssl.crl_location),
"Path to CRL for verifying broker's certificate validity."
},
+ { _RK_GLOBAL, "ssl.keystore.location", _RK_C_STR,
+ _RK(ssl.keystore_location),
+ "Path to client's keystore (PKCS#12) used for authentication."
+ },
+ { _RK_GLOBAL, "ssl.keystore.password", _RK_C_STR,
+ _RK(ssl.keystore_password),
+ "Client's keystore (PKCS#12) password."
+ },
#endif /* WITH_SSL */
/* Point user in the right direction if they try to apply
* Java client SSL / JAAS properties. */
- { _RK_GLOBAL, "ssl.keystore.location", _RK_C_INVALID,
- _RK(dummy),
- "Java KeyStores are not supported, use `ssl.key.location` and "
- "a private key (PEM) file instead. "
- "See https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka for more information."
- },
{ _RK_GLOBAL, "ssl.truststore.location", _RK_C_INVALID,
_RK(dummy),
"Java TrustStores are not supported, use `ssl.ca.location` "
@@ -501,6 +557,8 @@
"**NOTE**: Despite the name only one mechanism must be configured.",
.sdef = "GSSAPI",
.validate = rd_kafka_conf_validate_single },
+ {_RK_GLOBAL,"sasl.mechanism", _RK_C_ALIAS,
+ .sdef = "sasl.mechanisms" },
{ _RK_GLOBAL, "sasl.kerberos.service.name", _RK_C_STR,
_RK(sasl.service_name),
"Kerberos principal name that Kafka runs as, "
@@ -541,7 +599,7 @@
/* Plugins */
{ _RK_GLOBAL, "plugin.library.paths", _RK_C_STR,
_RK(plugin_paths),
- "List of plugin libaries to load (; separated). "
+ "List of plugin libraries to load (; separated). "
"The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the "
"platform-specific extension (such as .dll or .so) will be appended automatically.",
.set = rd_kafka_plugins_conf_set },
@@ -593,7 +651,11 @@
/* Global consumer properties */
{ _RK_GLOBAL|_RK_CONSUMER, "enable.auto.commit", _RK_C_BOOL,
_RK(enable_auto_commit),
- "Automatically and periodically commit offsets in the background.",
+ "Automatically and periodically commit offsets in the background. "
+ "Note: setting this to false does not prevent the consumer from "
+ "fetching previously committed start offsets. To circumvent this "
+ "behaviour set specific start offsets per partition in the call "
+ "to assign().",
0, 1, 1 },
{ _RK_GLOBAL|_RK_CONSUMER, "auto.commit.interval.ms", _RK_C_INT,
_RK(auto_commit_interval_ms),
@@ -633,6 +695,19 @@
1, 1000000000, 1024*1024 },
{ _RK_GLOBAL|_RK_CONSUMER, "max.partition.fetch.bytes", _RK_C_ALIAS,
.sdef = "fetch.message.max.bytes" },
+ { _RK_GLOBAL|_RK_CONSUMER, "fetch.max.bytes", _RK_C_INT,
+ _RK(fetch_max_bytes),
+ "Maximum amount of data the broker shall return for a Fetch request. "
+ "Messages are fetched in batches by the consumer and if the first "
+ "message batch in the first non-empty partition of the Fetch request "
+ "is larger than this value, then the message batch will still be "
+ "returned to ensure the consumer can make progress. "
+ "The maximum message batch size accepted by the broker is defined "
+ "via `message.max.bytes` (broker config) or "
+ "`max.message.bytes` (broker topic config). "
+ "`fetch.max.bytes` is automatically adjusted upwards to be "
+ "at least `message.max.bytes` (consumer config).",
+ 0, INT_MAX-512, 50*1024*1024 /* 50MB */ },
{ _RK_GLOBAL|_RK_CONSUMER, "fetch.min.bytes", _RK_C_INT,
_RK(fetch_min_bytes),
"Minimum number of bytes the broker responds with. "
@@ -709,12 +784,25 @@
.sdef = "message.send.max.retries" },
{ _RK_GLOBAL|_RK_PRODUCER, "retry.backoff.ms", _RK_C_INT,
_RK(retry_backoff_ms),
- "The backoff time in milliseconds before retrying a message send.",
+ "The backoff time in milliseconds before retrying a protocol request.",
1, 300*1000, 100 },
+
+ { _RK_GLOBAL|_RK_PRODUCER, "queue.buffering.backpressure.threshold",
+ _RK_C_INT, _RK(queue_backpressure_thres),
+ "The threshold of outstanding not yet transmitted broker requests "
+ "needed to backpressure the producer's message accumulator. "
+ "If the number of not yet transmitted requests equals or exceeds "
+ "this number, produce request creation that would have otherwise "
+ "been triggered (for example, in accordance with linger.ms) will be "
+ "delayed. A lower number yields larger and more effective batches. "
+ "A higher value can improve latency when using compression on slow "
+ "machines.",
+ 1, 1000000, 1 },
+
{ _RK_GLOBAL|_RK_PRODUCER, "compression.codec", _RK_C_S2I,
_RK(compression_codec),
"compression codec to use for compressing message sets. "
- "This is the default value for all topics, may be overriden by "
+ "This is the default value for all topics, may be overridden by "
"the topic configuration property `compression.codec`. ",
.vdef = RD_KAFKA_COMPRESSION_NONE,
.s2i = {
@@ -728,6 +816,8 @@
{ RD_KAFKA_COMPRESSION_LZ4, "lz4" },
{ 0 }
} },
+ { _RK_GLOBAL|_RK_PRODUCER, "compression.type", _RK_C_ALIAS,
+ .sdef = "compression.codec" },
{ _RK_GLOBAL|_RK_PRODUCER, "batch.num.messages", _RK_C_INT,
_RK(batch_num_messages),
"Maximum number of messages batched in one MessageSet. "
@@ -757,7 +847,7 @@
"*0*=Broker does not send any response/ack to client, "
"*1*=Only the leader broker will need to ack the message, "
"*-1* or *all*=broker will block until message is committed by all "
- "in sync replicas (ISRs) or broker's `in.sync.replicas` "
+ "in sync replicas (ISRs) or broker's `min.insync.replicas` "
"setting before sending response. ",
-1, 1000, 1,
.s2i = {
@@ -778,24 +868,54 @@
"Local message timeout. "
"This value is only enforced locally and limits the time a "
"produced message waits for successful delivery. "
- "A time of 0 is infinite.",
+ "A time of 0 is infinite. "
+ "This is the maximum time librdkafka may use to deliver a message "
+ "(including retries). Delivery error occurs when either the retry "
+ "count or the message timeout are exceeded.",
0, 900*1000, 300*1000 },
+ { _RK_TOPIC|_RK_PRODUCER, "queuing.strategy", _RK_C_S2I,
+ _RKT(queuing_strategy),
+ "Producer queuing strategy. FIFO preserves produce ordering, "
+ "while LIFO prioritizes new messages. "
+ "WARNING: `lifo` is experimental and subject to change or removal.",
+ .vdef = 0,
+ .s2i = {
+ { RD_KAFKA_QUEUE_FIFO, "fifo" },
+ { RD_KAFKA_QUEUE_LIFO, "lifo" }
+ }
+ },
{ _RK_TOPIC|_RK_PRODUCER, "produce.offset.report", _RK_C_BOOL,
_RKT(produce_offset_report),
"Report offset of produced message back to application. "
"The application must be use the `dr_msg_cb` to retrieve the offset "
"from `rd_kafka_message_t.offset`.",
0, 1, 0 },
+ { _RK_TOPIC|_RK_PRODUCER, "partitioner", _RK_C_STR,
+ _RKT(partitioner_str),
+ "Partitioner: "
+ "`random` - random distribution, "
+ "`consistent` - CRC32 hash of key (Empty and NULL keys are mapped to single partition), "
+ "`consistent_random` - CRC32 hash of key (Empty and NULL keys are randomly partitioned), "
+ "`murmur2` - Java Producer compatible Murmur2 hash of key (NULL keys are mapped to single partition), "
+ "`murmur2_random` - Java Producer compatible Murmur2 hash of key (NULL keys are randomly partitioned. This is functionally equivalent to the default partitioner in the Java Producer.).",
+ .sdef = "consistent_random",
+ .validate = rd_kafka_conf_validate_partitioner },
{ _RK_TOPIC|_RK_PRODUCER, "partitioner_cb", _RK_C_PTR,
_RKT(partitioner),
- "Partitioner callback "
+ "Custom partitioner callback "
"(set with rd_kafka_topic_conf_set_partitioner_cb())" },
+ { _RK_TOPIC|_RK_PRODUCER, "msg_order_cmp", _RK_C_PTR,
+ _RKT(msg_order_cmp),
+ "Message queue ordering comparator "
+ "(set with rd_kafka_topic_conf_set_msg_order_cmp()). "
+ "Also see `queuing.strategy`." },
{ _RK_TOPIC, "opaque", _RK_C_PTR,
_RKT(opaque),
"Application opaque (set with rd_kafka_topic_conf_set_opaque())" },
{ _RK_TOPIC | _RK_PRODUCER, "compression.codec", _RK_C_S2I,
_RKT(compression_codec),
- "Compression codec to use for compressing message sets. ",
+ "Compression codec to use for compressing message sets. "
+ "inherit = inherit global compression.codec configuration.",
.vdef = RD_KAFKA_COMPRESSION_INHERIT,
.s2i = {
{ RD_KAFKA_COMPRESSION_NONE, "none" },
@@ -809,19 +929,31 @@
{ RD_KAFKA_COMPRESSION_INHERIT, "inherit" },
{ 0 }
} },
+ { _RK_TOPIC | _RK_PRODUCER, "compression.type", _RK_C_ALIAS,
+ .sdef = "compression.codec" },
+ { _RK_TOPIC|_RK_PRODUCER, "compression.level", _RK_C_INT,
+ _RKT(compression_level),
+ "Compression level parameter for algorithm selected by configuration "
+ "property `compression.codec`. Higher values will result in better "
+ "compression at the cost of more CPU usage. Usable range is "
+ "algorithm-dependent: [0-9] for gzip; [0-12] for lz4; only 0 for snappy; "
+ "-1 = codec-dependent default compression level.",
+ RD_KAFKA_COMPLEVEL_MIN,
+ RD_KAFKA_COMPLEVEL_MAX,
+ RD_KAFKA_COMPLEVEL_DEFAULT },
/* Topic consumer properties */
{ _RK_TOPIC|_RK_CONSUMER, "auto.commit.enable", _RK_C_BOOL,
_RKT(auto_commit),
+ "[**LEGACY PROPERTY:** This property is used by the simple legacy "
+ "consumer only. When using the high-level KafkaConsumer, the global "
+ "`enable.auto.commit` property must be used instead]. "
"If true, periodically commit offset of the last message handed "
"to the application. This committed offset will be used when the "
"process restarts to pick up where it left off. "
"If false, the application will have to call "
"`rd_kafka_offset_store()` to store an offset (optional). "
- "**NOTE:** This property should only be used with the simple "
- "legacy consumer, when using the high-level KafkaConsumer the global "
- "`enable.auto.commit` property must be used instead. "
"**NOTE:** There is currently no zookeeper integration, offsets "
"will be written to broker or local file according to "
"offset.store.method.",
@@ -830,9 +962,11 @@
.sdef = "auto.commit.enable" },
{ _RK_TOPIC|_RK_CONSUMER, "auto.commit.interval.ms", _RK_C_INT,
_RKT(auto_commit_interval_ms),
+ "[**LEGACY PROPERTY:** This setting is used by the simple legacy "
+ "consumer only. When using the high-level KafkaConsumer, the "
+ "global `auto.commit.interval.ms` property must be used instead]. "
"The frequency in milliseconds that the consumer offsets "
- "are committed (written) to offset storage. "
- "This setting is used by the low-level legacy consumer.",
+ "are committed (written) to offset storage.",
10, 86400*1000, 60*1000 },
{ _RK_TOPIC|_RK_CONSUMER, "auto.offset.reset", _RK_C_S2I,
_RKT(auto_offset_reset),
@@ -1250,8 +1384,9 @@
/* No match */
rd_snprintf(errstr, errstr_size,
- "Invalid value for "
- "configuration property \"%s\"", prop->name);
+ "Invalid value \"%.*s\" for "
+ "configuration property \"%s\"",
+ (int)(t-s), s, prop->name);
return RD_KAFKA_CONF_INVALID;
}
@@ -1621,11 +1756,25 @@
return new;
}
+rd_kafka_topic_conf_t *rd_kafka_default_topic_conf_dup (rd_kafka_t *rk) {
+ if (rk->rk_conf.topic_conf)
+ return rd_kafka_topic_conf_dup(rk->rk_conf.topic_conf);
+ else
+ return rd_kafka_topic_conf_new();
+}
void rd_kafka_conf_set_events (rd_kafka_conf_t *conf, int events) {
conf->enabled_events = events;
}
+void
+rd_kafka_conf_set_background_event_cb (rd_kafka_conf_t *conf,
+ void (*event_cb) (rd_kafka_t *rk,
+ rd_kafka_event_t *rkev,
+ void *opaque)) {
+ conf->background_event_cb = event_cb;
+}
+
void rd_kafka_conf_set_dr_cb (rd_kafka_conf_t *conf,
void (*dr_cb) (rd_kafka_t *rk,
@@ -1767,6 +1916,15 @@
topic_conf->partitioner = partitioner;
}
+void
+rd_kafka_topic_conf_set_msg_order_cmp (rd_kafka_topic_conf_t *topic_conf,
+ int (*msg_order_cmp) (
+ const rd_kafka_message_t *a,
+ const rd_kafka_message_t *b)) {
+ topic_conf->msg_order_cmp =
+ (int (*)(const void *, const void *))msg_order_cmp;
+}
+
void rd_kafka_topic_conf_set_opaque (rd_kafka_topic_conf_t *topic_conf,
void *opaque) {
topic_conf->opaque = opaque;
@@ -2151,3 +2309,206 @@
fprintf(fp, "\n");
fprintf(fp, "### C/P legend: C = Consumer, P = Producer, * = both\n");
}
+
+
+
+
+/**
+ * @name Configuration value methods
+ *
+ * @remark This generic interface will eventually replace the config property
+ * used above.
+ * @{
+ */
+
+
+/**
+ * @brief Set up an INT confval.
+ *
+ * @oaram name Property name, must be a const static string (will not be copied)
+ */
+void rd_kafka_confval_init_int (rd_kafka_confval_t *confval,
+ const char *name,
+ int vmin, int vmax, int vdef) {
+ confval->name = name;
+ confval->is_enabled = 1;
+ confval->valuetype = RD_KAFKA_CONFVAL_INT;
+ confval->u.INT.vmin = vmin;
+ confval->u.INT.vmax = vmax;
+ confval->u.INT.vdef = vdef;
+ confval->u.INT.v = vdef;
+}
+
+/**
+ * @brief Set up a PTR confval.
+ *
+ * @oaram name Property name, must be a const static string (will not be copied)
+ */
+void rd_kafka_confval_init_ptr (rd_kafka_confval_t *confval,
+ const char *name) {
+ confval->name = name;
+ confval->is_enabled = 1;
+ confval->valuetype = RD_KAFKA_CONFVAL_PTR;
+ confval->u.PTR = NULL;
+}
+
+/**
+ * @brief Set up but disable an intval, attempt to set this confval will fail.
+ *
+ * @oaram name Property name, must be a const static string (will not be copied)
+ */
+void rd_kafka_confval_disable (rd_kafka_confval_t *confval, const char *name) {
+ confval->name = name;
+ confval->is_enabled = 0;
+}
+
+/**
+ * @brief Set confval's value to \p valuep, verifying the passed
+ * \p valuetype matches (or can be cast to) \p confval's type.
+ *
+ * @param dispname is the display name for the configuration value and is
+ * included in error strings.
+ * @param valuep is a pointer to the value, or NULL to revert to default.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR if the new value was set, or
+ * RD_KAFKA_RESP_ERR__INVALID_ARG if the value was of incorrect type,
+ * out of range, or otherwise not a valid value.
+ */
+rd_kafka_resp_err_t
+rd_kafka_confval_set_type (rd_kafka_confval_t *confval,
+ rd_kafka_confval_type_t valuetype,
+ const void *valuep,
+ char *errstr, size_t errstr_size) {
+
+ if (!confval->is_enabled) {
+ rd_snprintf(errstr, errstr_size,
+ "\"%s\" is not supported for this operation",
+ confval->name);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ switch (confval->valuetype)
+ {
+ case RD_KAFKA_CONFVAL_INT:
+ {
+ int v;
+ const char *end;
+
+ if (!valuep) {
+ /* Revert to default */
+ confval->u.INT.v = confval->u.INT.vdef;
+ confval->is_set = 0;
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+
+ switch (valuetype)
+ {
+ case RD_KAFKA_CONFVAL_INT:
+ v = *(const int *)valuep;
+ break;
+ case RD_KAFKA_CONFVAL_STR:
+ v = (int)strtol((const char *)valuep, (char **)&end, 0);
+ if (end == (const char *)valuep) {
+ rd_snprintf(errstr, errstr_size,
+ "Invalid value type for \"%s\": "
+ "expecting integer",
+ confval->name);
+ return RD_KAFKA_RESP_ERR__INVALID_TYPE;
+ }
+ default:
+ rd_snprintf(errstr, errstr_size,
+ "Invalid value type for \"%s\": "
+ "expecting integer", confval->name);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+
+ if ((confval->u.INT.vmin || confval->u.INT.vmax) &&
+ (v < confval->u.INT.vmin || v > confval->u.INT.vmax)) {
+ rd_snprintf(errstr, errstr_size,
+ "Invalid value type for \"%s\": "
+ "expecting integer in range %d..%d",
+ confval->name,
+ confval->u.INT.vmin,
+ confval->u.INT.vmax);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ confval->u.INT.v = v;
+ confval->is_set = 1;
+ }
+ break;
+
+ case RD_KAFKA_CONFVAL_STR:
+ {
+ size_t vlen;
+ const char *v = (const char *)valuep;
+
+ if (!valuep) {
+ confval->is_set = 0;
+ if (confval->u.STR.vdef)
+ confval->u.STR.v = rd_strdup(confval->u.STR.
+ vdef);
+ else
+ confval->u.STR.v = NULL;
+ }
+
+ if (valuetype != RD_KAFKA_CONFVAL_STR) {
+ rd_snprintf(errstr, errstr_size,
+ "Invalid value type for \"%s\": "
+ "expecting string", confval->name);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ vlen = strlen(v);
+ if ((confval->u.STR.minlen || confval->u.STR.maxlen) &&
+ (vlen < confval->u.STR.minlen ||
+ vlen > confval->u.STR.maxlen)) {
+ rd_snprintf(errstr, errstr_size,
+ "Invalid value for \"%s\": "
+ "expecting string with length "
+ "%"PRIusz"..%"PRIusz,
+ confval->name,
+ confval->u.STR.minlen,
+ confval->u.STR.maxlen);
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ if (confval->u.STR.v)
+ rd_free(confval->u.STR.v);
+
+ confval->u.STR.v = rd_strdup(v);
+ }
+ break;
+
+ case RD_KAFKA_CONFVAL_PTR:
+ confval->u.PTR = (void *)valuep;
+ break;
+
+ default:
+ RD_NOTREACHED();
+ return RD_KAFKA_RESP_ERR__NOENT;
+ }
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+int rd_kafka_confval_get_int (const rd_kafka_confval_t *confval) {
+ rd_assert(confval->valuetype == RD_KAFKA_CONFVAL_INT);
+ return confval->u.INT.v;
+}
+
+
+const char *rd_kafka_confval_get_str (const rd_kafka_confval_t *confval) {
+ rd_assert(confval->valuetype == RD_KAFKA_CONFVAL_STR);
+ return confval->u.STR.v;
+}
+
+void *rd_kafka_confval_get_ptr (const rd_kafka_confval_t *confval) {
+ rd_assert(confval->valuetype == RD_KAFKA_CONFVAL_PTR);
+ return confval->u.PTR;
+}
+
+
+/**@}*/
diff -Nru librdkafka-0.11.3/src/rdkafka_conf.h librdkafka-0.11.6/src/rdkafka_conf.h
--- librdkafka-0.11.3/src/rdkafka_conf.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_conf.h 2018-10-10 06:54:38.000000000 +0000
@@ -1,4 +1,33 @@
-#pragma once
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2014-2018 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RDKAFKA_CONF_H_
+#define _RDKAFKA_CONF_H_
#include "rdlist.h"
@@ -20,6 +49,17 @@
RD_KAFKA_COMPRESSION_INHERIT /* Inherit setting from global conf */
} rd_kafka_compression_t;
+/**
+ * MessageSet compression levels
+ */
+typedef enum {
+ RD_KAFKA_COMPLEVEL_DEFAULT = -1,
+ RD_KAFKA_COMPLEVEL_MIN = -1,
+ RD_KAFKA_COMPLEVEL_GZIP_MAX = 9,
+ RD_KAFKA_COMPLEVEL_LZ4_MAX = 12,
+ RD_KAFKA_COMPLEVEL_SNAPPY_MAX = 0,
+ RD_KAFKA_COMPLEVEL_MAX = 12
+} rd_kafka_complevel_t;
typedef enum {
RD_KAFKA_PROTO_PLAINTEXT,
@@ -108,11 +148,17 @@
struct {
SSL_CTX *ctx;
char *cipher_suites;
+#if OPENSSL_VERSION_NUMBER >= 0x1000200fL && !defined(LIBRESSL_VERSION_NUMBER)
+ char *curves_list;
+ char *sigalgs_list;
+#endif
char *key_location;
char *key_password;
char *cert_location;
char *ca_location;
char *crl_location;
+ char *keystore_location;
+ char *keystore_password;
} ssl;
#endif
@@ -156,6 +202,7 @@
rd_list_t on_acknowledgement; /* .. (copied) */
rd_list_t on_consume; /* .. (copied) */
rd_list_t on_commit; /* .. (copied) */
+ rd_list_t on_request_sent; /* .. (copied) */
/* rd_strtup_t list */
rd_list_t config; /* Configuration name=val's
@@ -175,6 +222,7 @@
int64_t queued_max_msg_bytes;
int fetch_wait_max_ms;
int fetch_msg_max_bytes;
+ int fetch_max_bytes;
int fetch_min_bytes;
int fetch_error_backoff_ms;
char *group_id_str;
@@ -213,6 +261,7 @@
int queue_buffering_max_msgs;
int queue_buffering_max_kbytes;
int buffering_max_ms;
+ int queue_backpressure_thres;
int max_retries;
int retry_backoff_ms;
int batch_num_messages;
@@ -287,11 +336,22 @@
int (*open_cb) (const char *pathname, int flags, mode_t mode,
void *opaque);
+ /* Background queue event callback */
+ void (*background_event_cb) (rd_kafka_t *rk, rd_kafka_event_t *rkev,
+ void *opaque);
+
+
/* Opaque passed to callbacks. */
void *opaque;
/* For use with value-less properties. */
int dummy;
+
+
+ /* Admin client defaults */
+ struct {
+ int request_timeout_ms; /* AdminOptions.request_timeout */
+ } admin;
};
int rd_kafka_socket_cb_linux (int domain, int type, int protocol, void *opaque);
@@ -316,8 +376,13 @@
int32_t partition_cnt,
void *rkt_opaque,
void *msg_opaque);
+ char *partitioner_str;
+
+ int queuing_strategy; /* RD_KAFKA_QUEUE_FIFO|LIFO */
+ int (*msg_order_cmp) (const void *a, const void *b);
rd_kafka_compression_t compression_codec;
+ rd_kafka_complevel_t compression_level;
int produce_offset_report;
int consume_callback_max_msgs;
@@ -336,3 +401,8 @@
void rd_kafka_anyconf_destroy (int scope, void *conf);
+
+
+#include "rdkafka_confval.h"
+
+#endif /* _RDKAFKA_CONF_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_confval.h librdkafka-0.11.6/src/rdkafka_confval.h
--- librdkafka-0.11.3/src/rdkafka_confval.h 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_confval.h 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,96 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2014-2018 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RDKAFKA_CONFVAL_H_
+#define _RDKAFKA_CONFVAL_H_
+/**
+ * @name Next generation configuration values
+ * @{
+ *
+ */
+
+/**
+ * @brief Configuration value type
+ */
+typedef enum rd_kafka_confval_type_t {
+ RD_KAFKA_CONFVAL_INT,
+ RD_KAFKA_CONFVAL_STR,
+ RD_KAFKA_CONFVAL_PTR,
+} rd_kafka_confval_type_t;
+
+/**
+ * @brief Configuration value (used by AdminOption).
+ * Comes with a type, backed by a union, and a flag to indicate
+ * if the value has been set or not.
+ */
+typedef struct rd_kafka_confval_s {
+ const char *name; /**< Property name */
+ rd_kafka_confval_type_t valuetype; /**< Value type, maps to union.*/
+ int is_set; /**< Value has been set. */
+ int is_enabled; /**< Confval is enabled. */
+ union {
+ struct {
+ int v; /**< Current value */
+ int vmin; /**< Minimum value (inclusive) */
+ int vmax; /**< Maximum value (inclusive) */
+ int vdef; /**< Default value */
+ } INT;
+ struct {
+ char *v; /**< Current value */
+ int allowempty; /**< Allow empty string as value */
+ size_t minlen; /**< Minimum string length excl \0 */
+ size_t maxlen; /**< Maximum string length excl \0 */
+ const char *vdef; /**< Default value */
+ } STR;
+ void *PTR; /**< Pointer */
+ } u;
+} rd_kafka_confval_t;
+
+
+
+void rd_kafka_confval_init_int (rd_kafka_confval_t *confval,
+ const char *name,
+ int vmin, int vmax, int vdef);
+void rd_kafka_confval_init_ptr (rd_kafka_confval_t *confval,
+ const char *name);
+void rd_kafka_confval_disable (rd_kafka_confval_t *confval, const char *name);
+
+rd_kafka_resp_err_t
+rd_kafka_confval_set_type (rd_kafka_confval_t *confval,
+ rd_kafka_confval_type_t valuetype,
+ const void *valuep,
+ char *errstr, size_t errstr_size);
+
+int rd_kafka_confval_get_int (const rd_kafka_confval_t *confval);
+const char *rd_kafka_confval_get_str (const rd_kafka_confval_t *confval);
+void *rd_kafka_confval_get_ptr (const rd_kafka_confval_t *confval);
+
+/**@}*/
+
+
+#endif /* _RDKAFKA_CONFVAL_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_event.c librdkafka-0.11.6/src/rdkafka_event.c
--- librdkafka-0.11.3/src/rdkafka_event.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_event.c 2018-10-10 06:54:38.000000000 +0000
@@ -53,6 +53,16 @@
return "OffsetCommit";
case RD_KAFKA_EVENT_STATS:
return "Stats";
+ case RD_KAFKA_EVENT_CREATETOPICS_RESULT:
+ return "CreateTopicsResult";
+ case RD_KAFKA_EVENT_DELETETOPICS_RESULT:
+ return "DeleteTopicsResult";
+ case RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT:
+ return "CreatePartitionsResult";
+ case RD_KAFKA_EVENT_ALTERCONFIGS_RESULT:
+ return "AlterConfigsResult";
+ case RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT:
+ return "DescribeConfigsResult";
default:
return "?unknown?";
}
@@ -134,7 +144,7 @@
switch (rkev->rko_evtype)
{
case RD_KAFKA_EVENT_DR:
- return rd_atomic32_get(&rkev->rko_u.dr.msgq.rkmq_msg_cnt);
+ return (size_t)rkev->rko_u.dr.msgq.rkmq_msg_cnt;
case RD_KAFKA_EVENT_FETCH:
return 1;
default:
@@ -154,10 +164,16 @@
case RD_KAFKA_OP_CONSUMER_ERR:
if (rkev->rko_u.err.errstr)
return rkev->rko_u.err.errstr;
- /* FALLTHRU */
- default:
- return rd_kafka_err2str(rkev->rko_err);
- }
+ break;
+ case RD_KAFKA_OP_ADMIN_RESULT:
+ if (rkev->rko_u.admin_result.errstr)
+ return rkev->rko_u.admin_result.errstr;
+ break;
+ default:
+ break;
+ }
+
+ return rd_kafka_err2str(rkev->rko_err);
}
@@ -166,6 +182,8 @@
{
case RD_KAFKA_OP_OFFSET_COMMIT:
return rkev->rko_u.offset_commit.opaque;
+ case RD_KAFKA_OP_ADMIN_RESULT:
+ return rkev->rko_u.admin_result.opaque;
default:
return NULL;
}
@@ -230,3 +248,49 @@
return rktpar;
}
+
+
+
+const rd_kafka_CreateTopics_result_t *
+rd_kafka_event_CreateTopics_result (rd_kafka_event_t *rkev) {
+ if (!rkev || rkev->rko_evtype != RD_KAFKA_EVENT_CREATETOPICS_RESULT)
+ return NULL;
+ else
+ return (const rd_kafka_CreateTopics_result_t *)rkev;
+}
+
+
+const rd_kafka_DeleteTopics_result_t *
+rd_kafka_event_DeleteTopics_result (rd_kafka_event_t *rkev) {
+ if (!rkev || rkev->rko_evtype != RD_KAFKA_EVENT_DELETETOPICS_RESULT)
+ return NULL;
+ else
+ return (const rd_kafka_DeleteTopics_result_t *)rkev;
+}
+
+
+const rd_kafka_CreatePartitions_result_t *
+rd_kafka_event_CreatePartitions_result (rd_kafka_event_t *rkev) {
+ if (!rkev || rkev->rko_evtype != RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT)
+ return NULL;
+ else
+ return (const rd_kafka_CreatePartitions_result_t *)rkev;
+}
+
+
+const rd_kafka_AlterConfigs_result_t *
+rd_kafka_event_AlterConfigs_result (rd_kafka_event_t *rkev) {
+ if (!rkev || rkev->rko_evtype != RD_KAFKA_EVENT_ALTERCONFIGS_RESULT)
+ return NULL;
+ else
+ return (const rd_kafka_AlterConfigs_result_t *)rkev;
+}
+
+
+const rd_kafka_DescribeConfigs_result_t *
+rd_kafka_event_DescribeConfigs_result (rd_kafka_event_t *rkev) {
+ if (!rkev || rkev->rko_evtype != RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT)
+ return NULL;
+ else
+ return (const rd_kafka_DescribeConfigs_result_t *)rkev;
+}
diff -Nru librdkafka-0.11.3/src/rdkafka_event.h librdkafka-0.11.6/src/rdkafka_event.h
--- librdkafka-0.11.3/src/rdkafka_event.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_event.h 2018-10-10 06:54:38.000000000 +0000
@@ -54,7 +54,8 @@
*/
static RD_UNUSED RD_INLINE
int rd_kafka_event_setup (rd_kafka_t *rk, rd_kafka_op_t *rko) {
- rko->rko_evtype = rd_kafka_op2event(rko->rko_type);
+ if (!rko->rko_evtype)
+ rko->rko_evtype = rd_kafka_op2event(rko->rko_type);
switch (rko->rko_evtype)
{
case RD_KAFKA_EVENT_NONE:
@@ -72,6 +73,11 @@
case RD_KAFKA_EVENT_LOG:
case RD_KAFKA_EVENT_OFFSET_COMMIT:
case RD_KAFKA_EVENT_STATS:
+ case RD_KAFKA_EVENT_CREATETOPICS_RESULT:
+ case RD_KAFKA_EVENT_DELETETOPICS_RESULT:
+ case RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT:
+ case RD_KAFKA_EVENT_ALTERCONFIGS_RESULT:
+ case RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT:
return 1;
default:
diff -Nru librdkafka-0.11.3/src/rdkafka_feature.c librdkafka-0.11.6/src/rdkafka_feature.c
--- librdkafka-0.11.3/src/rdkafka_feature.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_feature.c 2018-10-10 06:54:38.000000000 +0000
@@ -172,6 +172,15 @@
{ -1 },
}
},
+ {
+ /* @brief >=0.10.2.0: Topic Admin API (KIP-4) */
+ .feature = RD_KAFKA_FEATURE_TOPIC_ADMIN_API,
+ .depends = {
+ { RD_KAFKAP_CreateTopics, 0, 0 },
+ { RD_KAFKAP_DeleteTopics, 0, 0 },
+ { -1 },
+ }
+ },
{ .feature = 0 }, /* sentinel */
};
diff -Nru librdkafka-0.11.3/src/rdkafka_feature.h librdkafka-0.11.6/src/rdkafka_feature.h
--- librdkafka-0.11.3/src/rdkafka_feature.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_feature.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_FEATURE_H_
+#define _RDKAFKA_FEATURE_H_
/**
@@ -65,6 +66,12 @@
* + EOS message format KIP-98 */
#define RD_KAFKA_FEATURE_MSGVER2 0x200
+/* >= 0.10.2.0: Topic Admin API */
+#define RD_KAFKA_FEATURE_TOPIC_ADMIN_API 0x400
+
+/* >= 1.0.0: CreatePartitions */
+#define RD_KAFKA_FEATURE_TOPIC_ADMIN_API 0x400
+
int rd_kafka_get_legacy_ApiVersions (const char *broker_version,
struct rd_kafka_ApiVersion **apisp,
@@ -77,3 +84,5 @@
size_t broker_api_cnt);
const char *rd_kafka_features2str (int features);
+
+#endif /* _RDKAFKA_FEATURE_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka.h librdkafka-0.11.6/src/rdkafka.h
--- librdkafka-0.11.3/src/rdkafka.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka.h 2018-10-10 06:54:38.000000000 +0000
@@ -1,7 +1,7 @@
/*
* librdkafka - Apache Kafka C library
*
- * Copyright (c) 2012-2013 Magnus Edenhill
+ * Copyright (c) 2012-2018 Magnus Edenhill
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -41,7 +41,8 @@
/* @cond NO_DOC */
-#pragma once
+#ifndef _RDKAFKA_H_
+#define _RDKAFKA_H_
#include
#include
@@ -108,9 +109,19 @@
TYPE2 __t2 RD_UNUSED = (ARG2); \
} \
RET; })
+
+#define _LRK_TYPECHECK3(RET,TYPE,ARG,TYPE2,ARG2,TYPE3,ARG3) \
+ ({ \
+ if (0) { \
+ TYPE __t RD_UNUSED = (ARG); \
+ TYPE2 __t2 RD_UNUSED = (ARG2); \
+ TYPE3 __t3 RD_UNUSED = (ARG3); \
+ } \
+ RET; })
#else
#define _LRK_TYPECHECK(RET,TYPE,ARG) (RET)
#define _LRK_TYPECHECK2(RET,TYPE,ARG,TYPE2,ARG2) (RET)
+#define _LRK_TYPECHECK3(RET,TYPE,ARG,TYPE2,ARG2,TYPE3,ARG3) (RET)
#endif
/* @endcond */
@@ -137,7 +148,7 @@
* @remark This value should only be used during compile time,
* for runtime checks of version use rd_kafka_version()
*/
-#define RD_KAFKA_VERSION 0x000b03ff
+#define RD_KAFKA_VERSION 0x000b06ff
/**
* @brief Returns the librdkafka version as integer.
@@ -212,7 +223,7 @@
* Use rd_kafka_get_debug_contexts() instead.
*/
#define RD_KAFKA_DEBUG_CONTEXTS \
- "all,generic,broker,topic,metadata,queue,msg,protocol,cgrp,security,fetch,feature"
+ "all,generic,broker,topic,metadata,feature,queue,msg,protocol,cgrp,security,fetch,interceptor,plugin,consumer,admin"
/* @cond NO_DOC */
@@ -222,6 +233,8 @@
typedef struct rd_kafka_conf_s rd_kafka_conf_t;
typedef struct rd_kafka_topic_conf_s rd_kafka_topic_conf_t;
typedef struct rd_kafka_queue_s rd_kafka_queue_t;
+typedef struct rd_kafka_op_s rd_kafka_event_t;
+typedef struct rd_kafka_topic_result_s rd_kafka_topic_result_t;
/* @endcond */
@@ -326,6 +339,14 @@
RD_KAFKA_RESP_ERR__VALUE_DESERIALIZATION = -159,
/** Partial response */
RD_KAFKA_RESP_ERR__PARTIAL = -158,
+ /** Modification attempted on read-only object */
+ RD_KAFKA_RESP_ERR__READ_ONLY = -157,
+ /** No such entry / item not found */
+ RD_KAFKA_RESP_ERR__NOENT = -156,
+ /** Read underflow */
+ RD_KAFKA_RESP_ERR__UNDERFLOW = -155,
+ /** Invalid type */
+ RD_KAFKA_RESP_ERR__INVALID_TYPE = -154,
/** End internal error codes */
RD_KAFKA_RESP_ERR__END = -100,
@@ -785,6 +806,9 @@
RD_KAFKA_VTYPE_OPAQUE, /**< (void *) Application opaque */
RD_KAFKA_VTYPE_MSGFLAGS, /**< (int) RD_KAFKA_MSG_F_.. flags */
RD_KAFKA_VTYPE_TIMESTAMP, /**< (int64_t) Milliseconds since epoch UTC */
+ RD_KAFKA_VTYPE_HEADER, /**< (const char *, const void *, ssize_t)
+ * Message Header */
+ RD_KAFKA_VTYPE_HEADERS, /**< (rd_kafka_headers_t *) Headers list */
} rd_kafka_vtype_t;
@@ -829,7 +853,8 @@
_LRK_TYPECHECK2(RD_KAFKA_VTYPE_KEY, const void *, KEY, size_t, LEN), \
(void *)KEY, (size_t)LEN
/*!
- * Opaque pointer (void *)
+ * Message opaque pointer (void *)
+ * Same as \c produce(.., msg_opaque), and \c rkmessage->_private .
*/
#define RD_KAFKA_V_OPAQUE(opaque) \
_LRK_TYPECHECK(RD_KAFKA_VTYPE_OPAQUE, void *, opaque), \
@@ -847,10 +872,164 @@
#define RD_KAFKA_V_TIMESTAMP(timestamp) \
_LRK_TYPECHECK(RD_KAFKA_VTYPE_TIMESTAMP, int64_t, timestamp), \
(int64_t)timestamp
+/*!
+ * Add Message Header (const char *NAME, const void *VALUE, ssize_t LEN).
+ * @sa rd_kafka_header_add()
+ * @remark RD_KAFKA_V_HEADER() and RD_KAFKA_V_HEADERS() MUST NOT be mixed
+ * in the same call to producev().
+ */
+#define RD_KAFKA_V_HEADER(NAME,VALUE,LEN) \
+ _LRK_TYPECHECK3(RD_KAFKA_VTYPE_HEADER, const char *, NAME, \
+ const void *, VALUE, ssize_t, LEN), \
+ (const char *)NAME, (const void *)VALUE, (ssize_t)LEN
+
+/*!
+ * Message Headers list (rd_kafka_headers_t *).
+ * The message object will assume ownership of the headers (unless producev()
+ * fails).
+ * Any existing headers will be replaced.
+ * @sa rd_kafka_message_set_headers()
+ * @remark RD_KAFKA_V_HEADER() and RD_KAFKA_V_HEADERS() MUST NOT be mixed
+ * in the same call to producev().
+ */
+#define RD_KAFKA_V_HEADERS(HDRS) \
+ _LRK_TYPECHECK(RD_KAFKA_VTYPE_HEADERS, rd_kafka_headers_t *, HDRS), \
+ (rd_kafka_headers_t *)HDRS
+
+
+/**@}*/
+
+
+/**
+ * @name Message headers
+ * @{
+ *
+ * @brief Message headers consist of a list of (string key, binary value) pairs.
+ * Duplicate keys are supported and the order in which keys were
+ * added are retained.
+ *
+ * Header values are considered binary and may have three types of
+ * value:
+ * - proper value with size > 0 and a valid pointer
+ * - empty value with size = 0 and any non-NULL pointer
+ * - null value with size = 0 and a NULL pointer
+ *
+ * Headers require Apache Kafka broker version v0.11.0.0 or later.
+ *
+ * Header operations are O(n).
+ */
+
+typedef struct rd_kafka_headers_s rd_kafka_headers_t;
+
+/**
+ * @brief Create a new headers list.
+ *
+ * @param initial_count Preallocate space for this number of headers.
+ * Any number of headers may be added, updated and
+ * removed regardless of the initial count.
+ */
+RD_EXPORT rd_kafka_headers_t *rd_kafka_headers_new (size_t initial_count);
+
+/**
+ * @brief Destroy the headers list. The object and any returned value pointers
+ * are not usable after this call.
+ */
+RD_EXPORT void rd_kafka_headers_destroy (rd_kafka_headers_t *hdrs);
+
+/**
+ * @brief Make a copy of headers list \p src.
+ */
+RD_EXPORT rd_kafka_headers_t *
+rd_kafka_headers_copy (const rd_kafka_headers_t *src);
+
+/**
+ * @brief Add header with name \p name and value \p val (copied) of size
+ * \p size (not including null-terminator).
+ *
+ * @param name Header name.
+ * @param name_size Header name size (not including the null-terminator).
+ * If -1 the \p name length is automatically acquired using
+ * strlen().
+ * @param value Pointer to header value, or NULL (set size to 0 or -1).
+ * @param value_size Size of header value. If -1 the \p value is assumed to be a
+ * null-terminated string and the length is automatically
+ * acquired using strlen().
+ *
+ * @returns RD_KAFKA_RESP_ERR__READ_ONLY if the headers are read-only,
+ * else RD_KAFKA_RESP_ERR_NO_ERROR.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_header_add (rd_kafka_headers_t *hdrs,
+ const char *name, ssize_t name_size,
+ const void *value, ssize_t value_size);
+
+/**
+ * @brief Remove all headers for the given key (if any).
+ *
+ * @returns RD_KAFKA_RESP_ERR__READ_ONLY if the headers are read-only,
+ * RD_KAFKA_RESP_ERR__NOENT if no matching headers were found,
+ * else RD_KAFKA_RESP_ERR_NO_ERROR if headers were removed.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_header_remove (rd_kafka_headers_t *hdrs, const char *name);
+
+
+/**
+ * @brief Find last header in list \p hdrs matching \p name.
+ *
+ * @param name Header to find (last match).
+ * @param valuep (out) Set to a (null-terminated) const pointer to the value
+ * (may be NULL).
+ * @param sizep (out) Set to the value's size (not including null-terminator).
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR if an entry was found, else
+ * RD_KAFKA_RESP_ERR__NOENT.
+ *
+ * @remark The returned pointer in \p valuep includes a trailing null-terminator
+ * that is not accounted for in \p sizep.
+ * @remark The returned pointer is only valid as long as the headers list and
+ * the header item is valid.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_header_get_last (const rd_kafka_headers_t *hdrs,
+ const char *name, const void **valuep, size_t *sizep);
+
+/**
+ * @brief Iterator for headers matching \p name.
+ *
+ * Same semantics as rd_kafka_header_get_last()
+ *
+ * @param hdrs Headers to iterate.
+ * @param idx Iterator index, start at 0 and increment by one for each call
+ * as long as RD_KAFKA_RESP_ERR_NO_ERROR is returned.
+ * @param name Header name to match.
+ * @param valuep (out) Set to a (null-terminated) const pointer to the value
+ * (may be NULL).
+ * @param sizep (out) Set to the value's size (not including null-terminator).
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_header_get (const rd_kafka_headers_t *hdrs, size_t idx,
+ const char *name, const void **valuep, size_t *sizep);
+
+
+/**
+ * @brief Iterator for all headers.
+ *
+ * Same semantics as rd_kafka_header_get()
+ *
+ * @sa rd_kafka_header_get()
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_header_get_all (const rd_kafka_headers_t *hdrs, size_t idx,
+ const char **namep,
+ const void **valuep, size_t *sizep);
+
+
/**@}*/
+
/**
* @name Kafka messages
* @{
@@ -963,6 +1142,66 @@
int64_t rd_kafka_message_latency (const rd_kafka_message_t *rkmessage);
+/**
+ * @brief Get the message header list.
+ *
+ * The returned pointer in \p *hdrsp is associated with the \p rkmessage and
+ * must not be used after destruction of the message object or the header
+ * list is replaced with rd_kafka_message_set_headers().
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR if headers were returned,
+ * RD_KAFKA_RESP_ERR__NOENT if the message has no headers,
+ * or another error code if the headers could not be parsed.
+ *
+ * @remark Headers require broker version 0.11.0.0 or later.
+ *
+ * @remark As an optimization the raw protocol headers are parsed on
+ * the first call to this function.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_message_headers (const rd_kafka_message_t *rkmessage,
+ rd_kafka_headers_t **hdrsp);
+
+/**
+ * @brief Get the message header list and detach the list from the message
+ * making the application the owner of the headers.
+ * The application must eventually destroy the headers using
+ * rd_kafka_headers_destroy().
+ * The message's headers will be set to NULL.
+ *
+ * Otherwise same semantics as rd_kafka_message_headers()
+ *
+ * @sa rd_kafka_message_headers
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_message_detach_headers (rd_kafka_message_t *rkmessage,
+ rd_kafka_headers_t **hdrsp);
+
+
+/**
+ * @brief Replace the message's current headers with a new list.
+ *
+ * @param hdrs New header list. The message object assumes ownership of
+ * the list, the list will be destroyed automatically with
+ * the message object.
+ * The new headers list may be updated until the message object
+ * is passed or returned to librdkafka.
+ *
+ * @remark The existing headers object, if any, will be destroyed.
+ */
+RD_EXPORT
+void rd_kafka_message_set_headers (rd_kafka_message_t *rkmessage,
+ rd_kafka_headers_t *hdrs);
+
+
+/**
+ * @brief Returns the number of header key/value pairs
+ *
+ * @param hdrs Headers to count
+ */
+RD_EXPORT size_t rd_kafka_header_cnt (const rd_kafka_headers_t *hdrs);
+
+
/**@}*/
@@ -1080,6 +1319,48 @@
/**
+ * @brief Generic event callback to be used with the event API to trigger
+ * callbacks for \c rd_kafka_event_t objects from a background
+ * thread serving the background queue.
+ *
+ * How to use:
+ * 1. First set the event callback on the configuration object with this
+ * function, followed by creating an rd_kafka_t instance
+ * with rd_kafka_new().
+ * 2. Get the instance's background queue with rd_kafka_queue_get_background()
+ * and pass it as the reply/response queue to an API that takes an
+ * event queue, such as rd_kafka_CreateTopics().
+ * 3. As the response event is ready and enqueued on the background queue the
+ * event callback will be triggered from the background thread.
+ * 4. Prior to destroying the client instance, loose your reference to the
+ * background queue by calling rd_kafka_queue_destroy().
+ *
+ * The application must destroy the \c rkev passed to \p event cb using
+ * rd_kafka_event_destroy().
+ *
+ * The \p event_cb \c opaque argument is the opaque set with
+ * rd_kafka_conf_set_opaque().
+ *
+ * @remark This callback is a specialized alternative to the poll-based
+ * event API described in the Event interface section.
+ *
+ * @remark The \p event_cb will be called spontaneously from a background
+ * thread completely managed by librdkafka.
+ * Take care to perform proper locking of application objects.
+ *
+ * @warning The application MUST NOT call rd_kafka_destroy() from the
+ * event callback.
+ *
+ * @sa rd_kafka_queue_get_background
+ */
+RD_EXPORT void
+rd_kafka_conf_set_background_event_cb (rd_kafka_conf_t *conf,
+ void (*event_cb) (rd_kafka_t *rk,
+ rd_kafka_event_t *rkev,
+ void *opaque));
+
+
+/**
@deprecated See rd_kafka_conf_set_dr_msg_cb()
*/
RD_EXPORT
@@ -1097,8 +1378,10 @@
* the result of the produce request.
*
* The callback is called when a message is succesfully produced or
- * if librdkafka encountered a permanent failure, or the retry counter for
- * temporary errors has been exhausted.
+ * if librdkafka encountered a permanent failure.
+ * Delivery errors occur when the retry count is exceeded, when the
+ * message.timeout.ms timeout is exceeded or there is a permanent error
+ * like RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART.
*
* An application must call rd_kafka_poll() at regular intervals to
* serve queued delivery report callbacks.
@@ -1142,13 +1425,25 @@
*
* Without a rebalance callback this is done automatically by librdkafka
* but registering a rebalance callback gives the application flexibility
- * in performing other operations along with the assinging/revocation,
+ * in performing other operations along with the assigning/revocation,
* such as fetching offsets from an alternate location (on assign)
* or manually committing offsets (on revoke).
*
* @remark The \p partitions list is destroyed by librdkafka on return
* return from the rebalance_cb and must not be freed or
* saved by the application.
+ *
+ * @remark Be careful when modifying the \p partitions list.
+ * Changing this list should only be done to change the initial
+ * offsets for each partition.
+ * But a function like `rd_kafka_position()` might have unexpected
+ * effects for instance when a consumer gets assigned a partition
+ * it used to consume at an earlier rebalance. In this case, the
+ * list of partitions will be updated with the old offset for that
+ * partition. In this case, it is generally better to pass a copy
+ * of the list (see `rd_kafka_topic_partition_list_copy()`).
+ * The result of `rd_kafka_position()` is typically outdated in
+ * RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS.
*
* The following example shows the application's responsibilities:
* @code
@@ -1219,7 +1514,8 @@
* The error callback is used by librdkafka to signal critical errors
* back to the application.
*
- * If no \p error_cb is registered then the errors will be logged instead.
+ * If no \p error_cb is registered, or RD_KAFKA_EVENT_ERROR has not been set
+ * with rd_kafka_conf_set_events, then the errors will be logged instead.
*/
RD_EXPORT
void rd_kafka_conf_set_error_cb(rd_kafka_conf_t *conf,
@@ -1284,10 +1580,15 @@
* - \p json_len - Length of \p json string.
* - \p opaque - Application-provided opaque.
*
+ * For more information on the format of \p json, see
+ * https://github.com/edenhill/librdkafka/wiki/Statistics
+ *
* If the application wishes to hold on to the \p json pointer and free
* it at a later time it must return 1 from the \p stats_cb.
* If the application returns 0 from the \p stats_cb then librdkafka
* will immediately free the \p json pointer.
+ *
+ * See STATISTICS.md for a full definition of the JSON object.
*/
RD_EXPORT
void rd_kafka_conf_set_stats_cb(rd_kafka_conf_t *conf,
@@ -1503,6 +1804,13 @@
rd_kafka_topic_conf_t *rd_kafka_topic_conf_dup(const rd_kafka_topic_conf_t
*conf);
+/**
+ * @brief Creates a copy/duplicate of \p rk 's default topic configuration
+ * object.
+ */
+RD_EXPORT
+rd_kafka_topic_conf_t *rd_kafka_default_topic_conf_dup (rd_kafka_t *rk);
+
/**
* @brief Destroys a topic conf object.
@@ -1558,6 +1866,39 @@
void *rkt_opaque,
void *msg_opaque));
+
+/**
+ * @brief \b Producer: Set message queueing order comparator callback.
+ *
+ * The callback may be called in any thread at any time,
+ * it may be called multiple times for the same message.
+ *
+ * Ordering comparator function constraints:
+ * - MUST be stable sort (same input gives same output).
+ * - MUST NOT call any rd_kafka_*() functions.
+ * - MUST NOT block or execute for prolonged periods of time.
+ *
+ * The comparator shall compare the two messages and return:
+ * - < 0 if message \p a should be inserted before message \p b.
+ * - >=0 if message \p a should be inserted after message \p b.
+ *
+ * @remark Insert sorting will be used to enqueue the message in the
+ * correct queue position, this comes at a cost of O(n).
+ *
+ * @remark If `queuing.strategy=fifo` new messages are enqueued to the
+ * tail of the queue regardless of msg_order_cmp, but retried messages
+ * are still affected by msg_order_cmp.
+ *
+ * @warning THIS IS AN EXPERIMENTAL API, SUBJECT TO CHANGE OR REMOVAL,
+ * DO NOT USE IN PRODUCTION.
+ */
+RD_EXPORT void
+rd_kafka_topic_conf_set_msg_order_cmp (rd_kafka_topic_conf_t *topic_conf,
+ int (*msg_order_cmp) (
+ const rd_kafka_message_t *a,
+ const rd_kafka_message_t *b));
+
+
/**
* @brief Check if partition is available (has a leader broker).
*
@@ -1621,6 +1962,38 @@
void *opaque, void *msg_opaque);
+/**
+ * @brief Murmur2 partitioner (Java compatible).
+ *
+ * Uses consistent hashing to map identical keys onto identical partitions
+ * using Java-compatible Murmur2 hashing.
+ *
+ * @returns a partition between 0 and \p partition_cnt - 1.
+ */
+RD_EXPORT
+int32_t rd_kafka_msg_partitioner_murmur2 (const rd_kafka_topic_t *rkt,
+ const void *key, size_t keylen,
+ int32_t partition_cnt,
+ void *rkt_opaque,
+ void *msg_opaque);
+
+/**
+ * @brief Consistent-Random Murmur2 partitioner (Java compatible).
+ *
+ * Uses consistent hashing to map identical keys onto identical partitions
+ * using Java-compatible Murmur2 hashing.
+ * Messages without keys will be assigned via the random partitioner.
+ *
+ * @returns a partition between 0 and \p partition_cnt - 1.
+ */
+RD_EXPORT
+int32_t rd_kafka_msg_partitioner_murmur2_random (const rd_kafka_topic_t *rkt,
+ const void *key, size_t keylen,
+ int32_t partition_cnt,
+ void *rkt_opaque,
+ void *msg_opaque);
+
+
/**@}*/
@@ -1670,11 +2043,44 @@
* @brief Destroy Kafka handle.
*
* @remark This is a blocking operation.
+ * @remark rd_kafka_consumer_close() will be called from this function
+ * if the instance type is RD_KAFKA_CONSUMER, a \c group.id was
+ * configured, and the rd_kafka_consumer_close() was not
+ * explicitly called by the application. This in turn may
+ * trigger consumer callbacks, such as rebalance_cb.
+ * Use rd_kafka_destroy_flags() with
+ * RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE to avoid this behaviour.
+ *
+ * @sa rd_kafka_destroy_flags()
*/
RD_EXPORT
void rd_kafka_destroy(rd_kafka_t *rk);
+/**
+ * @brief Destroy Kafka handle according to specified destroy flags
+ *
+ */
+RD_EXPORT
+void rd_kafka_destroy_flags (rd_kafka_t *rk, int flags);
+
+/**
+ * @brief Flags for rd_kafka_destroy_flags()
+ */
+
+/*!
+ * Don't call consumer_close() to leave group and commit final offsets.
+ *
+ * This also disables consumer callbacks to be called from rd_kafka_destroy*(),
+ * such as rebalance_cb.
+ *
+ * The consumer group handler is still closed internally, but from an
+ * application perspective none of the functionality from consumer_close()
+ * is performed.
+ */
+#define RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE 0x8
+
+
/**
* @brief Returns Kafka handle name.
@@ -1727,6 +2133,24 @@
/**
+ * @brief Returns the current ControllerId as reported in broker metadata.
+ *
+ * @param timeout_ms If there is no cached value from metadata retrieval
+ * then this specifies the maximum amount of time
+ * (in milliseconds) the call will block waiting
+ * for metadata to be retrieved.
+ * Use 0 for non-blocking calls.
+
+ * @remark Requires broker version >=0.10.0 and api.version.request=true.
+ *
+ * @returns the controller broker id (>= 0), or -1 if no ControllerId could be
+ * retrieved in the allotted timespan.
+ */
+RD_EXPORT
+int32_t rd_kafka_controllerid (rd_kafka_t *rk, int timeout_ms);
+
+
+/**
* @brief Creates a new topic handle for topic named \p topic.
*
* \p conf is an optional configuration for the topic created with
@@ -1800,6 +2224,10 @@
*
* @remark An application should make sure to call poll() at regular
* intervals to serve any queued callbacks waiting to be called.
+ * @remark If your producer doesn't have any callback set (in particular
+ * via rd_kafka_conf_set_dr_msg_cb or rd_kafka_conf_set_error_cb)
+ * you might chose not to call poll(), though this is not
+ * recommended.
*
* Events:
* - delivery report callbacks (if dr_cb/dr_msg_cb is configured) [producer]
@@ -2011,6 +2439,32 @@
int32_t partition);
/**
+ * @returns a reference to the background thread queue, or NULL if the
+ * background queue is not enabled.
+ *
+ * To enable the background thread queue set a generic event handler callback
+ * with rd_kafka_conf_set_background_event_cb() on the client instance
+ * configuration object (rd_kafka_conf_t).
+ *
+ * The background queue is polled and served by librdkafka and MUST NOT be
+ * polled, forwarded, or otherwise managed by the application, it may only
+ * be used as the destination queue passed to queue-enabled APIs, such as
+ * the Admin API.
+ *
+ * The background thread queue provides the application with an automatically
+ * polled queue that triggers the event callback in a background thread,
+ * this background thread is completely managed by librdkafka.
+ *
+ * Use rd_kafka_queue_destroy() to loose the reference.
+ *
+ * @warning The background queue MUST NOT be read from (polled, consumed, etc),
+ * or forwarded from.
+ */
+RD_EXPORT
+rd_kafka_queue_t *rd_kafka_queue_get_background (rd_kafka_t *rk);
+
+
+/**
* @brief Forward/re-route queue \p src to \p dst.
* If \p dst is \c NULL the forwarding is removed.
*
@@ -2063,6 +2517,7 @@
*
* librdkafka will maintain a copy of the \p payload.
*
+ * @remark IO and callback event triggering are mutually exclusive.
* @remark When using forwarded queues the IO event must only be enabled
* on the final forwarded-to (destination) queue.
*/
@@ -2070,6 +2525,26 @@
void rd_kafka_queue_io_event_enable (rd_kafka_queue_t *rkqu, int fd,
const void *payload, size_t size);
+/**
+ * @brief Enable callback event triggering for queue.
+ *
+ * The callback will be called from an internal librdkafka thread
+ * when a new element is enqueued on a previously empty queue.
+ *
+ * To remove event triggering call with \p event_cb = NULL.
+ *
+ * @remark IO and callback event triggering are mutually exclusive.
+ * @remark Since the callback may be triggered from internal librdkafka
+ * threads, the application must not perform any pro-longed work in
+ * the callback, or call any librdkafka APIs (for the same rd_kafka_t
+ * handle).
+ */
+RD_EXPORT
+void rd_kafka_queue_cb_event_enable (rd_kafka_queue_t *rkqu,
+ void (*event_cb) (rd_kafka_t *rk,
+ void *opaque),
+ void *opaque);
+
/**@}*/
/**
@@ -2400,11 +2875,27 @@
/**
* @brief Subscribe to topic set using balanced consumer groups.
*
- * Wildcard (regex) topics are supported by the librdkafka assignor:
+ * Wildcard (regex) topics are supported:
* any topic name in the \p topics list that is prefixed with \c \"^\" will
* be regex-matched to the full list of topics in the cluster and matching
* topics will be added to the subscription list.
*
+ * The full topic list is retrieved every \c topic.metadata.refresh.interval.ms
+ * to pick up new or delete topics that match the subscription.
+ * If there is any change to the matched topics the consumer will
+ * immediately rejoin the group with the updated set of subscribed topics.
+ *
+ * Regex and full topic names can be mixed in \p topics.
+ *
+ * @remark Only the \c .topic field is used in the supplied \p topics list,
+ * all other fields are ignored.
+ *
+ * @remark subscribe() is an asynchronous method which returns immediately:
+ * background threads will (re)join the group, wait for group rebalance,
+ * issue any registered rebalance_cb, assign() the assigned partitions,
+ * and then start fetching messages. This cycle may take up to
+ * \c session.timeout.ms * 2 or more to complete.
+ *
* @returns RD_KAFKA_RESP_ERR_NO_ERROR on success or
* RD_KAFKA_RESP_ERR__INVALID_ARG if list is empty, contains invalid
* topics or regexes.
@@ -2600,6 +3091,10 @@
* of the last consumed message + 1, or RD_KAFKA_OFFSET_INVALID in case there was
* no previous message.
*
+ * @remark In this context the last consumed message is the offset consumed
+ * by the current librdkafka instance and, in case of rebalancing, not
+ * necessarily the last message fetched from the partition.
+ *
* @returns RD_KAFKA_RESP_ERR_NO_ERROR on success in which case the
* \p offset or \p err field of each \p partitions' element is filled
* in with the stored offset, or a partition specific error.
@@ -2637,8 +3132,9 @@
* Failure to do so will result
* in indefinately blocking on
* the produce() call when the
- * message queue is full.
- */
+ * message queue is full. */
+#define RD_KAFKA_MSG_F_PARTITION 0x8 /**< produce_batch() will honor
+ * per-message partition. */
@@ -2649,6 +3145,26 @@
* `rd_kafka_topic_new()`.
*
* `rd_kafka_produce()` is an asynch non-blocking API.
+ * See `rd_kafka_conf_set_dr_msg_cb` on how to setup a callback to be called
+ * once the delivery status (success or failure) is known. The delivery report
+ * is trigged by the application calling `rd_kafka_poll()` (at regular
+ * intervals) or `rd_kafka_flush()` (at termination).
+ *
+ * Since producing is asynchronous, you should call `rd_kafka_flush()` before
+ * you destroy the producer. Otherwise, any outstanding messages will be
+ * silently discarded.
+ *
+ * When temporary errors occur, librdkafka automatically retries to produce the
+ * messages. Retries are triggered after retry.backoff.ms and when the
+ * leader broker for the given partition is available. Otherwise, librdkafka
+ * falls back to polling the topic metadata to monitor when a new leader is
+ * elected (see the topic.metadata.refresh.fast.interval.ms and
+ * topic.metadata.refresh.interval.ms configurations) and then performs a
+ * retry. A delivery error will occur if the message could not be produced
+ * within message.timeout.ms.
+ *
+ * See the "Message reliability" chapter in INTRODUCTION.md for more
+ * information.
*
* \p partition is the target partition, either:
* - RD_KAFKA_PARTITION_UA (unassigned) for
@@ -2672,6 +3188,9 @@
* RD_KAFKA_MSG_F_COPY - the \p payload data will be copied and the
* \p payload pointer will not be used by rdkafka
* after the call returns.
+ * RD_KAFKA_MSG_F_PARTITION - produce_batch() will honour per-message
+ * partition, either set manually or by the
+ * configured partitioner.
*
* .._F_FREE and .._F_COPY are mutually exclusive.
*
@@ -2722,6 +3241,8 @@
* tag tuples which must be terminated with a single \c RD_KAFKA_V_END.
*
* @returns \c RD_KAFKA_RESP_ERR_NO_ERROR on success, else an error code.
+ * \c RD_KAFKA_RESP_ERR__CONFLICT is returned if _V_HEADER and
+ * _V_HEADERS are mixed.
*
* @sa rd_kafka_produce, RD_KAFKA_V_END
*/
@@ -2749,6 +3270,9 @@
* return value != \p message_cnt.
*
* @returns the number of messages succesfully enqueued for producing.
+ *
+ * @remark This interface does NOT support setting message headers on
+ * the provided \p rkmessages.
*/
RD_EXPORT
int rd_kafka_produce_batch(rd_kafka_topic_t *rkt, int32_t partition,
@@ -2768,6 +3292,8 @@
*
* @returns RD_KAFKA_RESP_ERR__TIMED_OUT if \p timeout_ms was reached before all
* outstanding requests were completed, else RD_KAFKA_RESP_ERR_NO_ERROR
+ *
+ * @sa rd_kafka_outq_len()
*/
RD_EXPORT
rd_kafka_resp_err_t rd_kafka_flush (rd_kafka_t *rk, int timeout_ms);
@@ -3052,14 +3578,24 @@
/**
* @brief Returns the current out queue length.
*
- * The out queue contains messages waiting to be sent to, or acknowledged by,
- * the broker.
+ * The out queue length is the sum of:
+ * - number of messages waiting to be sent to, or acknowledged by,
+ * the broker.
+ * - number of delivery reports (e.g., dr_msg_cb) waiting to be served
+ * by rd_kafka_poll() or rd_kafka_flush().
+ * - number of callbacks (e.g., error_cb, stats_cb, etc) waiting to be
+ * served by rd_kafka_poll(), rd_kafka_consumer_poll() or rd_kafka_flush().
+ * - number of events waiting to be served by background_event_cb() in
+ * the background queue (see rd_kafka_conf_set_background_event_cb).
+ *
+ * An application should wait for the return value of this function to reach
+ * zero before terminating to make sure outstanding messages,
+ * requests (such as offset commits), callbacks and events are fully processed.
+ * See rd_kafka_flush().
*
- * An application should wait for this queue to reach zero before terminating
- * to make sure outstanding requests (such as offset commits) are fully
- * processed.
+ * @returns number of messages and events waiting in queues.
*
- * @returns number of messages in the out queue.
+ * @sa rd_kafka_flush()
*/
RD_EXPORT
int rd_kafka_outq_len(rd_kafka_t *rk);
@@ -3152,9 +3688,11 @@
#define RD_KAFKA_EVENT_REBALANCE 0x10 /**< Group rebalance (consumer) */
#define RD_KAFKA_EVENT_OFFSET_COMMIT 0x20 /**< Offset commit result */
#define RD_KAFKA_EVENT_STATS 0x40 /**< Stats */
-
-
-typedef struct rd_kafka_op_s rd_kafka_event_t;
+#define RD_KAFKA_EVENT_CREATETOPICS_RESULT 100 /**< CreateTopics_result_t */
+#define RD_KAFKA_EVENT_DELETETOPICS_RESULT 101 /**< DeleteTopics_result_t */
+#define RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT 102 /**< CreatePartitions_result_t */
+#define RD_KAFKA_EVENT_ALTERCONFIGS_RESULT 103 /**< AlterConfigs_result_t */
+#define RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT 104 /**< DescribeConfigs_result_t */
/**
@@ -3265,7 +3803,12 @@
* @returns the user opaque (if any)
*
* Event types:
- * - RD_KAFKA_OFFSET_COMMIT
+ * - RD_KAFKA_EVENT_OFFSET_COMMIT
+ * - RD_KAFKA_EVENT_CREATETOPICS_RESULT
+ * - RD_KAFKA_EVENT_DELETETOPICS_RESULT
+ * - RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT
+ * - RD_KAFKA_EVENT_ALTERCONFIGS_RESULT
+ * - RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
*/
RD_EXPORT
void *rd_kafka_event_opaque (rd_kafka_event_t *rkev);
@@ -3325,12 +3868,74 @@
rd_kafka_event_topic_partition (rd_kafka_event_t *rkev);
+
+typedef rd_kafka_event_t rd_kafka_CreateTopics_result_t;
+typedef rd_kafka_event_t rd_kafka_DeleteTopics_result_t;
+typedef rd_kafka_event_t rd_kafka_CreatePartitions_result_t;
+typedef rd_kafka_event_t rd_kafka_AlterConfigs_result_t;
+typedef rd_kafka_event_t rd_kafka_DescribeConfigs_result_t;
+
+/**
+ * @returns the result of a CreateTopics request, or NULL if event is of
+ * different type.
+ *
+ * Event types:
+ * RD_KAFKA_EVENT_CREATETOPICS_RESULT
+ */
+RD_EXPORT const rd_kafka_CreateTopics_result_t *
+rd_kafka_event_CreateTopics_result (rd_kafka_event_t *rkev);
+
+/**
+ * @returns the result of a DeleteTopics request, or NULL if event is of
+ * different type.
+ *
+ * Event types:
+ * RD_KAFKA_EVENT_DELETETOPICS_RESULT
+ */
+RD_EXPORT const rd_kafka_DeleteTopics_result_t *
+rd_kafka_event_DeleteTopics_result (rd_kafka_event_t *rkev);
+
+/**
+ * @returns the result of a CreatePartitions request, or NULL if event is of
+ * different type.
+ *
+ * Event types:
+ * RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT
+ */
+RD_EXPORT const rd_kafka_CreatePartitions_result_t *
+rd_kafka_event_CreatePartitions_result (rd_kafka_event_t *rkev);
+
+/**
+ * @returns the result of a AlterConfigs request, or NULL if event is of
+ * different type.
+ *
+ * Event types:
+ * RD_KAFKA_EVENT_ALTERCONFIGS_RESULT
+ */
+RD_EXPORT const rd_kafka_AlterConfigs_result_t *
+rd_kafka_event_AlterConfigs_result (rd_kafka_event_t *rkev);
+
+/**
+ * @returns the result of a DescribeConfigs request, or NULL if event is of
+ * different type.
+ *
+ * Event types:
+ * RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
+ */
+RD_EXPORT const rd_kafka_DescribeConfigs_result_t *
+rd_kafka_event_DescribeConfigs_result (rd_kafka_event_t *rkev);
+
+
+
+
/**
* @brief Poll a queue for an event for max \p timeout_ms.
*
* @returns an event, or NULL.
*
* @remark Use rd_kafka_event_destroy() to free the event.
+ *
+ * @sa rd_kafka_conf_set_background_event_cb()
*/
RD_EXPORT
rd_kafka_event_t *rd_kafka_queue_poll (rd_kafka_queue_t *rkqu, int timeout_ms);
@@ -3342,6 +3947,11 @@
*
* @remark This API must only be used for queues with callbacks registered
* for all expected event types. E.g., not a message queue.
+*
+* @remark Also see rd_kafka_conf_set_background_event_cb() for triggering
+* event callbacks from a librdkafka-managed background thread.
+*
+* @sa rd_kafka_conf_set_background_event_cb()
*/
RD_EXPORT
int rd_kafka_queue_poll_callback (rd_kafka_queue_t *rkqu, int timeout_ms);
@@ -3427,9 +4037,9 @@
* interceptor chains.
*
* @remark Contrary to the Java client the librdkafka interceptor interface
- * does not support message modification. Message mutability is
- * discouraged in the Java client and the combination of
- * serializers and headers cover most use-cases.
+ * does not support message key and value modification.
+ * Message mutability is discouraged in the Java client and the
+ * combination of serializers and headers cover most use-cases.
*
* @remark Interceptors are NOT copied to the new configuration on
* rd_kafka_conf_dup() since it would be hard for interceptors to
@@ -3650,6 +4260,40 @@
rd_kafka_resp_err_t err, void *ic_opaque);
+/**
+ * @brief on_request_sent() is called when a request has been fully written
+ * to a broker TCP connections socket.
+ *
+ * @param rk The client instance.
+ * @param sockfd Socket file descriptor.
+ * @param brokername Broker request is being sent to.
+ * @param brokerid Broker request is being sent to.
+ * @param ApiKey Kafka protocol request type.
+ * @param ApiVersion Kafka protocol request type version.
+ * @param Corrid Kafka protocol request correlation id.
+ * @param size Size of request.
+ * @param ic_opaque The interceptor's opaque pointer specified in ..add..().
+ *
+ * @warning The on_request_sent() interceptor is called from internal
+ * librdkafka broker threads. An on_request_sent() interceptor MUST NOT
+ * call any librdkafka API's associated with the \p rk, or perform
+ * any blocking or prolonged work.
+ *
+ * @returns an error code on failure, the error is logged but otherwise ignored.
+ */
+typedef rd_kafka_resp_err_t
+(rd_kafka_interceptor_f_on_request_sent_t) (
+ rd_kafka_t *rk,
+ int sockfd,
+ const char *brokername,
+ int32_t brokerid,
+ int16_t ApiKey,
+ int16_t ApiVersion,
+ int32_t CorrId,
+ size_t size,
+ void *ic_opaque);
+
+
/**
* @brief Append an on_conf_set() interceptor.
@@ -3831,11 +4475,932 @@
void *ic_opaque);
+/**
+ * @brief Append an on_request_sent() interceptor.
+ *
+ * @param rk Client instance.
+ * @param ic_name Interceptor name, used in logging.
+ * @param on_request_sent() Function pointer.
+ * @param ic_opaque Opaque value that will be passed to the function.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success or RD_KAFKA_RESP_ERR__CONFLICT
+ * if an existing intercepted with the same \p ic_name and function
+ * has already been added to \p conf.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_interceptor_add_on_request_sent (
+ rd_kafka_t *rk, const char *ic_name,
+ rd_kafka_interceptor_f_on_request_sent_t *on_request_sent,
+ void *ic_opaque);
+
+
+
+
+/**@}*/
+
+/**
+ * @name Auxiliary types
+ *
+ * @{
+ */
+
+
+
+/**
+ * @brief Topic result provides per-topic operation result information.
+ *
+ */
+
+/**
+ * @returns the error code for the given topic result.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_topic_result_error (const rd_kafka_topic_result_t *topicres);
+
+/**
+ * @returns the human readable error string for the given topic result,
+ * or NULL if there was no error.
+ *
+ * @remark lifetime of the returned string is the same as the \p topicres.
+ */
+RD_EXPORT const char *
+rd_kafka_topic_result_error_string (const rd_kafka_topic_result_t *topicres);
+
+/**
+ * @returns the name of the topic for the given topic result.
+ * @remark lifetime of the returned string is the same as the \p topicres.
+ *
+ */
+RD_EXPORT const char *
+rd_kafka_topic_result_name (const rd_kafka_topic_result_t *topicres);
+
+
+/**@}*/
+
+
+/**
+ * @name Admin API
+ *
+ * @{
+ *
+ * @brief The Admin API enables applications to perform administrative
+ * Apache Kafka tasks, such as creating and deleting topics,
+ * altering and reading broker configuration, etc.
+ *
+ * The Admin API is asynchronous and makes use of librdkafka's standard
+ * \c rd_kafka_queue_t queues to propagate the result of an admin operation
+ * back to the application.
+ * The supplied queue may be any queue, such as a temporary single-call queue,
+ * a shared queue used for multiple requests, or even the main queue or
+ * consumer queues.
+ *
+ * Use \c rd_kafka_queue_poll() to collect the result of an admin operation
+ * from the queue of your choice, then extract the admin API-specific result
+ * type by using the corresponding \c rd_kafka_event_CreateTopics_result,
+ * \c rd_kafka_event_DescribeConfigs_result, etc, methods.
+ * Use the getter methods on the \c .._result_t type to extract response
+ * information and finally destroy the result and event by calling
+ * \c rd_kafka_event_destroy().
+ *
+ * Use rd_kafka_event_error() and rd_kafka_event_error_string() to acquire
+ * the request-level error/success for an Admin API request.
+ * Even if the returned value is \c RD_KAFKA_RESP_ERR_NO_ERROR there
+ * may be individual objects (topics, resources, etc) that have failed.
+ * Extract per-object error information with the corresponding
+ * \c rd_kafka_..._result_topics|resources|..() to check per-object errors.
+ *
+ * Locally triggered errors:
+ * - \c RD_KAFKA_RESP_ERR__TIMED_OUT - (Controller) broker connection did not
+ * become available in the time allowed by AdminOption_set_request_timeout.
+ */
+
+
+/**
+ * @enum rd_kafka_admin_op_t
+ *
+ * @brief Admin operation enum name for use with rd_kafka_AdminOptions_new()
+ *
+ * @sa rd_kafka_AdminOptions_new()
+ */
+typedef enum rd_kafka_admin_op_t {
+ RD_KAFKA_ADMIN_OP_ANY = 0, /**< Default value */
+ RD_KAFKA_ADMIN_OP_CREATETOPICS, /**< CreateTopics */
+ RD_KAFKA_ADMIN_OP_DELETETOPICS, /**< DeleteTopics */
+ RD_KAFKA_ADMIN_OP_CREATEPARTITIONS, /**< CreatePartitions */
+ RD_KAFKA_ADMIN_OP_ALTERCONFIGS, /**< AlterConfigs */
+ RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS, /**< DescribeConfigs */
+ RD_KAFKA_ADMIN_OP__CNT /**< Number of ops defined */
+} rd_kafka_admin_op_t;
+
+/**
+ * @brief AdminOptions provides a generic mechanism for setting optional
+ * parameters for the Admin API requests.
+ *
+ * @remark Since AdminOptions is decoupled from the actual request type
+ * there is no enforcement to prevent setting unrelated properties,
+ * e.g. setting validate_only on a DescribeConfigs request is allowed
+ * but is silently ignored by DescribeConfigs.
+ * Future versions may introduce such enforcement.
+ */
+
+
+typedef struct rd_kafka_AdminOptions_s rd_kafka_AdminOptions_t;
+
+/**
+ * @brief Create a new AdminOptions object.
+ *
+ * The options object is not modified by the Admin API request APIs,
+ * (e.g. CreateTopics) and may be reused for multiple calls.
+ *
+ * @param rk Client instance.
+ * @param for_api Specifies what Admin API this AdminOptions object will be used
+ * for, which will enforce what AdminOptions_set_..() calls may
+ * be used based on the API, causing unsupported set..() calls
+ * to fail.
+ * Specifying RD_KAFKA_ADMIN_OP_ANY disables the enforcement
+ * allowing any option to be set, even if the option
+ * is not used in a future call to an Admin API method.
+ *
+ * @returns a new AdminOptions object (which must be freed with
+ * rd_kafka_AdminOptions_destroy()), or NULL if \p for_api was set to
+ * an unknown API op type.
+ */
+RD_EXPORT rd_kafka_AdminOptions_t *
+rd_kafka_AdminOptions_new (rd_kafka_t *rk, rd_kafka_admin_op_t for_api);
+
+
+/**
+ * @brief Destroy a AdminOptions object.
+ */
+RD_EXPORT void rd_kafka_AdminOptions_destroy (rd_kafka_AdminOptions_t *options);
+
+
+/**
+ * @brief Sets the overall request timeout, including broker lookup,
+ * request transmission, operation time on broker, and response.
+ *
+ * @param timeout_ms Timeout in milliseconds, use -1 for indefinite timeout.
+ * Defaults to `socket.timeout.ms`.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, or
+ * RD_KAFKA_RESP_ERR__INVALID_ARG if timeout was out of range in which
+ * case an error string will be written \p errstr.
+ *
+ * @remark This option is valid for all Admin API requests.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_AdminOptions_set_request_timeout (rd_kafka_AdminOptions_t *options,
+ int timeout_ms,
+ char *errstr, size_t errstr_size);
+
+
+/**
+ * @brief Sets the broker's operation timeout, such as the timeout for
+ * CreateTopics to complete the creation of topics on the controller
+ * before returning a result to the application.
+ *
+ * CreateTopics: values <= 0 will return immediately after triggering topic
+ * creation, while > 0 will wait this long for topic creation to propagate
+ * in cluster. Default: 0.
+ *
+ * DeleteTopics: same semantics as CreateTopics.
+ * CreatePartitions: same semantics as CreateTopics.
+ *
+ *
+ * @param timeout_ms Timeout in milliseconds.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, or
+ * RD_KAFKA_RESP_ERR__INVALID_ARG if timeout was out of range in which
+ * case an error string will be written \p errstr.
+ *
+ * @remark This option is valid for CreateTopics, DeleteTopics and
+ * CreatePartitions.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_AdminOptions_set_operation_timeout (rd_kafka_AdminOptions_t *options,
+ int timeout_ms,
+ char *errstr, size_t errstr_size);
+
+
+/**
+ * @brief Tell broker to only validate the request, without performing
+ * the requested operation (create topics, etc).
+ *
+ * @param true_or_false Defaults to false.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success or an
+ * error code on failure in which case an error string will
+ * be written \p errstr.
+ *
+ * @remark This option is valid for CreateTopics,
+ * CreatePartitions, AlterConfigs.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_AdminOptions_set_validate_only (rd_kafka_AdminOptions_t *options,
+ int true_or_false,
+ char *errstr, size_t errstr_size);
+
+
+/**
+ * @brief Override what broker the Admin request will be sent to.
+ *
+ * By default, Admin requests are sent to the controller broker, with
+ * the following exceptions:
+ * - AlterConfigs with a BROKER resource are sent to the broker id set
+ * as the resource name.
+ * - DescribeConfigs with a BROKER resource are sent to the broker id set
+ * as the resource name.
+ *
+ * @param broker_id The broker to send the request to.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success or an
+ * error code on failure in which case an error string will
+ * be written \p errstr.
+ *
+ * @remark This API should typically not be used, but serves as a workaround
+ * if new resource types are to the broker that the client
+ * does not know where to send.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_AdminOptions_set_broker (rd_kafka_AdminOptions_t *options,
+ int32_t broker_id,
+ char *errstr, size_t errstr_size);
+
+
+
+/**
+ * @brief Set application opaque value that can be extracted from the
+ * result event using rd_kafka_event_opaque()
+ */
+RD_EXPORT void
+rd_kafka_AdminOptions_set_opaque (rd_kafka_AdminOptions_t *options,
+ void *opaque);
+
+
+
+
+
+
+/**
+ * @section CreateTopics - create topics in cluster
+ *
+ *
+ */
+
+
+typedef struct rd_kafka_NewTopic_s rd_kafka_NewTopic_t;
+
+/**
+ * @brief Create a new NewTopic object. This object is later passed to
+ * rd_kafka_CreateTopics().
+ *
+ * @param topic Topic name to create.
+ * @param num_partitions Number of partitions in topic.
+ * @param replication_factor Default replication factor for the topic's
+ * partitions, or -1 if set_replica_assignment()
+ * will be used.
+ *
+ * @returns a new allocated NewTopic object, or NULL if the input parameters
+ * are invalid.
+ * Use rd_kafka_NewTopic_destroy() to free object when done.
+ */
+RD_EXPORT rd_kafka_NewTopic_t *
+rd_kafka_NewTopic_new (const char *topic, int num_partitions,
+ int replication_factor,
+ char *errstr, size_t errstr_size);
+
+/**
+ * @brief Destroy and free a NewTopic object previously created with
+ * rd_kafka_NewTopic_new()
+ */
+RD_EXPORT void
+rd_kafka_NewTopic_destroy (rd_kafka_NewTopic_t *new_topic);
+
+
+/**
+ * @brief Helper function to destroy all NewTopic objects in the \p new_topics
+ * array (of \p new_topic_cnt elements).
+ * The array itself is not freed.
+ */
+RD_EXPORT void
+rd_kafka_NewTopic_destroy_array (rd_kafka_NewTopic_t **new_topics,
+ size_t new_topic_cnt);
+
+
+/**
+ * @brief Set the replica (broker) assignment for \p partition to the
+ * replica set in \p broker_ids (of \p broker_id_cnt elements).
+ *
+ * @remark When this method is used, rd_kafka_NewTopic_new() must have
+ * been called with a \c replication_factor of -1.
+ *
+ * @remark An application must either set the replica assignment for
+ * all new partitions, or none.
+ *
+ * @remark If called, this function must be called consecutively for each
+ * partition, starting at 0.
+ *
+ * @remark Use rd_kafka_metadata() to retrieve the list of brokers
+ * in the cluster.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, or an error code
+ * if the arguments were invalid.
+ *
+ * @sa rd_kafka_AdminOptions_set_validate_only()
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_NewTopic_set_replica_assignment (rd_kafka_NewTopic_t *new_topic,
+ int32_t partition,
+ int32_t *broker_ids,
+ size_t broker_id_cnt,
+ char *errstr, size_t errstr_size);
+
+/**
+ * @brief Set (broker-side) topic configuration name/value pair.
+ *
+ * @remark The name and value are not validated by the client, the validation
+ * takes place on the broker.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, or an error code
+ * if the arguments were invalid.
+ *
+ * @sa rd_kafka_AdminOptions_set_validate_only()
+ * @sa http://kafka.apache.org/documentation.html#topicconfigs
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_NewTopic_set_config (rd_kafka_NewTopic_t *new_topic,
+ const char *name, const char *value);
+
+
+/**
+ * @brief Create topics in cluster as specified by the \p new_topics
+ * array of size \p new_topic_cnt elements.
+ *
+ * @param new_topics Array of new topics to create.
+ * @param new_topic_cnt Number of elements in \p new_topics array.
+ * @param options Optional admin options, or NULL for defaults.
+ * @param rkqu Queue to emit result on.
+ *
+ * Supported admin options:
+ * - rd_kafka_AdminOptions_set_validate_only() - default false
+ * - rd_kafka_AdminOptions_set_operation_timeout() - default 0
+ * - rd_kafka_AdminOptions_set_timeout() - default socket.timeout.ms
+ *
+ * @remark The result event type emitted on the supplied queue is of type
+ * \c RD_KAFKA_EVENT_CREATETOPICS_RESULT
+ */
+RD_EXPORT void
+rd_kafka_CreateTopics (rd_kafka_t *rk,
+ rd_kafka_NewTopic_t **new_topics,
+ size_t new_topic_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu);
+
+
+/**
+ * @brief CreateTopics result type and methods
+ */
+
+/**
+ * @brief Get an array of topic results from a CreateTopics result.
+ *
+ * The returned \p topics life-time is the same as the \p result object.
+ * @param cntp is updated to the number of elements in the array.
+ */
+RD_EXPORT const rd_kafka_topic_result_t **
+rd_kafka_CreateTopics_result_topics (
+ const rd_kafka_CreateTopics_result_t *result,
+ size_t *cntp);
+
+
+
+
+
+/**
+ * @section DeleteTopics - delete topics from cluster
+ *
+ *
+ */
+
+typedef struct rd_kafka_DeleteTopic_s rd_kafka_DeleteTopic_t;
+
+/**
+ * @brief Create a new DeleteTopic object. This object is later passed to
+ * rd_kafka_DeleteTopics().
+ *
+ * @param topic Topic name to delete.
+ *
+ * @returns a new allocated DeleteTopic object.
+ * Use rd_kafka_DeleteTopic_destroy() to free object when done.
+ */
+RD_EXPORT rd_kafka_DeleteTopic_t *
+rd_kafka_DeleteTopic_new (const char *topic);
+
+/**
+ * @brief Destroy and free a DeleteTopic object previously created with
+ * rd_kafka_DeleteTopic_new()
+ */
+RD_EXPORT void
+rd_kafka_DeleteTopic_destroy (rd_kafka_DeleteTopic_t *del_topic);
+
+/**
+ * @brief Helper function to destroy all DeleteTopic objects in
+ * the \p del_topics array (of \p del_topic_cnt elements).
+ * The array itself is not freed.
+ */
+RD_EXPORT void
+rd_kafka_DeleteTopic_destroy_array (rd_kafka_DeleteTopic_t **del_topics,
+ size_t del_topic_cnt);
+
+/**
+ * @brief Delete topics from cluster as specified by the \p topics
+ * array of size \p topic_cnt elements.
+ *
+ * @param topics Array of topics to delete.
+ * @param topic_cnt Number of elements in \p topics array.
+ * @param options Optional admin options, or NULL for defaults.
+ * @param rkqu Queue to emit result on.
+ *
+ * @remark The result event type emitted on the supplied queue is of type
+ * \c RD_KAFKA_EVENT_DELETETOPICS_RESULT
+ */
+RD_EXPORT
+void rd_kafka_DeleteTopics (rd_kafka_t *rk,
+ rd_kafka_DeleteTopic_t **del_topics,
+ size_t del_topic_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu);
+
+
+
+/**
+ * @brief DeleteTopics result type and methods
+ */
+
+/**
+ * @brief Get an array of topic results from a DeleteTopics result.
+ *
+ * The returned \p topics life-time is the same as the \p result object.
+ * @param cntp is updated to the number of elements in the array.
+ */
+RD_EXPORT const rd_kafka_topic_result_t **
+rd_kafka_DeleteTopics_result_topics (
+ const rd_kafka_DeleteTopics_result_t *result,
+ size_t *cntp);
+
+
+
+
+
+
+/**
+ * @section CreatePartitions - add partitions to topic.
+ *
+ *
+ */
+
+typedef struct rd_kafka_NewPartitions_s rd_kafka_NewPartitions_t;
+
+/**
+ * @brief Create a new NewPartitions. This object is later passed to
+ * rd_kafka_CreatePartitions() to increas the number of partitions
+ * to \p new_total_cnt for an existing topic.
+ *
+ * @param topic Topic name to create more partitions for.
+ * @param new_total_cnt Increase the topic's partition count to this value.
+ *
+ * @returns a new allocated NewPartitions object, or NULL if the
+ * input parameters are invalid.
+ * Use rd_kafka_NewPartitions_destroy() to free object when done.
+ */
+RD_EXPORT rd_kafka_NewPartitions_t *
+rd_kafka_NewPartitions_new (const char *topic, size_t new_total_cnt,
+ char *errstr, size_t errstr_size);
+
+/**
+ * @brief Destroy and free a NewPartitions object previously created with
+ * rd_kafka_NewPartitions_new()
+ */
+RD_EXPORT void
+rd_kafka_NewPartitions_destroy (rd_kafka_NewPartitions_t *new_parts);
+
+/**
+ * @brief Helper function to destroy all NewPartitions objects in the
+ * \p new_parts array (of \p new_parts_cnt elements).
+ * The array itself is not freed.
+ */
+RD_EXPORT void
+rd_kafka_NewPartitions_destroy_array (rd_kafka_NewPartitions_t **new_parts,
+ size_t new_parts_cnt);
+
+/**
+ * @brief Set the replica (broker id) assignment for \p new_partition_idx to the
+ * replica set in \p broker_ids (of \p broker_id_cnt elements).
+ *
+ * @remark An application must either set the replica assignment for
+ * all new partitions, or none.
+ *
+ * @remark If called, this function must be called consecutively for each
+ * new partition being created,
+ * where \p new_partition_idx 0 is the first new partition,
+ * 1 is the second, and so on.
+ *
+ * @remark \p broker_id_cnt should match the topic's replication factor.
+ *
+ * @remark Use rd_kafka_metadata() to retrieve the list of brokers
+ * in the cluster.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR on success, or an error code
+ * if the arguments were invalid.
+ *
+ * @sa rd_kafka_AdminOptions_set_validate_only()
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_NewPartitions_set_replica_assignment (rd_kafka_NewPartitions_t *new_parts,
+ int32_t new_partition_idx,
+ int32_t *broker_ids,
+ size_t broker_id_cnt,
+ char *errstr,
+ size_t errstr_size);
+
+
+/**
+ * @brief Create additional partitions for the given topics, as specified
+ * by the \p new_parts array of size \p new_parts_cnt elements.
+ *
+ * @param new_parts Array of topics for which new partitions are to be created.
+ * @param new_parts_cnt Number of elements in \p new_parts array.
+ * @param options Optional admin options, or NULL for defaults.
+ * @param rkqu Queue to emit result on.
+ *
+ * Supported admin options:
+ * - rd_kafka_AdminOptions_set_validate_only() - default false
+ * - rd_kafka_AdminOptions_set_operation_timeout() - default 0
+ * - rd_kafka_AdminOptions_set_timeout() - default socket.timeout.ms
+ *
+ * @remark The result event type emitted on the supplied queue is of type
+ * \c RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT
+ */
+RD_EXPORT void
+rd_kafka_CreatePartitions (rd_kafka_t *rk,
+ rd_kafka_NewPartitions_t **new_parts,
+ size_t new_parts_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu);
+
+
+
+/**
+ * @brief CreatePartitions result type and methods
+ */
+
+/**
+ * @brief Get an array of topic results from a CreatePartitions result.
+ *
+ * The returned \p topics life-time is the same as the \p result object.
+ * @param cntp is updated to the number of elements in the array.
+ */
+RD_EXPORT const rd_kafka_topic_result_t **
+rd_kafka_CreatePartitions_result_topics (
+ const rd_kafka_CreatePartitions_result_t *result,
+ size_t *cntp);
+
+
+
+
+
+/**
+ * @section Cluster, broker, topic configuration entries, sources, etc.
+ *
+ * These entities relate to the cluster, not the local client.
+ *
+ * @sa rd_kafka_conf_set(), et.al. for local client configuration.
+ *
+ */
+
+/**
+ * @enum Apache Kafka config sources
+ */
+typedef enum rd_kafka_ConfigSource_t {
+ /**< Source unknown, e.g., in the ConfigEntry used for alter requests
+ * where source is not set */
+ RD_KAFKA_CONFIG_SOURCE_UNKNOWN_CONFIG = 0,
+ /**< Dynamic topic config that is configured for a specific topic */
+ RD_KAFKA_CONFIG_SOURCE_DYNAMIC_TOPIC_CONFIG = 1,
+ /**< Dynamic broker config that is configured for a specific broker */
+ RD_KAFKA_CONFIG_SOURCE_DYNAMIC_BROKER_CONFIG = 2,
+ /**< Dynamic broker config that is configured as default for all
+ * brokers in the cluster */
+ RD_KAFKA_CONFIG_SOURCE_DYNAMIC_DEFAULT_BROKER_CONFIG = 3,
+ /**< Static broker config provided as broker properties at startup
+ * (e.g. from server.properties file) */
+ RD_KAFKA_CONFIG_SOURCE_STATIC_BROKER_CONFIG = 4,
+ /**< Built-in default configuration for configs that have a
+ * default value */
+ RD_KAFKA_CONFIG_SOURCE_DEFAULT_CONFIG = 5,
+
+ /**< Number of source types defined */
+ RD_KAFKA_CONFIG_SOURCE__CNT,
+} rd_kafka_ConfigSource_t;
+
+
+/**
+ * @returns a string representation of the \p confsource.
+ */
+RD_EXPORT const char *
+rd_kafka_ConfigSource_name (rd_kafka_ConfigSource_t confsource);
+
+
+typedef struct rd_kafka_ConfigEntry_s rd_kafka_ConfigEntry_t;
+
+/**
+ * @returns the configuration property name
+ */
+RD_EXPORT const char *
+rd_kafka_ConfigEntry_name (const rd_kafka_ConfigEntry_t *entry);
+
+/**
+ * @returns the configuration value, may be NULL for sensitive or unset
+ * properties.
+ */
+RD_EXPORT const char *
+rd_kafka_ConfigEntry_value (const rd_kafka_ConfigEntry_t *entry);
+
+/**
+ * @returns the config source.
+ */
+RD_EXPORT rd_kafka_ConfigSource_t
+rd_kafka_ConfigEntry_source (const rd_kafka_ConfigEntry_t *entry);
+
+/**
+ * @returns 1 if the config property is read-only on the broker, else 0.
+ * @remark Shall only be used on a DescribeConfigs result, otherwise returns -1.
+ */
+RD_EXPORT int
+rd_kafka_ConfigEntry_is_read_only (const rd_kafka_ConfigEntry_t *entry);
+
+/**
+ * @returns 1 if the config property is set to its default value on the broker,
+ * else 0.
+ * @remark Shall only be used on a DescribeConfigs result, otherwise returns -1.
+ */
+RD_EXPORT int
+rd_kafka_ConfigEntry_is_default (const rd_kafka_ConfigEntry_t *entry);
+
+/**
+ * @returns 1 if the config property contains sensitive information (such as
+ * security configuration), else 0.
+ * @remark An application should take care not to include the value of
+ * sensitive configuration entries in its output.
+ * @remark Shall only be used on a DescribeConfigs result, otherwise returns -1.
+ */
+RD_EXPORT int
+rd_kafka_ConfigEntry_is_sensitive (const rd_kafka_ConfigEntry_t *entry);
+
+/**
+ * @returns 1 if this entry is a synonym, else 0.
+ */
+RD_EXPORT int
+rd_kafka_ConfigEntry_is_synonym (const rd_kafka_ConfigEntry_t *entry);
+
+
+/**
+ * @returns the synonym config entry array.
+ *
+ * @param cntp is updated to the number of elements in the array.
+ *
+ * @remark The lifetime of the returned entry is the same as \p conf .
+ * @remark Shall only be used on a DescribeConfigs result,
+ * otherwise returns NULL.
+ */
+RD_EXPORT const rd_kafka_ConfigEntry_t **
+rd_kafka_ConfigEntry_synonyms (const rd_kafka_ConfigEntry_t *entry,
+ size_t *cntp);
+
+
+
+
+/**
+ * @enum Apache Kafka resource types
+ */
+typedef enum rd_kafka_ResourceType_t {
+ RD_KAFKA_RESOURCE_UNKNOWN = 0, /**< Unknown */
+ RD_KAFKA_RESOURCE_ANY = 1, /**< Any (used for lookups) */
+ RD_KAFKA_RESOURCE_TOPIC = 2, /**< Topic */
+ RD_KAFKA_RESOURCE_GROUP = 3, /**< Group */
+ RD_KAFKA_RESOURCE_BROKER = 4, /**< Broker */
+ RD_KAFKA_RESOURCE__CNT, /**< Number of resource types defined */
+} rd_kafka_ResourceType_t;
+
+/**
+ * @returns a string representation of the \p restype
+ */
+RD_EXPORT const char *
+rd_kafka_ResourceType_name (rd_kafka_ResourceType_t restype);
+
+typedef struct rd_kafka_ConfigResource_s rd_kafka_ConfigResource_t;
+
+
+RD_EXPORT rd_kafka_ConfigResource_t *
+rd_kafka_ConfigResource_new (rd_kafka_ResourceType_t restype,
+ const char *resname);
+
+/**
+ * @brief Destroy and free a ConfigResource object previously created with
+ * rd_kafka_ConfigResource_new()
+ */
+RD_EXPORT void
+rd_kafka_ConfigResource_destroy (rd_kafka_ConfigResource_t *config);
+
+
+/**
+ * @brief Helper function to destroy all ConfigResource objects in
+ * the \p configs array (of \p config_cnt elements).
+ * The array itself is not freed.
+ */
+RD_EXPORT void
+rd_kafka_ConfigResource_destroy_array (rd_kafka_ConfigResource_t **config,
+ size_t config_cnt);
+
+
+/**
+ * @brief Set configuration name value pair.
+ *
+ * @param name Configuration name, depends on resource type.
+ * @param value Configuration value, depends on resource type and \p name.
+ * Set to \c NULL to revert configuration value to default.
+ *
+ * This will overwrite the current value.
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR if config was added to resource,
+ * or RD_KAFKA_RESP_ERR__INVALID_ARG on invalid input.
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_ConfigResource_set_config (rd_kafka_ConfigResource_t *config,
+ const char *name, const char *value);
+
+
+/**
+ * @brief Get an array of config entries from a ConfigResource object.
+ *
+ * The returned object life-times are the same as the \p config object.
+ *
+ * @param cntp is updated to the number of elements in the array.
+ */
+RD_EXPORT const rd_kafka_ConfigEntry_t **
+rd_kafka_ConfigResource_configs (const rd_kafka_ConfigResource_t *config,
+ size_t *cntp);
+
+
+
+/**
+ * @returns the ResourceType for \p config
+ */
+RD_EXPORT rd_kafka_ResourceType_t
+rd_kafka_ConfigResource_type (const rd_kafka_ConfigResource_t *config);
+
+/**
+ * @returns the name for \p config
+ */
+RD_EXPORT const char *
+rd_kafka_ConfigResource_name (const rd_kafka_ConfigResource_t *config);
+
+/**
+ * @returns the error for this resource from an AlterConfigs request
+ */
+RD_EXPORT rd_kafka_resp_err_t
+rd_kafka_ConfigResource_error (const rd_kafka_ConfigResource_t *config);
+
+/**
+ * @returns the error string for this resource from an AlterConfigs
+ * request, or NULL if no error.
+ */
+RD_EXPORT const char *
+rd_kafka_ConfigResource_error_string (const rd_kafka_ConfigResource_t *config);
+
+
+/**
+ * @section AlterConfigs - alter cluster configuration.
+ *
+ *
+ */
+
+
+/**
+ * @brief Update the configuration for the specified resources.
+ * Updates are not transactional so they may succeed for a subset
+ * of the provided resources while the others fail.
+ * The configuration for a particular resource is updated atomically,
+ * replacing values using the provided ConfigEntrys and reverting
+ * unspecified ConfigEntrys to their default values.
+ *
+ * @remark Requires broker version >=0.11.0.0
+ *
+ * @warning AlterConfigs will replace all existing configuration for
+ * the provided resources with the new configuration given,
+ * reverting all other configuration to their default values.
+ *
+ * @remark Multiple resources and resource types may be set, but at most one
+ * resource of type \c RD_KAFKA_RESOURCE_BROKER is allowed per call
+ * since these resource requests must be sent to the broker specified
+ * in the resource.
+ *
+ */
+RD_EXPORT
+void rd_kafka_AlterConfigs (rd_kafka_t *rk,
+ rd_kafka_ConfigResource_t **configs,
+ size_t config_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu);
+
+
+/**
+ * @brief AlterConfigs result type and methods
+ */
+
+/**
+ * @brief Get an array of resource results from a AlterConfigs result.
+ *
+ * Use \c rd_kafka_ConfigResource_error() and
+ * \c rd_kafka_ConfigResource_error_string() to extract per-resource error
+ * results on the returned array elements.
+ *
+ * The returned object life-times are the same as the \p result object.
+ *
+ * @param cntp is updated to the number of elements in the array.
+ *
+ * @returns an array of ConfigResource elements, or NULL if not available.
+ */
+RD_EXPORT const rd_kafka_ConfigResource_t **
+rd_kafka_AlterConfigs_result_resources (
+ const rd_kafka_AlterConfigs_result_t *result,
+ size_t *cntp);
+
+
+
+
+
+
+/**
+ * @section DescribeConfigs - retrieve cluster configuration.
+ *
+ *
+ */
+
+
+/**
+ * @brief Get configuration for the specified resources in \p configs.
+ *
+ * The returned configuration includes default values and the
+ * rd_kafka_ConfigEntry_is_default() or rd_kafka_ConfigEntry_source()
+ * methods may be used to distinguish them from user supplied values.
+ *
+ * The value of config entries where rd_kafka_ConfigEntry_is_sensitive()
+ * is true will always be NULL to avoid disclosing sensitive
+ * information, such as security settings.
+ *
+ * Configuration entries where rd_kafka_ConfigEntry_is_read_only()
+ * is true can't be updated (with rd_kafka_AlterConfigs()).
+ *
+ * Synonym configuration entries are returned if the broker supports
+ * it (broker version >= 1.1.0). See rd_kafka_ConfigEntry_synonyms().
+ *
+ * @remark Requires broker version >=0.11.0.0
+ *
+ * @remark Multiple resources and resource types may be requested, but at most
+ * one resource of type \c RD_KAFKA_RESOURCE_BROKER is allowed per call
+ * since these resource requests must be sent to the broker specified
+ * in the resource.
+ */
+RD_EXPORT
+void rd_kafka_DescribeConfigs (rd_kafka_t *rk,
+ rd_kafka_ConfigResource_t **configs,
+ size_t config_cnt,
+ const rd_kafka_AdminOptions_t *options,
+ rd_kafka_queue_t *rkqu);
+
+
+/**
+ * @brief DescribeConfigs result type and methods
+ */
+
+/**
+ * @brief Get an array of resource results from a DescribeConfigs result.
+ *
+ * The returned \p resources life-time is the same as the \p result object.
+ * @param cntp is updated to the number of elements in the array.
+ */
+RD_EXPORT const rd_kafka_ConfigResource_t **
+rd_kafka_DescribeConfigs_result_resources (
+ const rd_kafka_DescribeConfigs_result_t *result,
+ size_t *cntp);
+
/**@}*/
#ifdef __cplusplus
}
#endif
+#endif /* _RDKAFKA_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_header.c librdkafka-0.11.6/src/rdkafka_header.c
--- librdkafka-0.11.3/src/rdkafka_header.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_header.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,222 @@
+/*
+ * librdkafka - The Apache Kafka C/C++ library
+ *
+ * Copyright (c) 2017 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "rdkafka_int.h"
+#include "rdkafka_header.h"
+
+
+
+#define rd_kafka_header_destroy rd_free
+
+void rd_kafka_headers_destroy (rd_kafka_headers_t *hdrs) {
+ rd_list_destroy(&hdrs->rkhdrs_list);
+ rd_free(hdrs);
+}
+
+rd_kafka_headers_t *rd_kafka_headers_new (size_t initial_count) {
+ rd_kafka_headers_t *hdrs;
+
+ hdrs = rd_malloc(sizeof(*hdrs));
+ rd_list_init(&hdrs->rkhdrs_list, (int)initial_count,
+ rd_kafka_header_destroy);
+ hdrs->rkhdrs_ser_size = 0;
+
+ return hdrs;
+}
+
+static void *rd_kafka_header_copy (const void *_src, void *opaque) {
+ rd_kafka_headers_t *hdrs = opaque;
+ const rd_kafka_header_t *src = (const rd_kafka_header_t *)_src;
+
+ return (void *)rd_kafka_header_add(
+ hdrs,
+ src->rkhdr_name, src->rkhdr_name_size,
+ src->rkhdr_value, src->rkhdr_value_size);
+}
+
+rd_kafka_headers_t *
+rd_kafka_headers_copy (const rd_kafka_headers_t *src) {
+ rd_kafka_headers_t *dst;
+
+ dst = rd_malloc(sizeof(*dst));
+ rd_list_init(&dst->rkhdrs_list, rd_list_cnt(&src->rkhdrs_list),
+ rd_kafka_header_destroy);
+ dst->rkhdrs_ser_size = 0; /* Updated by header_copy() */
+ rd_list_copy_to(&dst->rkhdrs_list, &src->rkhdrs_list,
+ rd_kafka_header_copy, dst);
+
+ return dst;
+}
+
+
+
+rd_kafka_resp_err_t
+rd_kafka_header_add (rd_kafka_headers_t *hdrs,
+ const char *name, ssize_t name_size,
+ const void *value, ssize_t value_size) {
+ rd_kafka_header_t *hdr;
+ char varint_NameLen[RD_UVARINT_ENC_SIZEOF(int32_t)];
+ char varint_ValueLen[RD_UVARINT_ENC_SIZEOF(int32_t)];
+
+ if (name_size == -1)
+ name_size = strlen(name);
+
+ if (value_size == -1)
+ value_size = value ? strlen(value) : 0;
+ else if (!value)
+ value_size = 0;
+
+ hdr = rd_malloc(sizeof(*hdr) + name_size + 1 + value_size + 1);
+ hdr->rkhdr_name_size = name_size;
+ memcpy((void *)hdr->rkhdr_name, name, name_size);
+ hdr->rkhdr_name[name_size] = '\0';
+
+ if (likely(value != NULL)) {
+ hdr->rkhdr_value = hdr->rkhdr_name+name_size+1;
+ memcpy((void *)hdr->rkhdr_value, value, value_size);
+ hdr->rkhdr_value[value_size] = '\0';
+ hdr->rkhdr_value_size = value_size;
+ } else {
+ hdr->rkhdr_value = NULL;
+ hdr->rkhdr_value_size = 0;
+ }
+
+ rd_list_add(&hdrs->rkhdrs_list, hdr);
+
+ /* Calculate serialized size of header */
+ hdr->rkhdr_ser_size = name_size + value_size;
+ hdr->rkhdr_ser_size += rd_uvarint_enc_i64(varint_NameLen,
+ sizeof(varint_NameLen),
+ name_size);
+ hdr->rkhdr_ser_size += rd_uvarint_enc_i64(varint_ValueLen,
+ sizeof(varint_ValueLen),
+ value_size);
+ hdrs->rkhdrs_ser_size += hdr->rkhdr_ser_size;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+/**
+ * @brief header_t(name) to char * comparator
+ */
+static int rd_kafka_header_cmp_str (void *_a, void *_b) {
+ const rd_kafka_header_t *a = _a;
+ const char *b = _b;
+
+ return strcmp(a->rkhdr_name, b);
+}
+
+rd_kafka_resp_err_t rd_kafka_header_remove (rd_kafka_headers_t *hdrs,
+ const char *name) {
+ size_t ser_size = 0;
+ rd_kafka_header_t *hdr;
+ int i;
+
+ RD_LIST_FOREACH_REVERSE(hdr, &hdrs->rkhdrs_list, i) {
+ if (rd_kafka_header_cmp_str(hdr, (void *)name))
+ continue;
+
+ ser_size += hdr->rkhdr_ser_size;
+ rd_list_remove_elem(&hdrs->rkhdrs_list, i);
+ rd_kafka_header_destroy(hdr);
+ }
+
+ if (ser_size == 0)
+ return RD_KAFKA_RESP_ERR__NOENT;
+
+ rd_dassert(hdrs->rkhdrs_ser_size >= ser_size);
+ hdrs->rkhdrs_ser_size -= ser_size;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+rd_kafka_resp_err_t
+rd_kafka_header_get_last (const rd_kafka_headers_t *hdrs,
+ const char *name,
+ const void **valuep, size_t *sizep) {
+ const rd_kafka_header_t *hdr;
+ int i;
+ size_t name_size = strlen(name);
+
+ RD_LIST_FOREACH_REVERSE(hdr, &hdrs->rkhdrs_list, i) {
+ if (hdr->rkhdr_name_size == name_size &&
+ !strcmp(hdr->rkhdr_name, name)) {
+ *valuep = hdr->rkhdr_value;
+ *sizep = hdr->rkhdr_value_size;
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+ }
+
+ return RD_KAFKA_RESP_ERR__NOENT;
+}
+
+
+rd_kafka_resp_err_t
+rd_kafka_header_get (const rd_kafka_headers_t *hdrs, size_t idx,
+ const char *name,
+ const void **valuep, size_t *sizep) {
+ const rd_kafka_header_t *hdr;
+ int i;
+ size_t mi = 0; /* index for matching names */
+ size_t name_size = strlen(name);
+
+ RD_LIST_FOREACH(hdr, &hdrs->rkhdrs_list, i) {
+ if (hdr->rkhdr_name_size == name_size &&
+ !strcmp(hdr->rkhdr_name, name) &&
+ mi++ == idx) {
+ *valuep = hdr->rkhdr_value;
+ *sizep = hdr->rkhdr_value_size;
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+ }
+
+ return RD_KAFKA_RESP_ERR__NOENT;
+}
+
+
+rd_kafka_resp_err_t
+rd_kafka_header_get_all (const rd_kafka_headers_t *hdrs, size_t idx,
+ const char **namep,
+ const void **valuep, size_t *sizep) {
+ const rd_kafka_header_t *hdr;
+
+ hdr = rd_list_elem(&hdrs->rkhdrs_list, (int)idx);
+ if (unlikely(!hdr))
+ return RD_KAFKA_RESP_ERR__NOENT;
+
+ *namep = hdr->rkhdr_name;
+ *valuep = hdr->rkhdr_value;
+ *sizep = hdr->rkhdr_value_size;
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+size_t rd_kafka_header_cnt(const rd_kafka_headers_t *hdrs) {
+ return (size_t)rd_list_cnt(&hdrs->rkhdrs_list);
+}
diff -Nru librdkafka-0.11.3/src/rdkafka_header.h librdkafka-0.11.6/src/rdkafka_header.h
--- librdkafka-0.11.3/src/rdkafka_header.h 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_header.h 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,76 @@
+/*
+ * librdkafka - The Apache Kafka C/C++ library
+ *
+ * Copyright (c) 2017 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RDKAFKA_HEADER_H
+#define _RDKAFKA_HEADER_H
+
+
+
+/**
+ * @brief The header list (rd_kafka_headers_t) wraps the generic rd_list_t
+ * with additional fields to keep track of the total on-wire size.
+ */
+struct rd_kafka_headers_s {
+ rd_list_t rkhdrs_list; /**< List of (rd_kafka_header_t *) */
+ size_t rkhdrs_ser_size; /**< Total serialized size of headers */
+};
+
+
+/**
+ * @brief The header item itself is a single-allocation immutable structure
+ * (rd_kafka_header_t) containing the header name, value and value
+ * length.
+ * Both the header name and header value are nul-terminated for
+ * API convenience.
+ * The header value is a tri-state:
+ * - proper value (considered binary) with length > 0
+ * - empty value with length = 0 (pointer is non-NULL and nul-termd)
+ * - null value with length = 0 (pointer is NULL)
+ */
+typedef struct rd_kafka_header_s {
+ size_t rkhdr_ser_size; /**< Serialized size */
+ size_t rkhdr_value_size; /**< Value length (without nul-term) */
+ size_t rkhdr_name_size; /**< Header name size (w/o nul-term) */
+ char *rkhdr_value; /**< Header value (nul-terminated string but
+ * considered binary).
+ * Will be NULL for null values, else
+ * points to rkhdr_name+.. */
+ char rkhdr_name[1]; /**< Header name (nul-terminated string).
+ * Followed by allocation for value+nul */
+} rd_kafka_header_t;
+
+
+/**
+ * @returns the serialized size for the headers
+ */
+static RD_INLINE RD_UNUSED size_t
+rd_kafka_headers_serialized_size (const rd_kafka_headers_t *hdrs) {
+ return hdrs->rkhdrs_ser_size;
+}
+
+#endif /* _RDKAFKA_HEADER_H */
diff -Nru librdkafka-0.11.3/src/rdkafka_interceptor.c librdkafka-0.11.6/src/rdkafka_interceptor.c
--- librdkafka-0.11.3/src/rdkafka_interceptor.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_interceptor.c 2018-10-10 06:54:38.000000000 +0000
@@ -44,6 +44,7 @@
rd_kafka_interceptor_f_on_acknowledgement_t *on_acknowledgement;
rd_kafka_interceptor_f_on_consume_t *on_consume;
rd_kafka_interceptor_f_on_commit_t *on_commit;
+ rd_kafka_interceptor_f_on_request_sent_t *on_request_sent;
void *generic; /* For easy assignment */
} u;
@@ -172,6 +173,7 @@
rd_list_destroy(&conf->interceptors.on_acknowledgement);
rd_list_destroy(&conf->interceptors.on_consume);
rd_list_destroy(&conf->interceptors.on_commit);
+ rd_list_destroy(&conf->interceptors.on_request_sent);
/* Interceptor config */
rd_list_destroy(&conf->interceptors.config);
@@ -211,6 +213,9 @@
rd_list_init(&conf->interceptors.on_commit, 0,
rd_kafka_interceptor_method_destroy)
->rl_flags |= RD_LIST_F_UNIQUE;
+ rd_list_init(&conf->interceptors.on_request_sent, 0,
+ rd_kafka_interceptor_method_destroy)
+ ->rl_flags |= RD_LIST_F_UNIQUE;
/* Interceptor config */
rd_list_init(&conf->interceptors.config, 0,
@@ -448,14 +453,20 @@
/**
* @brief Call on_acknowledgement methods for all messages in queue.
+ *
+ * @param force_err If non-zero, sets this error on each message.
+ *
* @locality broker thread
*/
void
rd_kafka_interceptors_on_acknowledgement_queue (rd_kafka_t *rk,
- rd_kafka_msgq_t *rkmq) {
+ rd_kafka_msgq_t *rkmq,
+ rd_kafka_resp_err_t force_err) {
rd_kafka_msg_t *rkm;
RD_KAFKA_MSGQ_FOREACH(rkm, rkmq) {
+ if (force_err)
+ rkm->rkm_err = force_err;
rd_kafka_interceptors_on_acknowledgement(rk,
&rkm->rkm_rkmessage);
}
@@ -511,6 +522,39 @@
}
+/**
+ * @brief Call interceptor on_request_sent methods
+ * @locality internal broker thread
+ */
+void rd_kafka_interceptors_on_request_sent (rd_kafka_t *rk,
+ int sockfd,
+ const char *brokername,
+ int32_t brokerid,
+ int16_t ApiKey,
+ int16_t ApiVersion,
+ int32_t CorrId,
+ size_t size) {
+ rd_kafka_interceptor_method_t *method;
+ int i;
+
+ RD_LIST_FOREACH(method, &rk->rk_conf.interceptors.on_request_sent, i) {
+ rd_kafka_resp_err_t ic_err;
+
+ ic_err = method->u.on_request_sent(rk,
+ sockfd,
+ brokername,
+ brokerid,
+ ApiKey,
+ ApiVersion,
+ CorrId,
+ size,
+ method->ic_opaque);
+ if (unlikely(ic_err))
+ rd_kafka_interceptor_failed(rk, method,
+ "on_request_sent",
+ ic_err, NULL, NULL);
+ }
+}
/**
@@ -622,3 +666,16 @@
ic_name, (void *)on_commit,
ic_opaque);
}
+
+
+rd_kafka_resp_err_t
+rd_kafka_interceptor_add_on_request_sent (
+ rd_kafka_t *rk, const char *ic_name,
+ rd_kafka_interceptor_f_on_request_sent_t *on_request_sent,
+ void *ic_opaque) {
+ assert(!rk->rk_initialized);
+ return rd_kafka_interceptor_method_add(&rk->rk_conf.interceptors.
+ on_request_sent,
+ ic_name, (void *)on_request_sent,
+ ic_opaque);
+}
diff -Nru librdkafka-0.11.3/src/rdkafka_interceptor.h librdkafka-0.11.6/src/rdkafka_interceptor.h
--- librdkafka-0.11.3/src/rdkafka_interceptor.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_interceptor.h 2018-10-10 06:54:38.000000000 +0000
@@ -50,7 +50,8 @@
rd_kafka_message_t *rkmessage);
void
rd_kafka_interceptors_on_acknowledgement_queue (rd_kafka_t *rk,
- rd_kafka_msgq_t *rkmq);
+ rd_kafka_msgq_t *rkmq,
+ rd_kafka_resp_err_t force_err);
void rd_kafka_interceptors_on_consume (rd_kafka_t *rk,
rd_kafka_message_t *rkmessage);
@@ -59,6 +60,15 @@
const rd_kafka_topic_partition_list_t *offsets,
rd_kafka_resp_err_t err);
+void rd_kafka_interceptors_on_request_sent (rd_kafka_t *rk,
+ int sockfd,
+ const char *brokername,
+ int32_t brokerid,
+ int16_t ApiKey,
+ int16_t ApiVersion,
+ int32_t CorrId,
+ size_t size);
+
void rd_kafka_conf_interceptor_ctor (int scope, void *pconf);
void rd_kafka_conf_interceptor_dtor (int scope, void *pconf);
diff -Nru librdkafka-0.11.3/src/rdkafka_int.h librdkafka-0.11.6/src/rdkafka_int.h
--- librdkafka-0.11.3/src/rdkafka_int.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_int.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,8 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
-
+#ifndef _RDKAFKA_INT_H_
+#define _RDKAFKA_INT_H_
#ifndef _MSC_VER
#define _GNU_SOURCE /* for strndup() */
@@ -78,6 +78,7 @@
struct rd_kafka_itopic_s;
struct rd_kafka_msg_s;
struct rd_kafka_broker_s;
+struct rd_kafka_toppar_s;
typedef RD_SHARED_PTR_TYPE(, struct rd_kafka_toppar_s) shptr_rd_kafka_toppar_t;
typedef RD_SHARED_PTR_TYPE(, struct rd_kafka_itopic_s) shptr_rd_kafka_itopic_t;
@@ -100,9 +101,9 @@
/**
* Protocol level sanity
*/
-#define RD_KAFKAP_BROKERS_MAX 1000
+#define RD_KAFKAP_BROKERS_MAX 10000
#define RD_KAFKAP_TOPICS_MAX 1000000
-#define RD_KAFKAP_PARTITIONS_MAX 10000
+#define RD_KAFKAP_PARTITIONS_MAX 100000
#define RD_KAFKA_OFFSET_IS_LOGICAL(OFF) ((OFF) < 0)
@@ -135,7 +136,9 @@
cnd_t rk_broker_state_change_cnd;
mtx_t rk_broker_state_change_lock;
int rk_broker_state_change_version;
-
+ /* List of (rd_kafka_enq_once_t*) objects waiting for broker
+ * state changes. Protected by rk_broker_state_change_lock. */
+ rd_list_t rk_broker_state_change_waiters; /**< (rd_kafka_enq_once_t*) */
TAILQ_HEAD(, rd_kafka_itopic_s) rk_topics;
int rk_topic_cnt;
@@ -149,7 +152,49 @@
rd_kafkap_str_t *rk_group_id; /* Consumer group id */
int rk_flags;
- rd_atomic32_t rk_terminate;
+ rd_atomic32_t rk_terminate; /**< Set to RD_KAFKA_DESTROY_F_..
+ * flags instance
+ * is being destroyed.
+ * The value set is the
+ * destroy flags from
+ * rd_kafka_destroy*() and
+ * the two internal flags shown
+ * below.
+ *
+ * Order:
+ * 1. user_flags | .._F_DESTROY_CALLED
+ * is set in rd_kafka_destroy*().
+ * 2. consumer_close() is called
+ * for consumers.
+ * 3. .._F_TERMINATE is set to
+ * signal all background threads
+ * to terminate.
+ */
+
+#define RD_KAFKA_DESTROY_F_TERMINATE 0x1 /**< Internal flag to make sure
+ * rk_terminate is set to non-zero
+ * value even if user passed
+ * no destroy flags. */
+#define RD_KAFKA_DESTROY_F_DESTROY_CALLED 0x2 /**< Application has called
+ * ..destroy*() and we've
+ * begun the termination
+ * process.
+ * This flag is needed to avoid
+ * rk_terminate from being
+ * 0 when destroy_flags()
+ * is called with flags=0
+ * and prior to _F_TERMINATE
+ * has been set. */
+#define RD_KAFKA_DESTROY_F_IMMEDIATE 0x4 /**< Immediate non-blocking
+ * destruction without waiting
+ * for all resources
+ * to be cleaned up.
+ * WARNING: Memory and resource
+ * leaks possible.
+ * This flag automatically sets
+ * .._NO_CONSUMER_CLOSE. */
+
+
rwlock_t rk_lock;
rd_kafka_type_t rk_type;
struct timeval rk_tv_state_change;
@@ -166,6 +211,7 @@
struct rd_kafka_metadata_cache rk_metadata_cache; /* Metadata cache */
char *rk_clusterid; /* ClusterId from metadata */
+ int32_t rk_controllerid; /* ControllerId from metadata */
/* Simple consumer count:
* >0: Running in legacy / Simple Consumer mode,
@@ -197,6 +243,20 @@
thrd_t rk_thread;
int rk_initialized;
+
+ /**
+ * Background thread and queue,
+ * enabled by setting `background_event_cb()`.
+ */
+ struct {
+ rd_kafka_q_t *q; /**< Queue served by background thread. */
+ thrd_t thread; /**< Background thread. */
+ int calling; /**< Indicates whether the event callback
+ * is being called, reset back to 0
+ * when the callback returns.
+ * This can be used for troubleshooting
+ * purposes. */
+ } rk_background;
};
#define rd_kafka_wrlock(rk) rwlock_wrlock(&(rk)->rk_lock)
@@ -213,10 +273,14 @@
* \p block the function either blocks until enough space is available
* if \p block is 1, else immediately returns
* RD_KAFKA_RESP_ERR__QUEUE_FULL.
+ *
+ * @param rdmtx If non-null and \p block is set and blocking is to ensue,
+ * then unlock this mutex for the duration of the blocking
+ * and then reacquire with a read-lock.
*/
static RD_INLINE RD_UNUSED rd_kafka_resp_err_t
rd_kafka_curr_msgs_add (rd_kafka_t *rk, unsigned int cnt, size_t size,
- int block) {
+ int block, rwlock_t *rdlock) {
if (rk->rk_type != RD_KAFKA_PRODUCER)
return RD_KAFKA_RESP_ERR_NO_ERROR;
@@ -231,7 +295,14 @@
return RD_KAFKA_RESP_ERR__QUEUE_FULL;
}
+ if (rdlock)
+ rwlock_rdunlock(rdlock);
+
cnd_wait(&rk->rk_curr_msgs.cnd, &rk->rk_curr_msgs.lock);
+
+ if (rdlock)
+ rwlock_rdlock(rdlock);
+
}
rk->rk_curr_msgs.cnt += cnt;
@@ -308,9 +379,29 @@
/**
- * Returns true if 'rk' handle is terminating.
+ * @returns true if \p rk handle is terminating.
+ *
+ * @remark If consumer_close() is called from destroy*() it will be
+ * called prior to _F_TERMINATE being set and will thus not
+ * be able to use rd_kafka_terminating() to know it is shutting down.
+ * That code should instead just check that rk_terminate is non-zero
+ * (the _F_DESTROY_CALLED flag will be set).
+ */
+#define rd_kafka_terminating(rk) (rd_atomic32_get(&(rk)->rk_terminate) & \
+ RD_KAFKA_DESTROY_F_TERMINATE)
+
+/**
+ * @returns the destroy flags set matching \p flags, which might be
+ * a subset of the flags.
*/
-#define rd_kafka_terminating(rk) (rd_atomic32_get(&(rk)->rk_terminate))
+#define rd_kafka_destroy_flags_check(rk,flags) \
+ (rd_atomic32_get(&(rk)->rk_terminate) & (flags))
+
+/**
+ * @returns true if no consumer callbacks, or standard consumer_close
+ * behaviour, should be triggered. */
+#define rd_kafka_destroy_flags_no_consumer_close(rk) \
+ rd_kafka_destroy_flags_check(rk, RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE)
#define rd_kafka_is_simple_consumer(rk) \
(rd_atomic32_get(&(rk)->rk_simple_cnt) > 0)
@@ -349,8 +440,10 @@
#define RD_KAFKA_DBG_FETCH 0x400
#define RD_KAFKA_DBG_INTERCEPTOR 0x800
#define RD_KAFKA_DBG_PLUGIN 0x1000
+#define RD_KAFKA_DBG_CONSUMER 0x2000
+#define RD_KAFKA_DBG_ADMIN 0x4000
#define RD_KAFKA_DBG_ALL 0xffff
-
+#define RD_KAFKA_DBG_NONE 0x0
void rd_kafka_log0(const rd_kafka_conf_t *conf,
const rd_kafka_t *rk, const char *extra, int level,
@@ -400,17 +493,14 @@
rd_kafka_resp_err_t rd_kafka_set_last_error (rd_kafka_resp_err_t err,
int errnox) {
if (errnox) {
-#ifdef _MSC_VER
- /* This is the correct way to set errno on Windows,
+ /* MSVC:
+ * This is the correct way to set errno on Windows,
* but it is still pointless due to different errnos in
* in different runtimes:
* https://social.msdn.microsoft.com/Forums/vstudio/en-US/b4500c0d-1b69-40c7-9ef5-08da1025b5bf/setting-errno-from-within-a-dll?forum=vclanguage/
* errno is thus highly deprecated, and buggy, on Windows
* when using librdkafka as a dynamically loaded DLL. */
- _set_errno(errnox);
-#else
- errno = errnox;
-#endif
+ rd_set_errno(errnox);
}
rd_kafka_last_error_code = err;
return err;
@@ -419,11 +509,8 @@
extern rd_atomic32_t rd_kafka_thread_cnt_curr;
-extern char RD_TLS rd_kafka_thread_name[64];
-
-
-
-
+void rd_kafka_set_thread_name (const char *fmt, ...);
+void rd_kafka_set_thread_sysname (const char *fmt, ...);
int rd_kafka_path_is_dir (const char *path);
@@ -433,3 +520,11 @@
rd_kafka_resp_err_t rd_kafka_subscribe_rkt (rd_kafka_itopic_t *rkt);
+
+
+/**
+ * rdkafka_background.c
+ */
+int rd_kafka_background_thread_main (void *arg);
+
+#endif /* _RDKAFKA_INT_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_lz4.c librdkafka-0.11.6/src/rdkafka_lz4.c
--- librdkafka-0.11.3/src/rdkafka_lz4.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_lz4.c 2018-10-10 06:54:38.000000000 +0000
@@ -325,7 +325,7 @@
* @returns allocated buffer in \p *outbuf, length in \p *outlenp.
*/
rd_kafka_resp_err_t
-rd_kafka_lz4_compress (rd_kafka_broker_t *rkb, int proper_hc,
+rd_kafka_lz4_compress (rd_kafka_broker_t *rkb, int proper_hc, int comp_level,
rd_slice_t *slice, void **outbuf, size_t *outlenp) {
LZ4F_compressionContext_t cctx;
LZ4F_errorCode_t r;
@@ -339,8 +339,11 @@
/* Required by Kafka */
const LZ4F_preferences_t prefs =
- { .frameInfo = { .blockMode = LZ4F_blockIndependent } };
-
+ {
+ .frameInfo = { .blockMode = LZ4F_blockIndependent },
+ .compressionLevel = comp_level
+ };
+
*outbuf = NULL;
out_sz = LZ4F_compressBound(len, NULL) + 1000;
diff -Nru librdkafka-0.11.3/src/rdkafka_lz4.h librdkafka-0.11.6/src/rdkafka_lz4.h
--- librdkafka-0.11.3/src/rdkafka_lz4.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_lz4.h 2018-10-10 06:54:38.000000000 +0000
@@ -27,7 +27,8 @@
*/
-#pragma once
+#ifndef _RDKAFKA_LZ4_H_
+#define _RDKAFKA_LZ4_H_
rd_kafka_resp_err_t
@@ -36,5 +37,7 @@
void **outbuf, size_t *outlenp);
rd_kafka_resp_err_t
-rd_kafka_lz4_compress (rd_kafka_broker_t *rkb, int proper_hc,
+rd_kafka_lz4_compress (rd_kafka_broker_t *rkb, int proper_hc, int comp_level,
rd_slice_t *slice, void **outbuf, size_t *outlenp);
+
+#endif /* _RDKAFKA_LZ4_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_metadata.c librdkafka-0.11.6/src/rdkafka_metadata.c
--- librdkafka-0.11.3/src/rdkafka_metadata.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_metadata.c 2018-10-10 06:54:38.000000000 +0000
@@ -195,20 +195,23 @@
/**
- * Handle a Metadata response message.
+ * @brief Handle a Metadata response message.
*
* @param topics are the requested topics (may be NULL)
*
* The metadata will be marshalled into 'struct rd_kafka_metadata*' structs.
*
- * Returns the marshalled metadata, or NULL on parse error.
+ * The marshalled metadata is returned in \p *mdp, (NULL on error).
+
+ * @returns an error code on parse failure, else NO_ERRRO.
*
* @locality rdkafka main thread
*/
-struct rd_kafka_metadata *
+rd_kafka_resp_err_t
rd_kafka_parse_Metadata (rd_kafka_broker_t *rkb,
rd_kafka_buf_t *request,
- rd_kafka_buf_t *rkbuf) {
+ rd_kafka_buf_t *rkbuf,
+ struct rd_kafka_metadata **mdp) {
rd_kafka_t *rk = rkb->rkb_rk;
int i, j, k;
rd_tmpabuf_t tbuf;
@@ -223,6 +226,8 @@
int ApiVersion = request->rkbuf_reqhdr.ApiVersion;
rd_kafkap_str_t cluster_id = RD_ZERO_INIT;
int32_t controller_id = -1;
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ int broadcast_changes = 0;
rd_kafka_assert(NULL, thrd_is_current(rk->rk_thread));
@@ -239,8 +244,11 @@
sizeof(*md) + rkb_namelen + (rkbuf->rkbuf_totlen * 4),
0/*dont assert on fail*/);
- if (!(md = rd_tmpabuf_alloc(&tbuf, sizeof(*md))))
+ if (!(md = rd_tmpabuf_alloc(&tbuf, sizeof(*md)))) {
+ err = RD_KAFKA_RESP_ERR__CRIT_SYS_RESOURCE;
goto err;
+ }
+
md->orig_broker_id = rkb->rkb_nodeid;
md->orig_broker_name = rd_tmpabuf_write(&tbuf,
rkb->rkb_name, rkb_namelen);
@@ -401,18 +409,27 @@
partitions[j].isrs[k]);
}
+
+ /* Sort partitions by partition id */
+ qsort(md->topics[i].partitions,
+ md->topics[i].partition_cnt,
+ sizeof(*md->topics[i].partitions),
+ rd_kafka_metadata_partition_id_cmp);
}
/* Entire Metadata response now parsed without errors:
* update our internal state according to the response. */
/* Avoid metadata updates when we're terminating. */
- if (rd_kafka_terminating(rkb->rkb_rk))
+ if (rd_kafka_terminating(rkb->rkb_rk)) {
+ err = RD_KAFKA_RESP_ERR__DESTROY;
goto done;
+ }
if (md->broker_cnt == 0 && md->topic_cnt == 0) {
rd_rkb_dbg(rkb, METADATA, "METADATA",
- "No brokers or topics in metadata: retrying");
+ "No brokers or topics in metadata: should retry");
+ err = RD_KAFKA_RESP_ERR__PARTIAL;
goto err;
}
@@ -458,7 +475,9 @@
"topic %s (PartCnt %i): %s: ignoring",
mdt->topic, mdt->partition_cnt,
rd_kafka_err2str(mdt->err));
- rd_list_free_cb(missing_topics,
+ if (missing_topics)
+ rd_list_free_cb(
+ missing_topics,
rd_list_remove_cmp(missing_topics,
mdt->topic,
(void *)strcmp));
@@ -524,6 +543,15 @@
rkb->rkb_rk->rk_clusterid = RD_KAFKAP_STR_DUP(&cluster_id);
}
+ /* Update controller id. */
+ if (rkb->rkb_rk->rk_controllerid != controller_id) {
+ rd_rkb_dbg(rkb, BROKER, "CONTROLLERID",
+ "ControllerId update %"PRId32" -> %"PRId32,
+ rkb->rkb_rk->rk_controllerid, controller_id);
+ rkb->rkb_rk->rk_controllerid = controller_id;
+ broadcast_changes++;
+ }
+
if (all_topics) {
rd_kafka_metadata_cache_update(rkb->rkb_rk,
md, 1/*abs update*/);
@@ -547,6 +575,11 @@
rd_kafka_wrunlock(rkb->rkb_rk);
+ if (broadcast_changes) {
+ /* Broadcast metadata changes to listeners. */
+ rd_kafka_brokers_broadcast_state_change(rkb->rkb_rk);
+ }
+
/* Check if cgrp effective subscription is affected by
* new metadata. */
if (rkb->rkb_rk->rk_cgrp)
@@ -565,10 +598,13 @@
* the requestee will do.
* The tbuf is explicitly not destroyed as we return its memory
* to the caller. */
- return md;
+ *mdp = md;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
err_parse:
-err:
+ err = rkbuf->rkbuf_err;
+ err:
if (requested_topics) {
/* Failed requests shall purge cache hints for
* the requested topics. */
@@ -581,7 +617,8 @@
rd_list_destroy(missing_topics);
rd_tmpabuf_destroy(&tbuf);
- return NULL;
+
+ return err;
}
diff -Nru librdkafka-0.11.3/src/rdkafka_metadata_cache.c librdkafka-0.11.6/src/rdkafka_metadata_cache.c
--- librdkafka-0.11.3/src/rdkafka_metadata_cache.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_metadata_cache.c 2018-10-10 06:54:38.000000000 +0000
@@ -178,8 +178,8 @@
/**
* @brief Partition (id) comparator
*/
-static int rd_kafka_metadata_partition_id_cmp (const void *_a,
- const void *_b) {
+int rd_kafka_metadata_partition_id_cmp (const void *_a,
+ const void *_b) {
const rd_kafka_metadata_partition_t *a = _a, *b = _b;
return a->id - b->id;
}
diff -Nru librdkafka-0.11.3/src/rdkafka_metadata.h librdkafka-0.11.6/src/rdkafka_metadata.h
--- librdkafka-0.11.3/src/rdkafka_metadata.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_metadata.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,13 +26,15 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_METADATA_H_
+#define _RDKAFKA_METADATA_H_
#include "rdavl.h"
-struct rd_kafka_metadata *
+rd_kafka_resp_err_t
rd_kafka_parse_Metadata (rd_kafka_broker_t *rkb,
- rd_kafka_buf_t *request, rd_kafka_buf_t *rkbuf);
+ rd_kafka_buf_t *request, rd_kafka_buf_t *rkbuf,
+ struct rd_kafka_metadata **mdp);
struct rd_kafka_metadata *
rd_kafka_metadata_copy (const struct rd_kafka_metadata *md, size_t size);
@@ -69,6 +71,11 @@
const char *reason, rd_kafka_op_t *rko);
+
+int rd_kafka_metadata_partition_id_cmp (const void *_a,
+ const void *_b);
+
+
/**
* @{
*
@@ -155,3 +162,4 @@
void rd_kafka_metadata_cache_dump (FILE *fp, rd_kafka_t *rk);
/**@}*/
+#endif /* _RDKAFKA_METADATA_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_msg.c librdkafka-0.11.6/src/rdkafka_msg.c
--- librdkafka-0.11.3/src/rdkafka_msg.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_msg.c 2018-10-10 06:54:38.000000000 +0000
@@ -32,11 +32,13 @@
#include "rdkafka_topic.h"
#include "rdkafka_partition.h"
#include "rdkafka_interceptor.h"
+#include "rdkafka_header.h"
#include "rdcrc32.h"
+#include "rdmurmur2.h"
#include "rdrand.h"
#include "rdtime.h"
-
#include "rdsysqueue.h"
+#include "rdunittest.h"
#include
@@ -50,6 +52,9 @@
1, rkm->rkm_len);
}
+ if (rkm->rkm_headers)
+ rd_kafka_headers_destroy(rkm->rkm_headers);
+
if (likely(rkm->rkm_rkmessage.rkt != NULL))
rd_kafka_topic_destroy0(
rd_kafka_topic_a2s(rkm->rkm_rkmessage.rkt));
@@ -63,9 +68,9 @@
-
/**
- * @brief Create a new message, copying the payload as indicated by msgflags.
+ * @brief Create a new Producer message, copying the payload as
+ * indicated by msgflags.
*
* @returns the new message
*/
@@ -92,7 +97,8 @@
* are properly set up. */
rkm = rd_malloc(mlen);
rkm->rkm_err = 0;
- rkm->rkm_flags = RD_KAFKA_MSG_F_FREE_RKM | msgflags;
+ rkm->rkm_flags = (RD_KAFKA_MSG_F_PRODUCER |
+ RD_KAFKA_MSG_F_FREE_RKM | msgflags);
rkm->rkm_len = len;
rkm->rkm_opaque = msg_opaque;
rkm->rkm_rkmessage.rkt = rd_kafka_topic_keep_a(rkt);
@@ -101,6 +107,7 @@
rkm->rkm_offset = RD_KAFKA_OFFSET_INVALID;
rkm->rkm_timestamp = 0;
rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_NOT_AVAILABLE;
+ rkm->rkm_headers = NULL;
p = (char *)(rkm+1);
@@ -132,7 +139,7 @@
/**
- * @brief Create a new message.
+ * @brief Create a new Producer message.
*
* @remark Must only be used by producer code.
*
@@ -147,16 +154,20 @@
void *msg_opaque,
rd_kafka_resp_err_t *errp,
int *errnop,
+ rd_kafka_headers_t *hdrs,
int64_t timestamp,
rd_ts_t now) {
rd_kafka_msg_t *rkm;
+ size_t hdrs_size = 0;
if (unlikely(!payload))
len = 0;
if (!key)
keylen = 0;
+ if (hdrs)
+ hdrs_size = rd_kafka_headers_serialized_size(hdrs);
- if (unlikely(len + keylen >
+ if (unlikely(len + keylen + hdrs_size >
(size_t)rkt->rkt_rk->rk_conf.max_msg_size ||
keylen > INT32_MAX)) {
*errp = RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE;
@@ -165,9 +176,15 @@
return NULL;
}
- *errp = rd_kafka_curr_msgs_add(rkt->rkt_rk, 1, len,
- msgflags & RD_KAFKA_MSG_F_BLOCK);
- if (unlikely(*errp)) {
+ if (msgflags & RD_KAFKA_MSG_F_BLOCK)
+ *errp = rd_kafka_curr_msgs_add(
+ rkt->rkt_rk, 1, len, 1/*block*/,
+ (msgflags & RD_KAFKA_MSG_F_RKT_RDLOCKED) ?
+ &rkt->rkt_lock : NULL);
+ else
+ *errp = rd_kafka_curr_msgs_add(rkt->rkt_rk, 1, len, 0, NULL);
+
+ if (unlikely(*errp)) {
if (errnop)
*errnop = ENOBUFS;
return NULL;
@@ -178,12 +195,19 @@
msgflags|RD_KAFKA_MSG_F_ACCOUNT /* curr_msgs_add() */,
payload, len, key, keylen, msg_opaque);
+ memset(&rkm->rkm_u.producer, 0, sizeof(rkm->rkm_u.producer));
+
if (timestamp)
rkm->rkm_timestamp = timestamp;
else
rkm->rkm_timestamp = rd_uclock()/1000;
rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_CREATE_TIME;
+ if (hdrs) {
+ rd_dassert(!rkm->rkm_headers);
+ rkm->rkm_headers = hdrs;
+ }
+
rkm->rkm_ts_enq = now;
if (rkt->rkt_conf.message_timeout_ms == 0) {
@@ -222,9 +246,9 @@
int errnox;
/* Create message */
- rkm = rd_kafka_msg_new0(rkt, force_partition, msgflags,
+ rkm = rd_kafka_msg_new0(rkt, force_partition, msgflags,
payload, len, key, keylen, msg_opaque,
- &err, &errnox, 0, rd_clock());
+ &err, &errnox, NULL, 0, rd_clock());
if (unlikely(!rkm)) {
/* errno is already set by msg_new() */
rd_kafka_set_last_error(err, errnox);
@@ -278,9 +302,12 @@
shptr_rd_kafka_itopic_t *s_rkt = NULL;
rd_kafka_itopic_t *rkt;
rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ rd_kafka_headers_t *hdrs = NULL;
+ rd_kafka_headers_t *app_hdrs = NULL; /* App-provided headers list */
va_start(ap, rk);
- while ((vtype = va_arg(ap, rd_kafka_vtype_t)) != RD_KAFKA_VTYPE_END) {
+ while (!err &&
+ (vtype = va_arg(ap, rd_kafka_vtype_t)) != RD_KAFKA_VTYPE_END) {
switch (vtype)
{
case RD_KAFKA_VTYPE_TOPIC:
@@ -321,6 +348,36 @@
rkm->rkm_timestamp = va_arg(ap, int64_t);
break;
+ case RD_KAFKA_VTYPE_HEADER:
+ {
+ const char *name;
+ const void *value;
+ ssize_t size;
+
+ if (unlikely(app_hdrs != NULL)) {
+ err = RD_KAFKA_RESP_ERR__CONFLICT;
+ break;
+ }
+
+ if (unlikely(!hdrs))
+ hdrs = rd_kafka_headers_new(8);
+
+ name = va_arg(ap, const char *);
+ value = va_arg(ap, const void *);
+ size = va_arg(ap, ssize_t);
+
+ err = rd_kafka_header_add(hdrs, name, -1, value, size);
+ }
+ break;
+
+ case RD_KAFKA_VTYPE_HEADERS:
+ if (unlikely(hdrs != NULL)) {
+ err = RD_KAFKA_RESP_ERR__CONFLICT;
+ break;
+ }
+ app_hdrs = va_arg(ap, rd_kafka_headers_t *);
+ break;
+
default:
err = RD_KAFKA_RESP_ERR__INVALID_ARG;
break;
@@ -342,10 +399,14 @@
rkm->rkm_key, rkm->rkm_key_len,
rkm->rkm_opaque,
&err, NULL,
- rkm->rkm_timestamp, rd_clock());
+ app_hdrs ? app_hdrs : hdrs,
+ rkm->rkm_timestamp,
+ rd_clock());
if (unlikely(err)) {
rd_kafka_topic_destroy0(s_rkt);
+ if (hdrs)
+ rd_kafka_headers_destroy(hdrs);
return err;
}
@@ -366,6 +427,12 @@
* failure. */
rkm->rkm_flags &= ~RD_KAFKA_MSG_F_FREE;
+ /* Deassociate application owned headers from message
+ * since headers remain in application ownership
+ * when producev() fails */
+ if (app_hdrs && app_hdrs == rkm->rkm_headers)
+ rkm->rkm_headers = NULL;
+
rd_kafka_msg_destroy(rk, rkm);
}
@@ -387,13 +454,26 @@
int64_t utc_now = rd_uclock() / 1000;
rd_ts_t now = rd_clock();
int good = 0;
+ int multiple_partitions = (partition == RD_KAFKA_PARTITION_UA ||
+ (msgflags & RD_KAFKA_MSG_F_PARTITION));
rd_kafka_resp_err_t all_err = 0;
rd_kafka_itopic_t *rkt = rd_kafka_topic_a2i(app_rkt);
+ rd_kafka_toppar_t *rktp = NULL;
+ shptr_rd_kafka_toppar_t *s_rktp = NULL;
- /* For partitioner; hold lock for entire run,
- * for one partition: only acquire when needed at the end. */
- if (partition == RD_KAFKA_PARTITION_UA)
- rd_kafka_topic_rdlock(rkt);
+ /* For multiple partitions; hold lock for entire run,
+ * for one partition: only acquire for now. */
+ rd_kafka_topic_rdlock(rkt);
+ if (!multiple_partitions) {
+ s_rktp = rd_kafka_toppar_get_avail(rkt, partition,
+ 1/*ua on miss*/, &all_err);
+ rktp = rd_kafka_toppar_s2i(s_rktp);
+ rd_kafka_topic_rdunlock(rkt);
+ } else {
+ /* Indicate to lower-level msg_new..() that rkt is locked
+ * so that they may unlock it momentarily if blocking. */
+ msgflags |= RD_KAFKA_MSG_F_RKT_RDLOCKED;
+ }
for (i = 0 ; i < message_cnt ; i++) {
rd_kafka_msg_t *rkm;
@@ -406,13 +486,15 @@
/* Create message */
rkm = rd_kafka_msg_new0(rkt,
- partition , msgflags,
+ (msgflags & RD_KAFKA_MSG_F_PARTITION) ?
+ rkmessages[i].partition : partition,
+ msgflags,
rkmessages[i].payload,
rkmessages[i].len,
rkmessages[i].key,
rkmessages[i].key_len,
rkmessages[i]._private,
- &rkmessages[i].err,
+ &rkmessages[i].err, NULL,
NULL, utc_now, now);
if (unlikely(!rkm)) {
if (rkmessages[i].err == RD_KAFKA_RESP_ERR__QUEUE_FULL)
@@ -420,14 +502,32 @@
continue;
}
- /* Two cases here:
- * partition==UA: run the partitioner (slow)
- * fixed partition: simply concatenate the queue to partit */
- if (partition == RD_KAFKA_PARTITION_UA) {
- /* Partition the message */
- rkmessages[i].err =
- rd_kafka_msg_partitioner(rkt, rkm,
- 0/*already locked*/);
+ /* Three cases here:
+ * partition==UA: run the partitioner (slow)
+ * RD_KAFKA_MSG_F_PARTITION: produce message to specified
+ * partition
+ * fixed partition: simply concatenate the queue
+ * to partit */
+ if (multiple_partitions) {
+ if (rkm->rkm_partition == RD_KAFKA_PARTITION_UA) {
+ /* Partition the message */
+ rkmessages[i].err =
+ rd_kafka_msg_partitioner(
+ rkt, rkm, 0/*already locked*/);
+ } else {
+ if (s_rktp == NULL ||
+ rkm->rkm_partition !=
+ rd_kafka_toppar_s2i(s_rktp)->
+ rktp_partition) {
+ if (s_rktp != NULL)
+ rd_kafka_toppar_destroy(s_rktp);
+ s_rktp = rd_kafka_toppar_get_avail(
+ rkt, rkm->rkm_partition,
+ 1/*ua on miss*/, &all_err);
+ }
+ rktp = rd_kafka_toppar_s2i(s_rktp);
+ rd_kafka_toppar_enq_msg(rktp, rkm);
+ }
if (unlikely(rkmessages[i].err)) {
/* Interceptors: Unroll on_send by on_ack.. */
@@ -440,35 +540,18 @@
} else {
- /* Single destination partition, enqueue message
- * on temporary queue for later queue concat. */
- rd_kafka_msgq_enq(&tmpq, rkm);
+ /* Single destination partition. */
+ rd_kafka_toppar_enq_msg(rktp, rkm);
}
rkmessages[i].err = RD_KAFKA_RESP_ERR_NO_ERROR;
good++;
}
-
-
- /* Specific partition */
- if (partition != RD_KAFKA_PARTITION_UA) {
- shptr_rd_kafka_toppar_t *s_rktp;
-
- rd_kafka_topic_rdlock(rkt);
-
- s_rktp = rd_kafka_toppar_get_avail(rkt, partition,
- 1/*ua on miss*/, &all_err);
- /* Concatenate tmpq onto partition queue. */
- if (likely(s_rktp != NULL)) {
- rd_kafka_toppar_t *rktp = rd_kafka_toppar_s2i(s_rktp);
- rd_atomic64_add(&rktp->rktp_c.msgs, good);
- rd_kafka_toppar_concat_msgq(rktp, &tmpq);
- rd_kafka_toppar_destroy(s_rktp);
- }
- }
-
- rd_kafka_topic_rdunlock(rkt);
+ if (multiple_partitions)
+ rd_kafka_topic_rdunlock(rkt);
+ if (s_rktp != NULL)
+ rd_kafka_toppar_destroy(s_rktp);
return good;
}
@@ -483,10 +566,11 @@
rd_kafka_msgq_t *timedout,
rd_ts_t now) {
rd_kafka_msg_t *rkm, *tmp;
- int cnt = rd_atomic32_get(&timedout->rkmq_msg_cnt);
+ int cnt = timedout->rkmq_msg_cnt;
/* Assume messages are added in time sequencial order */
TAILQ_FOREACH_SAFE(rkm, &rkmq->rkmq_msgs, rkm_link, tmp) {
+ /* FIXME: this is no longer true */
if (likely(rkm->rkm_ts_timeout > now))
break;
@@ -494,10 +578,49 @@
rd_kafka_msgq_enq(timedout, rkm);
}
- return rd_atomic32_get(&timedout->rkmq_msg_cnt) - cnt;
+ return timedout->rkmq_msg_cnt - cnt;
}
+static RD_INLINE int
+rd_kafka_msgq_enq_sorted0 (rd_kafka_msgq_t *rkmq,
+ rd_kafka_msg_t *rkm,
+ int (*order_cmp) (const void *, const void *)) {
+ TAILQ_INSERT_SORTED(&rkmq->rkmq_msgs, rkm, rd_kafka_msg_t *,
+ rkm_link, order_cmp);
+ rkmq->rkmq_msg_bytes += rkm->rkm_len+rkm->rkm_key_len;
+ return ++rkmq->rkmq_msg_cnt;
+}
+
+int rd_kafka_msgq_enq_sorted (const rd_kafka_itopic_t *rkt,
+ rd_kafka_msgq_t *rkmq,
+ rd_kafka_msg_t *rkm) {
+ rd_dassert(rkm->rkm_u.producer.msgseq != 0);
+ return rd_kafka_msgq_enq_sorted0(rkmq, rkm,
+ rkt->rkt_conf.msg_order_cmp);
+}
+
+/**
+ * @brief Find the insert position (i.e., the previous element)
+ * for message \p rkm.
+ *
+ * @returns the insert position element, or NULL if \p rkm should be
+ * added at head of queue.
+ */
+rd_kafka_msg_t *rd_kafka_msgq_find_pos (const rd_kafka_msgq_t *rkmq,
+ const rd_kafka_msg_t *rkm,
+ int (*cmp) (const void *,
+ const void *)) {
+ const rd_kafka_msg_t *curr, *last = NULL;
+
+ TAILQ_FOREACH(curr, &rkmq->rkmq_msgs, rkm_link) {
+ if (cmp(rkm, curr) < 0)
+ return (rd_kafka_msg_t *)last;
+ last = curr;
+ }
+
+ return (rd_kafka_msg_t *)last;
+}
@@ -542,6 +665,31 @@
msg_opaque);
}
+int32_t
+rd_kafka_msg_partitioner_murmur2 (const rd_kafka_topic_t *rkt,
+ const void *key, size_t keylen,
+ int32_t partition_cnt,
+ void *rkt_opaque,
+ void *msg_opaque) {
+ return (rd_murmur2(key, keylen) & 0x7fffffff) % partition_cnt;
+}
+
+int32_t rd_kafka_msg_partitioner_murmur2_random (const rd_kafka_topic_t *rkt,
+ const void *key, size_t keylen,
+ int32_t partition_cnt,
+ void *rkt_opaque,
+ void *msg_opaque) {
+ if (!key)
+ return rd_kafka_msg_partitioner_random(rkt,
+ key,
+ keylen,
+ partition_cnt,
+ rkt_opaque,
+ msg_opaque);
+ else
+ return (rd_murmur2(key, keylen) & 0x7fffffff) % partition_cnt;
+}
+
/**
* Assigns a message to a topic partition using a partitioner.
@@ -637,7 +785,7 @@
}
rktp_new = rd_kafka_toppar_s2i(s_rktp_new);
- rd_atomic64_add(&rktp_new->rktp_c.msgs, 1);
+ rd_atomic64_add(&rktp_new->rktp_c.producer_enq_msgs, 1);
/* Update message partition */
if (rkm->rkm_partition == RD_KAFKA_PARTITION_UA)
@@ -757,6 +905,8 @@
case RD_KAFKA_OP_CONSUMER_ERR:
rkmessage = &rko->rko_u.err.rkm.rkm_rkmessage;
rkmessage->payload = rko->rko_u.err.errstr;
+ rkmessage->len = rkmessage->payload ?
+ strlen(rkmessage->payload) : 0;
rkmessage->offset = rko->rko_u.err.offset;
break;
@@ -800,3 +950,330 @@
return rd_clock() - rkm->rkm_ts_enq;
}
+
+
+/**
+ * @brief Parse serialized message headers and populate
+ * rkm->rkm_headers (which must be NULL).
+ */
+static rd_kafka_resp_err_t rd_kafka_msg_headers_parse (rd_kafka_msg_t *rkm) {
+ rd_kafka_buf_t *rkbuf;
+ int64_t HeaderCount;
+ const int log_decode_errors = 0;
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR__BAD_MSG;
+ int i;
+ rd_kafka_headers_t *hdrs = NULL;
+
+ rd_dassert(!rkm->rkm_headers);
+
+ if (RD_KAFKAP_BYTES_LEN(&rkm->rkm_u.consumer.binhdrs) == 0)
+ return RD_KAFKA_RESP_ERR__NOENT;
+
+ rkbuf = rd_kafka_buf_new_shadow(rkm->rkm_u.consumer.binhdrs.data,
+ RD_KAFKAP_BYTES_LEN(&rkm->rkm_u.
+ consumer.binhdrs),
+ NULL);
+
+ rd_kafka_buf_read_varint(rkbuf, &HeaderCount);
+
+ if (HeaderCount <= 0) {
+ rd_kafka_buf_destroy(rkbuf);
+ return RD_KAFKA_RESP_ERR__NOENT;
+ } else if (unlikely(HeaderCount > 100000)) {
+ rd_kafka_buf_destroy(rkbuf);
+ return RD_KAFKA_RESP_ERR__BAD_MSG;
+ }
+
+ hdrs = rd_kafka_headers_new((size_t)HeaderCount);
+
+ for (i = 0 ; (int64_t)i < HeaderCount ; i++) {
+ int64_t KeyLen, ValueLen;
+ const char *Key, *Value;
+
+ rd_kafka_buf_read_varint(rkbuf, &KeyLen);
+ rd_kafka_buf_read_ptr(rkbuf, &Key, (size_t)KeyLen);
+
+ rd_kafka_buf_read_varint(rkbuf, &ValueLen);
+ if (unlikely(ValueLen == -1))
+ Value = NULL;
+ else
+ rd_kafka_buf_read_ptr(rkbuf, &Value, (size_t)ValueLen);
+
+ rd_kafka_header_add(hdrs, Key, (ssize_t)KeyLen,
+ Value, (ssize_t)ValueLen);
+ }
+
+ rkm->rkm_headers = hdrs;
+
+ rd_kafka_buf_destroy(rkbuf);
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ err_parse:
+ err = rkbuf->rkbuf_err;
+ rd_kafka_buf_destroy(rkbuf);
+ if (hdrs)
+ rd_kafka_headers_destroy(hdrs);
+ return err;
+}
+
+
+
+
+rd_kafka_resp_err_t
+rd_kafka_message_headers (const rd_kafka_message_t *rkmessage,
+ rd_kafka_headers_t **hdrsp) {
+ rd_kafka_msg_t *rkm;
+ rd_kafka_resp_err_t err;
+
+ rkm = rd_kafka_message2msg((rd_kafka_message_t *)rkmessage);
+
+ if (rkm->rkm_headers) {
+ *hdrsp = rkm->rkm_headers;
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+
+ /* Producer (rkm_headers will be set if there were any headers) */
+ if (rkm->rkm_flags & RD_KAFKA_MSG_F_PRODUCER)
+ return RD_KAFKA_RESP_ERR__NOENT;
+
+ /* Consumer */
+
+ /* No previously parsed headers, check if the underlying
+ * protocol message had headers and if so, parse them. */
+ if (unlikely(!RD_KAFKAP_BYTES_LEN(&rkm->rkm_u.consumer.binhdrs)))
+ return RD_KAFKA_RESP_ERR__NOENT;
+
+ err = rd_kafka_msg_headers_parse(rkm);
+ if (unlikely(err))
+ return err;
+
+ *hdrsp = rkm->rkm_headers;
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+rd_kafka_resp_err_t
+rd_kafka_message_detach_headers (rd_kafka_message_t *rkmessage,
+ rd_kafka_headers_t **hdrsp) {
+ rd_kafka_msg_t *rkm;
+ rd_kafka_resp_err_t err;
+
+ err = rd_kafka_message_headers(rkmessage, hdrsp);
+ if (err)
+ return err;
+
+ rkm = rd_kafka_message2msg((rd_kafka_message_t *)rkmessage);
+ rkm->rkm_headers = NULL;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+void rd_kafka_message_set_headers (rd_kafka_message_t *rkmessage,
+ rd_kafka_headers_t *hdrs) {
+ rd_kafka_msg_t *rkm;
+
+ rkm = rd_kafka_message2msg((rd_kafka_message_t *)rkmessage);
+
+ if (rkm->rkm_headers) {
+ assert(rkm->rkm_headers != hdrs);
+ rd_kafka_headers_destroy(rkm->rkm_headers);
+ }
+
+ rkm->rkm_headers = hdrs;
+}
+
+
+
+void rd_kafka_msgq_dump (FILE *fp, const char *what, rd_kafka_msgq_t *rkmq) {
+ rd_kafka_msg_t *rkm;
+
+ fprintf(fp, "%s msgq_dump (%d messages, %"PRIusz" bytes):\n", what,
+ rd_kafka_msgq_len(rkmq), rd_kafka_msgq_size(rkmq));
+ TAILQ_FOREACH(rkm, &rkmq->rkmq_msgs, rkm_link) {
+ fprintf(fp, " [%"PRId32"]@%"PRId64
+ ": rkm msgseq %"PRIu64": \"%.*s\"\n",
+ rkm->rkm_partition, rkm->rkm_offset,
+ rkm->rkm_u.producer.msgseq,
+ (int)rkm->rkm_len, (const char *)rkm->rkm_payload);
+ }
+}
+
+/**
+ * @name Unit tests
+ */
+
+/**
+ * @brief Unittest: message allocator
+ */
+static rd_kafka_msg_t *ut_rd_kafka_msg_new (void) {
+ rd_kafka_msg_t *rkm;
+
+ rkm = rd_calloc(1, sizeof(*rkm));
+ rkm->rkm_flags = RD_KAFKA_MSG_F_FREE_RKM;
+ rkm->rkm_offset = RD_KAFKA_OFFSET_INVALID;
+ rkm->rkm_tstype = RD_KAFKA_TIMESTAMP_NOT_AVAILABLE;
+
+ return rkm;
+}
+
+
+
+/**
+ * @brief Unittest: destroy all messages in queue
+ */
+static void ut_rd_kafka_msgq_purge (rd_kafka_msgq_t *rkmq) {
+ rd_kafka_msg_t *rkm, *tmp;
+
+ TAILQ_FOREACH_SAFE(rkm, &rkmq->rkmq_msgs, rkm_link, tmp)
+ rd_kafka_msg_destroy(NULL, rkm);
+
+
+ rd_kafka_msgq_init(rkmq);
+}
+
+
+
+static int ut_verify_msgq_order (const char *what,
+ const rd_kafka_msgq_t *rkmq,
+ int first, int last) {
+ const rd_kafka_msg_t *rkm;
+ uint64_t expected = first;
+ int incr = first < last ? +1 : -1;
+ int fails = 0;
+ int cnt = 0;
+
+ TAILQ_FOREACH(rkm, &rkmq->rkmq_msgs, rkm_link) {
+ if (rkm->rkm_u.producer.msgseq != expected) {
+ RD_UT_SAY("%s: expected msgseq %"PRIu64
+ " not %"PRIu64" at index #%d",
+ what, expected,
+ rkm->rkm_u.producer.msgseq, cnt);
+ fails++;
+ }
+ cnt++;
+ expected += incr;
+ }
+
+ RD_UT_ASSERT(!fails, "See %d previous failure(s)", fails);
+ return fails;
+}
+
+/**
+ * @brief Verify ordering comparator for message queues.
+ */
+static int unittest_msgq_order (const char *what, int fifo,
+ int (*cmp) (const void *, const void *)) {
+ rd_kafka_msgq_t rkmq = RD_KAFKA_MSGQ_INITIALIZER(rkmq);
+ rd_kafka_msg_t *rkm;
+ rd_kafka_msgq_t sendq;
+ int i;
+
+ RD_UT_SAY("%s: testing in %s mode", what, fifo? "FIFO" : "LIFO");
+
+ for (i = 1 ; i <= 6 ; i++) {
+ rkm = ut_rd_kafka_msg_new();
+ rkm->rkm_u.producer.msgseq = i;
+ rd_kafka_msgq_enq_sorted0(&rkmq, rkm, cmp);
+ }
+
+ if (fifo) {
+ if (ut_verify_msgq_order("added", &rkmq, 1, 6))
+ return 1;
+ } else {
+ if (ut_verify_msgq_order("added", &rkmq, 6, 1))
+ return 1;
+ }
+
+ /* Move 3 messages to "send" queue which we then re-insert
+ * in the original queue (i.e., "retry"). */
+ rd_kafka_msgq_init(&sendq);
+ while (rd_kafka_msgq_len(&sendq) < 3)
+ rd_kafka_msgq_enq(&sendq, rd_kafka_msgq_pop(&rkmq));
+
+ if (fifo) {
+ if (ut_verify_msgq_order("send removed", &rkmq, 4, 6))
+ return 1;
+
+ if (ut_verify_msgq_order("sendq", &sendq, 1, 3))
+ return 1;
+ } else {
+ if (ut_verify_msgq_order("send removed", &rkmq, 3, 1))
+ return 1;
+
+ if (ut_verify_msgq_order("sendq", &sendq, 6, 4))
+ return 1;
+ }
+
+ /* Retry the messages, which moves them back to sendq
+ * maintaining the original order */
+ rd_kafka_retry_msgq(&rkmq, &sendq, 1, 1, 0, cmp);
+
+ RD_UT_ASSERT(rd_kafka_msgq_len(&sendq) == 0,
+ "sendq FIFO should be empty, not contain %d messages",
+ rd_kafka_msgq_len(&sendq));
+
+ if (fifo) {
+ if (ut_verify_msgq_order("readded", &rkmq, 1, 6))
+ return 1;
+ } else {
+ if (ut_verify_msgq_order("readded", &rkmq, 6, 1))
+ return 1;
+ }
+
+ /* Move 4 first messages to to "send" queue, then
+ * retry them with max_retries=1 which should now fail for
+ * the 3 first messages that were already retried. */
+ rd_kafka_msgq_init(&sendq);
+ while (rd_kafka_msgq_len(&sendq) < 4)
+ rd_kafka_msgq_enq(&sendq, rd_kafka_msgq_pop(&rkmq));
+
+ if (fifo) {
+ if (ut_verify_msgq_order("send removed #2", &rkmq, 5, 6))
+ return 1;
+
+ if (ut_verify_msgq_order("sendq #2", &sendq, 1, 4))
+ return 1;
+ } else {
+ if (ut_verify_msgq_order("send removed #2", &rkmq, 2, 1))
+ return 1;
+
+ if (ut_verify_msgq_order("sendq #2", &sendq, 6, 3))
+ return 1;
+ }
+
+ /* Retry the messages, which should now keep the 3 first messages
+ * on sendq (no more retries) and just number 4 moved back. */
+ rd_kafka_retry_msgq(&rkmq, &sendq, 1, 1, 0, cmp);
+
+ if (fifo) {
+ if (ut_verify_msgq_order("readded #2", &rkmq, 4, 6))
+ return 1;
+
+ if (ut_verify_msgq_order("no more retries", &sendq, 1, 3))
+ return 1;
+
+ } else {
+ if (ut_verify_msgq_order("readded #2", &rkmq, 3, 1))
+ return 1;
+
+ if (ut_verify_msgq_order("no more retries", &sendq, 6, 4))
+ return 1;
+ }
+
+ ut_rd_kafka_msgq_purge(&sendq);
+ ut_rd_kafka_msgq_purge(&rkmq);
+
+ return 0;
+
+}
+
+
+int unittest_msg (void) {
+ int fails = 0;
+
+ fails += unittest_msgq_order("FIFO", 1, rd_kafka_msg_cmp_msgseq);
+ fails += unittest_msgq_order("LIFO", 0, rd_kafka_msg_cmp_msgseq_lifo);
+
+ return fails;
+}
diff -Nru librdkafka-0.11.3/src/rdkafka_msg.h librdkafka-0.11.6/src/rdkafka_msg.h
--- librdkafka-0.11.3/src/rdkafka_msg.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_msg.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,11 +26,19 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_MSG_H_
+#define _RDKAFKA_MSG_H_
#include "rdsysqueue.h"
#include "rdkafka_proto.h"
+#include "rdkafka_header.h"
+
+
+/**
+ * @brief Internal RD_KAFKA_MSG_F_.. flags
+ */
+#define RD_KAFKA_MSG_F_RKT_RDLOCKED 0x100000 /* rkt is rdlock():ed */
/**
@@ -77,6 +85,7 @@
* the RD_KAFKA_MSG_F_* flags in rdkafka.h */
#define RD_KAFKA_MSG_F_FREE_RKM 0x10000 /* msg_t is allocated */
#define RD_KAFKA_MSG_F_ACCOUNT 0x20000 /* accounted for in curr_msgs */
+#define RD_KAFKA_MSG_F_PRODUCER 0x40000 /* Producer message */
int64_t rkm_timestamp; /* Message format V1.
* Meaning of timestamp depends on
@@ -85,13 +94,26 @@
* Unit is milliseconds since epoch (UTC).*/
rd_kafka_timestamp_type_t rkm_tstype; /* rkm_timestamp type */
+ rd_kafka_headers_t *rkm_headers; /**< Parsed headers list, if any. */
+
union {
struct {
rd_ts_t ts_timeout; /* Message timeout */
rd_ts_t ts_enq; /* Enqueue/Produce time */
+ rd_ts_t ts_backoff; /* Backoff next Produce until
+ * this time. */
+ uint64_t msgseq; /* Message sequence number,
+ * used to maintain ordering. */
+ int retries; /* Number of retries so far */
} producer;
#define rkm_ts_timeout rkm_u.producer.ts_timeout
#define rkm_ts_enq rkm_u.producer.ts_enq
+
+ struct {
+ rd_kafkap_bytes_t binhdrs; /**< Unparsed
+ * binary headers in
+ * protocol msg */
+ } consumer;
} rkm_u;
} rd_kafka_msg_t;
@@ -113,8 +135,14 @@
[1] = RD_KAFKAP_MESSAGE_V1_OVERHEAD,
[2] = RD_KAFKAP_MESSAGE_V2_OVERHEAD
};
+ size_t size;
rd_dassert(MsgVersion >= 0 && MsgVersion <= 2);
- return overheads[MsgVersion] + rkm->rkm_len + rkm->rkm_key_len;
+
+ size = overheads[MsgVersion] + rkm->rkm_len + rkm->rkm_key_len;
+ if (MsgVersion == 2 && rkm->rkm_headers)
+ size += rd_kafka_headers_serialized_size(rkm->rkm_headers);
+
+ return size;
}
@@ -131,10 +159,14 @@
+/**
+ * @brief Message queue with message and byte counters.
+ */
+TAILQ_HEAD(rd_kafka_msgs_head_s, rd_kafka_msg_s);
typedef struct rd_kafka_msgq_s {
- TAILQ_HEAD(, rd_kafka_msg_s) rkmq_msgs;
- rd_atomic32_t rkmq_msg_cnt;
- rd_atomic64_t rkmq_msg_bytes;
+ struct rd_kafka_msgs_head_s rkmq_msgs; /* TAILQ_HEAD */
+ int32_t rkmq_msg_cnt;
+ int64_t rkmq_msg_bytes;
} rd_kafka_msgq_t;
#define RD_KAFKA_MSGQ_INITIALIZER(rkmq) \
@@ -143,18 +175,21 @@
#define RD_KAFKA_MSGQ_FOREACH(elm,head) \
TAILQ_FOREACH(elm, &(head)->rkmq_msgs, rkm_link)
+/* @brief Check if queue is empty. Proper locks must be held. */
+#define RD_KAFKA_MSGQ_EMPTY(rkmq) TAILQ_EMPTY(&(rkmq)->rkmq_msgs)
+
/**
* Returns the number of messages in the specified queue.
*/
static RD_INLINE RD_UNUSED int rd_kafka_msgq_len (rd_kafka_msgq_t *rkmq) {
- return (int)rd_atomic32_get(&rkmq->rkmq_msg_cnt);
+ return (int)rkmq->rkmq_msg_cnt;
}
/**
* Returns the total number of bytes in the specified queue.
*/
static RD_INLINE RD_UNUSED size_t rd_kafka_msgq_size (rd_kafka_msgq_t *rkmq) {
- return (size_t)rd_atomic64_get(&rkmq->rkmq_msg_bytes);
+ return (size_t)rkmq->rkmq_msg_bytes;
}
@@ -167,9 +202,9 @@
void *msg_opaque);
static RD_INLINE RD_UNUSED void rd_kafka_msgq_init (rd_kafka_msgq_t *rkmq) {
- TAILQ_INIT(&rkmq->rkmq_msgs);
- rd_atomic32_init(&rkmq->rkmq_msg_cnt, 0);
- rd_atomic64_init(&rkmq->rkmq_msg_bytes, 0);
+ TAILQ_INIT(&rkmq->rkmq_msgs);
+ rkmq->rkmq_msg_cnt = 0;
+ rkmq->rkmq_msg_bytes = 0;
}
/**
@@ -180,8 +215,8 @@
static RD_INLINE RD_UNUSED void rd_kafka_msgq_concat (rd_kafka_msgq_t *dst,
rd_kafka_msgq_t *src) {
TAILQ_CONCAT(&dst->rkmq_msgs, &src->rkmq_msgs, rkm_link);
- rd_atomic32_add(&dst->rkmq_msg_cnt, rd_atomic32_get(&src->rkmq_msg_cnt));
- rd_atomic64_add(&dst->rkmq_msg_bytes, rd_atomic64_get(&src->rkmq_msg_bytes));
+ dst->rkmq_msg_cnt += src->rkmq_msg_cnt;
+ dst->rkmq_msg_bytes += src->rkmq_msg_bytes;
rd_kafka_msgq_init(src);
}
@@ -192,8 +227,8 @@
static RD_INLINE RD_UNUSED void rd_kafka_msgq_move (rd_kafka_msgq_t *dst,
rd_kafka_msgq_t *src) {
TAILQ_MOVE(&dst->rkmq_msgs, &src->rkmq_msgs, rkm_link);
- rd_atomic32_set(&dst->rkmq_msg_cnt, rd_atomic32_get(&src->rkmq_msg_cnt));
- rd_atomic64_set(&dst->rkmq_msg_bytes, rd_atomic64_get(&src->rkmq_msg_bytes));
+ dst->rkmq_msg_cnt = src->rkmq_msg_cnt;
+ dst->rkmq_msg_bytes = src->rkmq_msg_bytes;
rd_kafka_msgq_init(src);
}
@@ -225,11 +260,11 @@
rd_kafka_msg_t *rkm,
int do_count) {
if (likely(do_count)) {
- rd_kafka_assert(NULL, rd_atomic32_get(&rkmq->rkmq_msg_cnt) > 0);
- rd_kafka_assert(NULL, rd_atomic64_get(&rkmq->rkmq_msg_bytes) >= (int64_t)(rkm->rkm_len+rkm->rkm_key_len));
- rd_atomic32_sub(&rkmq->rkmq_msg_cnt, 1);
- rd_atomic64_sub(&rkmq->rkmq_msg_bytes,
- rkm->rkm_len+rkm->rkm_key_len);
+ rd_kafka_assert(NULL, rkmq->rkmq_msg_cnt > 0);
+ rd_kafka_assert(NULL, rkmq->rkmq_msg_bytes >=
+ (int64_t)(rkm->rkm_len+rkm->rkm_key_len));
+ rkmq->rkmq_msg_cnt--;
+ rkmq->rkmq_msg_bytes -= rkm->rkm_len+rkm->rkm_key_len;
}
TAILQ_REMOVE(&rkmq->rkmq_msgs, rkm, rkm_link);
@@ -247,24 +282,71 @@
return rkm;
}
+
+/**
+ * @brief Message ordering comparator using the message sequence
+ * number to order messages in ascending order (FIFO).
+ */
+static RD_INLINE
+int rd_kafka_msg_cmp_msgseq (const void *_a, const void *_b) {
+ const rd_kafka_msg_t *a = _a, *b = _b;
+
+ rd_dassert(a->rkm_u.producer.msgseq);
+
+ if (a->rkm_u.producer.msgseq > b->rkm_u.producer.msgseq)
+ return 1;
+ else if (a->rkm_u.producer.msgseq < b->rkm_u.producer.msgseq)
+ return -1;
+ else
+ return 0;
+}
+
+/**
+ * @brief Message ordering comparator using the message sequence
+ * number to order messages in descending order (LIFO).
+ */
+static RD_INLINE
+int rd_kafka_msg_cmp_msgseq_lifo (const void *_a, const void *_b) {
+ const rd_kafka_msg_t *a = _a, *b = _b;
+
+ rd_dassert(a->rkm_u.producer.msgseq);
+
+ if (a->rkm_u.producer.msgseq < b->rkm_u.producer.msgseq)
+ return 1;
+ else if (a->rkm_u.producer.msgseq > b->rkm_u.producer.msgseq)
+ return -1;
+ else
+ return 0;
+}
+
+/**
+ * @brief Insert message at its sorted position using the msgseq.
+ * @remark This is an O(n) operation.
+ * @warning The message must have a msgseq set.
+ * @returns the message count of the queue after enqueuing the message.
+ */
+int rd_kafka_msgq_enq_sorted (const rd_kafka_itopic_t *rkt,
+ rd_kafka_msgq_t *rkmq,
+ rd_kafka_msg_t *rkm);
+
/**
* Insert message at head of message queue.
*/
static RD_INLINE RD_UNUSED void rd_kafka_msgq_insert (rd_kafka_msgq_t *rkmq,
rd_kafka_msg_t *rkm) {
TAILQ_INSERT_HEAD(&rkmq->rkmq_msgs, rkm, rkm_link);
- rd_atomic32_add(&rkmq->rkmq_msg_cnt, 1);
- rd_atomic64_add(&rkmq->rkmq_msg_bytes, rkm->rkm_len+rkm->rkm_key_len);
+ rkmq->rkmq_msg_cnt++;
+ rkmq->rkmq_msg_bytes += rkm->rkm_len+rkm->rkm_key_len;
}
/**
* Append message to tail of message queue.
*/
-static RD_INLINE RD_UNUSED void rd_kafka_msgq_enq (rd_kafka_msgq_t *rkmq,
- rd_kafka_msg_t *rkm) {
- TAILQ_INSERT_TAIL(&rkmq->rkmq_msgs, rkm, rkm_link);
- rd_atomic32_add(&rkmq->rkmq_msg_cnt, 1);
- rd_atomic64_add(&rkmq->rkmq_msg_bytes, rkm->rkm_len+rkm->rkm_key_len);
+static RD_INLINE RD_UNUSED int rd_kafka_msgq_enq (rd_kafka_msgq_t *rkmq,
+ rd_kafka_msg_t *rkm) {
+ TAILQ_INSERT_TAIL(&rkmq->rkmq_msgs, rkm, rkm_link);
+ rkmq->rkmq_msg_bytes += rkm->rkm_len+rkm->rkm_key_len;
+ return (int)++rkmq->rkmq_msg_cnt;
}
@@ -278,6 +360,10 @@
rd_kafka_msgq_t *timedout,
rd_ts_t now);
+rd_kafka_msg_t *rd_kafka_msgq_find_pos (const rd_kafka_msgq_t *rkmq,
+ const rd_kafka_msg_t *rkm,
+ int (*cmp) (const void *,
+ const void *));
int rd_kafka_msg_partitioner (rd_kafka_itopic_t *rkt, rd_kafka_msg_t *rkm,
int do_lock);
@@ -288,3 +374,8 @@
rd_kafka_msg_t *rkm);
rd_kafka_message_t *rd_kafka_message_new (void);
+void rd_kafka_msgq_dump (FILE *fp, const char *what, rd_kafka_msgq_t *rkmq);
+
+int unittest_msg (void);
+
+#endif /* _RDKAFKA_MSG_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_msgset.h librdkafka-0.11.6/src/rdkafka_msgset.h
--- librdkafka-0.11.3/src/rdkafka_msgset.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_msgset.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_MSGSET_H_
+#define _RDKAFKA_MSGSET_H_
/**
@@ -45,3 +46,5 @@
rd_kafka_buf_t *request,
rd_kafka_toppar_t *rktp,
const struct rd_kafka_toppar_ver *tver);
+
+#endif /* _RDKAFKA_MSGSET_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_msgset_reader.c librdkafka-0.11.6/src/rdkafka_msgset_reader.c
--- librdkafka-0.11.3/src/rdkafka_msgset_reader.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_msgset_reader.c 2018-10-10 06:54:38.000000000 +0000
@@ -59,6 +59,7 @@
#include "rdkafka_msgset.h"
#include "rdkafka_topic.h"
#include "rdkafka_partition.h"
+#include "rdkafka_header.h"
#include "rdkafka_lz4.h"
#include "rdvarint.h"
@@ -116,12 +117,32 @@
* reference! */
int msetr_msgcnt; /**< Number of messages in rkq */
+ int64_t msetr_msg_bytes; /**< Number of bytes in rkq */
rd_kafka_q_t msetr_rkq; /**< Temp Message and error queue */
rd_kafka_q_t *msetr_par_rkq; /**< Parent message and error queue,
* the temp msetr_rkq will be moved
* to this queue when parsing
* is done.
* Refcount is not increased. */
+
+ int64_t msetr_next_offset; /**< Next offset to fetch after
+ * this reader run is done.
+ * Optional: only used for special
+ * cases where the per-message offset
+ * can't be relied on for next
+ * fetch offset, such as with
+ * compacted topics. */
+
+ int msetr_ctrl_cnt; /**< Number of control messages
+ * or MessageSets received. */
+
+ const char *msetr_srcname; /**< Optional message source string,
+ * used in debug logging to
+ * indicate messages were
+ * from an inner compressed
+ * message set.
+ * Not freed (use const memory).
+ * Add trailing space. */
} rd_kafka_msgset_reader_t;
@@ -149,6 +170,9 @@
msetr->msetr_rktp = rktp;
msetr->msetr_tver = tver;
msetr->msetr_rkbuf = rkbuf;
+ msetr->msetr_srcname = "";
+
+ rkbuf->rkbuf_uflow_mitigation = "truncated response from broker (ok)";
/* All parsed messages are put on this temporary op
* queue first and then moved in one go to the real op queue. */
@@ -366,6 +390,8 @@
msetr->msetr_tver,
&msetr->msetr_rkq);
+ inner_msetr.msetr_srcname = "compressed ";
+
if (MsgVersion == 1) {
/* postproc() will convert relative to
* absolute offsets */
@@ -384,11 +410,18 @@
/* Parse the inner MessageSet */
err = rd_kafka_msgset_reader_run(&inner_msetr);
+ /* Transfer message count from inner to outer */
+ msetr->msetr_msgcnt += inner_msetr.msetr_msgcnt;
+ msetr->msetr_msg_bytes += inner_msetr.msetr_msg_bytes;
+
} else {
/* MsgVersion 2 */
rd_kafka_buf_t *orig_rkbuf = msetr->msetr_rkbuf;
+ rkbufz->rkbuf_uflow_mitigation =
+ "truncated response from broker (ok)";
+
/* Temporarily replace read buffer with uncompressed buffer */
msetr->msetr_rkbuf = rkbufz;
@@ -587,6 +620,7 @@
/* Enqueue message on temporary queue */
rd_kafka_q_enq(&msetr->msetr_rkq, rko);
msetr->msetr_msgcnt++;
+ msetr->msetr_msg_bytes += rkm->rkm_key_len + rkm->rkm_len;
return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue */
@@ -608,13 +642,13 @@
rd_kafka_toppar_t *rktp = msetr->msetr_rktp;
struct {
int64_t Length;
- int64_t MsgAttributes; /* int8_t, but int64 req. for varint */
+ int8_t MsgAttributes;
int64_t TimestampDelta;
int64_t OffsetDelta;
int64_t Offset; /* Absolute offset */
rd_kafkap_bytes_t Key;
rd_kafkap_bytes_t Value;
- int64_t HeaderCnt;
+ rd_kafkap_bytes_t Headers;
} hdr;
rd_kafka_op_t *rko;
rd_kafka_msg_t *rkm;
@@ -625,7 +659,7 @@
rd_kafka_buf_read_varint(rkbuf, &hdr.Length);
message_end = rd_slice_offset(&rkbuf->rkbuf_reader)+(size_t)hdr.Length;
- rd_kafka_buf_read_varint(rkbuf, &hdr.MsgAttributes);
+ rd_kafka_buf_read_i8(rkbuf, &hdr.MsgAttributes);
rd_kafka_buf_read_varint(rkbuf, &hdr.TimestampDelta);
rd_kafka_buf_read_varint(rkbuf, &hdr.OffsetDelta);
@@ -634,7 +668,10 @@
/* Skip message if outdated */
if (hdr.Offset < rktp->rktp_offsets.fetch_offset) {
rd_rkb_dbg(msetr->msetr_rkb, MSG, "MSG",
+ "%s [%"PRId32"]: "
"Skip offset %"PRId64" < fetch_offset %"PRId64,
+ rktp->rktp_rkt->rkt_topic->str,
+ rktp->rktp_partition,
hdr.Offset, rktp->rktp_offsets.fetch_offset);
rd_kafka_buf_skip_to(rkbuf, message_end);
return RD_KAFKA_RESP_ERR_NO_ERROR; /* Continue with next msg */
@@ -644,8 +681,11 @@
rd_kafka_buf_read_bytes_varint(rkbuf, &hdr.Value);
- /* Ignore headers for now */
- rd_kafka_buf_skip_to(rkbuf, message_end);
+ /* We parse the Headers later, just store the size (possibly truncated)
+ * and pointer to the headers. */
+ hdr.Headers.len = (int32_t)(message_end -
+ rd_slice_offset(&rkbuf->rkbuf_reader));
+ rd_kafka_buf_read_ptr(rkbuf, &hdr.Headers.data, hdr.Headers.len);
/* Create op/message container for message. */
rko = rd_kafka_op_new_fetch_msg(&rkm,
@@ -658,6 +698,13 @@
RD_KAFKAP_BYTES_IS_NULL(&hdr.Value) ?
NULL : hdr.Value.data);
+ /* Store pointer to unparsed message headers, they will
+ * be parsed on the first access.
+ * This pointer points to the rkbuf payload.
+ * Note: can't perform struct copy here due to const fields (MSVC) */
+ rkm->rkm_u.consumer.binhdrs.len = hdr.Headers.len;
+ rkm->rkm_u.consumer.binhdrs.data = hdr.Headers.data;
+
/* Set timestamp.
*
* When broker assigns the timestamps (LOG_APPEND_TIME) it will
@@ -679,6 +726,7 @@
/* Enqueue message on temporary queue */
rd_kafka_q_enq(&msetr->msetr_rkq, rko);
msetr->msetr_msgcnt++;
+ msetr->msetr_msg_bytes += rkm->rkm_key_len + rkm->rkm_len;
return RD_KAFKA_RESP_ERR_NO_ERROR;
@@ -791,15 +839,13 @@
len_start);
if (unlikely(payload_size > rd_kafka_buf_read_remain(rkbuf)))
- rd_kafka_buf_parse_fail(rkbuf,
- "%s [%"PRId32"] "
- "MessageSet at offset %"PRId64
- " payload size %"PRIusz
- " > %"PRIusz" remaining bytes",
- rktp->rktp_rkt->rkt_topic->str,
- rktp->rktp_partition,
- hdr.BaseOffset, payload_size,
- rd_kafka_buf_read_remain(rkbuf));
+ rd_kafka_buf_underflow_fail(rkbuf, payload_size,
+ "%s [%"PRId32"] "
+ "MessageSet at offset %"PRId64
+ " payload size %"PRIusz,
+ rktp->rktp_rkt->rkt_topic->str,
+ rktp->rktp_partition,
+ hdr.BaseOffset, payload_size);
/* If entire MessageSet contains old outdated offsets, skip it. */
if (LastOffset < rktp->rktp_offsets.fetch_offset) {
@@ -809,6 +855,7 @@
/* Ignore control messages */
if (unlikely((hdr.Attributes & RD_KAFKA_MSGSET_V2_ATTR_CONTROL))) {
+ msetr->msetr_ctrl_cnt++;
rd_kafka_buf_skip(rkbuf, payload_size);
goto done;
}
@@ -856,8 +903,7 @@
* to avoid getting stuck on compacted MessageSets where the last
* Message in the MessageSet has an Offset < MessageSet header's
* last offset. See KAFKA-5443 */
- if (likely(LastOffset >= msetr->msetr_rktp->rktp_offsets.fetch_offset))
- msetr->msetr_rktp->rktp_offsets.fetch_offset = LastOffset + 1;
+ msetr->msetr_next_offset = LastOffset + 1;
msetr->msetr_v2_hdr = NULL;
@@ -1000,8 +1046,13 @@
/* The message set didn't contain at least one full message
* or no error was posted on the response queue.
* This means the size limit perhaps was too tight,
- * increase it automatically. */
- if (rktp->rktp_fetch_msg_max_bytes < (1 << 30)) {
+ * increase it automatically.
+ * If there was at least one control message there
+ * is probably not a size limit and nothing is done. */
+ if (msetr->msetr_ctrl_cnt > 0) {
+ /* Noop */
+
+ } else if (rktp->rktp_fetch_msg_max_bytes < (1 << 30)) {
rktp->rktp_fetch_msg_max_bytes *= 2;
rd_rkb_dbg(msetr->msetr_rkb, FETCH, "CONSUME",
"Topic %s [%"PRId32"]: Increasing "
@@ -1030,31 +1081,38 @@
/* Ignore parse errors if there was at least one
* good message since it probably indicates a
* partial response rather than an erroneous one. */
- if (err == RD_KAFKA_RESP_ERR__BAD_MSG &&
+ if (err == RD_KAFKA_RESP_ERR__UNDERFLOW &&
msetr->msetr_msgcnt > 0)
err = RD_KAFKA_RESP_ERR_NO_ERROR;
}
rd_rkb_dbg(msetr->msetr_rkb, MSG | RD_KAFKA_DBG_FETCH, "CONSUME",
- "Enqueue %i message(s) (%d ops) on %s [%"PRId32"] "
- "fetch queue (qlen %d, v%d, last_offset %"PRId64")",
- msetr->msetr_msgcnt, rd_kafka_q_len(&msetr->msetr_rkq),
+ "Enqueue %i %smessage(s) (%"PRId64" bytes, %d ops) on "
+ "%s [%"PRId32"] "
+ "fetch queue (qlen %d, v%d, last_offset %"PRId64
+ ", %d ctrl msgs)",
+ msetr->msetr_msgcnt, msetr->msetr_srcname,
+ msetr->msetr_msg_bytes,
+ rd_kafka_q_len(&msetr->msetr_rkq),
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition, rd_kafka_q_len(&msetr->msetr_rkq),
- msetr->msetr_tver->version, last_offset);
+ msetr->msetr_tver->version, last_offset,
+ msetr->msetr_ctrl_cnt);
/* Concat all messages&errors onto the parent's queue
* (the partition's fetch queue) */
if (rd_kafka_q_concat(msetr->msetr_par_rkq, &msetr->msetr_rkq) != -1) {
/* Update partition's fetch offset based on
* last message's offest. */
- if (likely(last_offset != -1)) {
+ if (likely(last_offset != -1))
rktp->rktp_offsets.fetch_offset = last_offset + 1;
- rd_atomic64_add(&rktp->rktp_c.msgs,
- msetr->msetr_msgcnt);
- }
}
+ /* Adjust next fetch offset if outlier code has indicated
+ * an even later next offset. */
+ if (msetr->msetr_next_offset > rktp->rktp_offsets.fetch_offset)
+ rktp->rktp_offsets.fetch_offset = msetr->msetr_next_offset;
+
rd_kafka_q_destroy_owner(&msetr->msetr_rkq);
/* Skip remaining part of slice so caller can continue
@@ -1079,12 +1137,24 @@
rd_kafka_toppar_t *rktp,
const struct rd_kafka_toppar_ver *tver) {
rd_kafka_msgset_reader_t msetr;
+ rd_kafka_resp_err_t err;
rd_kafka_msgset_reader_init(&msetr, rkbuf, rktp, tver,
rktp->rktp_fetchq);
/* Parse and handle the message set */
- return rd_kafka_msgset_reader_run(&msetr);
+ err = rd_kafka_msgset_reader_run(&msetr);
+
+ rd_atomic64_add(&rktp->rktp_c.rx_msgs, msetr.msetr_msgcnt);
+ rd_atomic64_add(&rktp->rktp_c.rx_msg_bytes, msetr.msetr_msg_bytes);
+
+ rd_avg_add(&rktp->rktp_rkt->rkt_avg_batchcnt,
+ (int64_t)msetr.msetr_msgcnt);
+ rd_avg_add(&rktp->rktp_rkt->rkt_avg_batchsize,
+ (int64_t)msetr.msetr_msg_bytes);
+
+ return err;
+
}
diff -Nru librdkafka-0.11.3/src/rdkafka_msgset_writer.c librdkafka-0.11.6/src/rdkafka_msgset_writer.c
--- librdkafka-0.11.3/src/rdkafka_msgset_writer.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_msgset_writer.c 2018-10-10 06:54:38.000000000 +0000
@@ -32,6 +32,7 @@
#include "rdkafka_msgset.h"
#include "rdkafka_topic.h"
#include "rdkafka_partition.h"
+#include "rdkafka_header.h"
#include "rdkafka_lz4.h"
#include "snappy.h"
@@ -47,8 +48,11 @@
int msetw_features; /* Protocol features to use */
int msetw_msgcntmax; /* Max number of messages to send
* in a batch. */
- size_t msetw_messages_len; /* Total size of Messages, without
+ size_t msetw_messages_len; /* Total size of Messages, with Message
+ * framing but without
* MessageSet header */
+ size_t msetw_messages_kvlen; /* Total size of Message keys
+ * and values */
size_t msetw_MessageSetSize; /* Current MessageSetSize value */
size_t msetw_of_MessageSetSize; /* offset of MessageSetSize */
@@ -364,11 +368,13 @@
*
* @remark This currently constructs the entire ProduceRequest, containing
* a single outer MessageSet for a single partition.
+ *
+ * @locality broker thread
*/
static int rd_kafka_msgset_writer_init (rd_kafka_msgset_writer_t *msetw,
rd_kafka_broker_t *rkb,
rd_kafka_toppar_t *rktp) {
- int msgcnt = rd_atomic32_get(&rktp->rktp_xmit_msgq.rkmq_msg_cnt);
+ int msgcnt = rktp->rktp_xmit_msgq.rkmq_msg_cnt;
if (msgcnt == 0)
return 0;
@@ -433,16 +439,53 @@
* room in the buffer we'll copy the payload to the buffer,
* otherwise we push a reference to the memory. */
if (rkm->rkm_len <= (size_t)rk->rk_conf.msg_copy_max_size &&
- rd_buf_write_remains(&rkbuf->rkbuf_buf) > rkm->rkm_len)
+ rd_buf_write_remains(&rkbuf->rkbuf_buf) > rkm->rkm_len) {
rd_kafka_buf_write(rkbuf,
rkm->rkm_payload, rkm->rkm_len);
- else
+ if (free_cb)
+ free_cb(rkm->rkm_payload);
+ } else
rd_kafka_buf_push(rkbuf, rkm->rkm_payload, rkm->rkm_len,
free_cb);
}
/**
+ * @brief Write message headers to buffer.
+ *
+ * @remark The enveloping HeaderCount varint must already have been written.
+ * @returns the number of bytes written to msetw->msetw_rkbuf
+ */
+static size_t
+rd_kafka_msgset_writer_write_msg_headers (rd_kafka_msgset_writer_t *msetw,
+ const rd_kafka_headers_t *hdrs) {
+ rd_kafka_buf_t *rkbuf = msetw->msetw_rkbuf;
+ const rd_kafka_header_t *hdr;
+ int i;
+ size_t start_pos = rd_buf_write_pos(&rkbuf->rkbuf_buf);
+ size_t written;
+
+ RD_LIST_FOREACH(hdr, &hdrs->rkhdrs_list, i) {
+ rd_kafka_buf_write_varint(rkbuf, hdr->rkhdr_name_size);
+ rd_kafka_buf_write(rkbuf,
+ hdr->rkhdr_name, hdr->rkhdr_name_size);
+ rd_kafka_buf_write_varint(rkbuf,
+ hdr->rkhdr_value ?
+ (int64_t)hdr->rkhdr_value_size : -1);
+ rd_kafka_buf_write(rkbuf,
+ hdr->rkhdr_value,
+ hdr->rkhdr_value_size);
+ }
+
+ written = rd_buf_write_pos(&rkbuf->rkbuf_buf) - start_pos;
+ rd_dassert(written == hdrs->rkhdrs_ser_size);
+
+ return written;
+}
+
+
+
+/**
* @brief Write message to messageset buffer with MsgVersion 0 or 1.
* @returns the number of bytes written.
*/
@@ -536,6 +579,13 @@
size_t sz_KeyLen;
size_t sz_ValueLen;
size_t sz_HeaderCount;
+ int HeaderCount = 0;
+ size_t HeaderSize = 0;
+
+ if (rkm->rkm_headers) {
+ HeaderCount = rkm->rkm_headers->rkhdrs_list.rl_cnt;
+ HeaderSize = rkm->rkm_headers->rkhdrs_ser_size;
+ }
/* All varints, except for Length, needs to be pre-built
* so that the Length field can be set correctly and thus have
@@ -555,7 +605,8 @@
rkm->rkm_payload ? (int32_t)rkm->rkm_len :
(int32_t)RD_KAFKAP_BYTES_LEN_NULL);
sz_HeaderCount = rd_uvarint_enc_i32(
- varint_HeaderCount, sizeof(varint_HeaderCount), 0);
+ varint_HeaderCount, sizeof(varint_HeaderCount),
+ (int32_t)HeaderCount);
/* Calculate MessageSize without length of Length (added later)
* to store it in Length. */
@@ -567,7 +618,8 @@
rkm->rkm_key_len +
sz_ValueLen +
rkm->rkm_len +
- sz_HeaderCount;
+ sz_HeaderCount +
+ HeaderSize;
/* Length */
sz_Length = rd_uvarint_enc_i64(varint_Length, sizeof(varint_Length),
@@ -599,9 +651,14 @@
if (rkm->rkm_payload)
rd_kafka_msgset_writer_write_msg_payload(msetw, rkm, free_cb);
- /* HeaderCount (headers currently not implemented) */
+ /* HeaderCount */
rd_kafka_buf_write(rkbuf, varint_HeaderCount, sz_HeaderCount);
+ /* Headers array */
+ if (rkm->rkm_headers)
+ rd_kafka_msgset_writer_write_msg_headers(msetw,
+ rkm->rkm_headers);
+
/* Return written message size */
return MessageSize;
}
@@ -649,6 +706,8 @@
/**
* @brief Write as many messages from the given message queue to
* the messageset.
+ *
+ * May not write any messages.
*/
static void
rd_kafka_msgset_writer_write_msgq (rd_kafka_msgset_writer_t *msetw,
@@ -663,10 +722,11 @@
rd_ts_t MaxTimestamp = 0;
rd_kafka_msg_t *rkm;
int msgcnt = 0;
+ const rd_ts_t now = rd_clock();
/* Internal latency calculation base.
* Uses rkm_ts_timeout which is enqueue time + timeout */
- int_latency_base = rd_clock() +
+ int_latency_base = now +
(rktp->rktp_rkt->rkt_conf.message_timeout_ms * 1000);
/* Acquire BaseTimestamp from first message. */
@@ -690,10 +750,18 @@
break;
}
+ if (unlikely(rkm->rkm_u.producer.ts_backoff > now)) {
+ /* Stop accumulation when we've reached
+ * a message with a retry backoff in the future */
+ break;
+ }
+
/* Move message to buffer's queue */
rd_kafka_msgq_deq(rkmq, rkm, 1);
rd_kafka_msgq_enq(&rkbuf->rkbuf_msgq, rkm);
+ msetw->msetw_messages_kvlen += rkm->rkm_len + rkm->rkm_key_len;
+
/* Add internal latency metrics */
rd_avg_add(&rkb->rkb_avg_int_latency,
int_latency_base - rkm->rkm_ts_timeout);
@@ -731,9 +799,11 @@
const void *p;
size_t rlen;
int r;
-
+ int comp_level =
+ msetw->msetw_rktp->rktp_rkt->rkt_conf.compression_level;
+
memset(&strm, 0, sizeof(strm));
- r = deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
+ r = deflateInit2(&strm, comp_level,
Z_DEFLATED, 15+16,
8, Z_DEFAULT_STRATEGY);
if (r != Z_OK) {
@@ -868,9 +938,12 @@
rd_kafka_msgset_writer_compress_lz4 (rd_kafka_msgset_writer_t *msetw,
rd_slice_t *slice, struct iovec *ciov) {
rd_kafka_resp_err_t err;
+ int comp_level =
+ msetw->msetw_rktp->rktp_rkt->rkt_conf.compression_level;
err = rd_kafka_lz4_compress(msetw->msetw_rkb,
/* Correct or incorrect HC */
msetw->msetw_MsgVersion >= 1 ? 1 : 0,
+ comp_level,
slice, &ciov->iov_base, &ciov->iov_len);
return (err ? -1 : 0);
}
@@ -1108,6 +1181,9 @@
rd_assert(len > 0);
rd_assert(len <= (size_t)rktp->rktp_rkt->rkt_rk->rk_conf.max_msg_size);
+ rd_atomic64_add(&rktp->rktp_c.tx_msgs, cnt);
+ rd_atomic64_add(&rktp->rktp_c.tx_msg_bytes, msetw->msetw_messages_kvlen);
+
/* Compress the message set */
if (rktp->rktp_rkt->rkt_conf.compression_codec)
rd_kafka_msgset_writer_compress(msetw, &len);
@@ -1128,7 +1204,6 @@
cnt, msetw->msetw_MessageSetSize,
msetw->msetw_ApiVersion, msetw->msetw_MsgVersion);
-
return rkbuf;
}
@@ -1144,6 +1219,8 @@
*
* @returns the buffer to transmit or NULL if there were no messages
* in messageset.
+ *
+ * @locality broker thread
*/
rd_kafka_buf_t *
rd_kafka_msgset_create_ProduceRequest (rd_kafka_broker_t *rkb,
diff -Nru librdkafka-0.11.3/src/rdkafka_offset.h librdkafka-0.11.6/src/rdkafka_offset.h
--- librdkafka-0.11.3/src/rdkafka_offset.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_offset.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_OFFSET_H_
+#define _RDKAFKA_OFFSET_H_
#include "rdkafka_partition.h"
@@ -70,3 +71,4 @@
rd_kafka_resp_err_t err,
const rd_kafka_topic_partition_list_t *offsets);
+#endif /* _RDKAFKA_OFFSET_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_op.c librdkafka-0.11.6/src/rdkafka_op.c
--- librdkafka-0.11.3/src/rdkafka_op.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_op.c 2018-10-10 06:54:38.000000000 +0000
@@ -72,6 +72,12 @@
[RD_KAFKA_OP_METADATA] = "REPLY:METADATA",
[RD_KAFKA_OP_LOG] = "REPLY:LOG",
[RD_KAFKA_OP_WAKEUP] = "REPLY:WAKEUP",
+ [RD_KAFKA_OP_CREATETOPICS] = "REPLY:CREATETOPICS",
+ [RD_KAFKA_OP_DELETETOPICS] = "REPLY:DELETETOPICS",
+ [RD_KAFKA_OP_CREATEPARTITIONS] = "REPLY:CREATEPARTITIONS",
+ [RD_KAFKA_OP_ALTERCONFIGS] = "REPLY:ALTERCONFIGS",
+ [RD_KAFKA_OP_DESCRIBECONFIGS] = "REPLY:DESCRIBECONFIGS",
+ [RD_KAFKA_OP_ADMIN_RESULT] = "REPLY:ADMIN_RESULT",
};
if (type & RD_KAFKA_OP_REPLY)
@@ -124,7 +130,7 @@
break;
case RD_KAFKA_OP_DR:
fprintf(fp, "%s %"PRId32" messages on %s\n", prefix,
- rd_atomic32_get(&rko->rko_u.dr.msgq.rkmq_msg_cnt),
+ rko->rko_u.dr.msgq.rkmq_msg_cnt,
rko->rko_u.dr.s_rkt ?
rd_kafka_topic_s2i(rko->rko_u.dr.s_rkt)->
rkt_topic->str : "(n/a)");
@@ -185,6 +191,12 @@
[RD_KAFKA_OP_METADATA] = sizeof(rko->rko_u.metadata),
[RD_KAFKA_OP_LOG] = sizeof(rko->rko_u.log),
[RD_KAFKA_OP_WAKEUP] = 0,
+ [RD_KAFKA_OP_CREATETOPICS] = sizeof(rko->rko_u.admin_request),
+ [RD_KAFKA_OP_DELETETOPICS] = sizeof(rko->rko_u.admin_request),
+ [RD_KAFKA_OP_CREATEPARTITIONS] = sizeof(rko->rko_u.admin_request),
+ [RD_KAFKA_OP_ALTERCONFIGS] = sizeof(rko->rko_u.admin_request),
+ [RD_KAFKA_OP_DESCRIBECONFIGS] = sizeof(rko->rko_u.admin_request),
+ [RD_KAFKA_OP_ADMIN_RESULT] = sizeof(rko->rko_u.admin_result),
};
size_t tsize = op2size[type & ~RD_KAFKA_OP_FLAGMASK];
@@ -291,6 +303,20 @@
rd_free(rko->rko_u.log.str);
break;
+ case RD_KAFKA_OP_CREATETOPICS:
+ case RD_KAFKA_OP_DELETETOPICS:
+ case RD_KAFKA_OP_CREATEPARTITIONS:
+ case RD_KAFKA_OP_ALTERCONFIGS:
+ case RD_KAFKA_OP_DESCRIBECONFIGS:
+ rd_kafka_replyq_destroy(&rko->rko_u.admin_request.replyq);
+ rd_list_destroy(&rko->rko_u.admin_request.args);
+ break;
+
+ case RD_KAFKA_OP_ADMIN_RESULT:
+ rd_list_destroy(&rko->rko_u.admin_result.results);
+ RD_IF_FREE(rko->rko_u.admin_result.errstr, rd_free);
+ break;
+
default:
break;
}
@@ -300,7 +326,8 @@
/* Let callback clean up */
rko->rko_err = RD_KAFKA_RESP_ERR__DESTROY;
res = rko->rko_op_cb(rko->rko_rk, NULL, rko);
- assert(res != RD_KAFKA_OP_RES_YIELD);
+ rd_assert(res != RD_KAFKA_OP_RES_YIELD);
+ rd_assert(res != RD_KAFKA_OP_RES_KEEP);
}
RD_IF_FREE(rko->rko_rktp, rd_kafka_toppar_destroy);
@@ -492,7 +519,8 @@
res = rko->rko_op_cb(rk, rkq, rko);
if (unlikely(res == RD_KAFKA_OP_RES_YIELD || rd_kafka_yield_thread))
return RD_KAFKA_OP_RES_YIELD;
- rko->rko_op_cb = NULL;
+ if (res != RD_KAFKA_OP_RES_KEEP)
+ rko->rko_op_cb = NULL;
return res;
}
@@ -586,8 +614,6 @@
return rd_kafka_op_call(rk, rkq, rko);
else if (rko->rko_type == RD_KAFKA_OP_RECV_BUF) /* Handle Response */
rd_kafka_buf_handle_op(rko, rko->rko_err);
- else if (rko->rko_type == RD_KAFKA_OP_WAKEUP)
- ;/* do nothing, wake up is a fact anyway */
else if (cb_type != RD_KAFKA_Q_CB_RETURN &&
rko->rko_type & RD_KAFKA_OP_REPLY &&
rko->rko_err == RD_KAFKA_RESP_ERR__DESTROY)
@@ -619,7 +645,10 @@
rd_kafka_op_res_t res;
res = rd_kafka_op_handle_std(rk, rkq, rko, cb_type);
- if (res == RD_KAFKA_OP_RES_HANDLED) {
+ if (res == RD_KAFKA_OP_RES_KEEP) {
+ /* Op was handled but must not be destroyed. */
+ return res;
+ } if (res == RD_KAFKA_OP_RES_HANDLED) {
rd_kafka_op_destroy(rko);
return res;
} else if (unlikely(res == RD_KAFKA_OP_RES_YIELD))
diff -Nru librdkafka-0.11.3/src/rdkafka_op.h librdkafka-0.11.6/src/rdkafka_op.h
--- librdkafka-0.11.3/src/rdkafka_op.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_op.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,10 +25,14 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_OP_H_
+#define _RDKAFKA_OP_H_
#include "rdkafka_msg.h"
+#include "rdkafka_timer.h"
+#include "rdkafka_admin.h"
+
/* Forward declarations */
typedef struct rd_kafka_q_s rd_kafka_q_t;
@@ -63,6 +67,7 @@
#define RD_KAFKA_OP_F_CRC 0x8 /* rkbuf: Perform CRC calculation */
#define RD_KAFKA_OP_F_BLOCKING 0x10 /* rkbuf: blocking protocol request */
#define RD_KAFKA_OP_F_REPROCESS 0x20 /* cgrp: Reprocess at a later time. */
+#define RD_KAFKA_OP_F_SENT 0x80 /* rkbuf: request sent on wire */
typedef enum {
@@ -107,12 +112,18 @@
RD_KAFKA_OP_METADATA, /* Metadata response */
RD_KAFKA_OP_LOG, /* Log */
RD_KAFKA_OP_WAKEUP, /* Wake-up signaling */
+ RD_KAFKA_OP_CREATETOPICS, /**< Admin: CreateTopics: u.admin_request*/
+ RD_KAFKA_OP_DELETETOPICS, /**< Admin: DeleteTopics: u.admin_request*/
+ RD_KAFKA_OP_CREATEPARTITIONS,/**< Admin: CreatePartitions: u.admin_request*/
+ RD_KAFKA_OP_ALTERCONFIGS, /**< Admin: AlterConfigs: u.admin_request*/
+ RD_KAFKA_OP_DESCRIBECONFIGS, /**< Admin: DescribeConfigs: u.admin_request*/
+ RD_KAFKA_OP_ADMIN_RESULT, /**< Admin API .._result_t */
RD_KAFKA_OP__END
} rd_kafka_op_type_t;
/* Flags used with op_type_t */
-#define RD_KAFKA_OP_CB (1 << 30) /* Callback op. */
-#define RD_KAFKA_OP_REPLY (1 << 31) /* Reply op. */
+#define RD_KAFKA_OP_CB (int)(1 << 29) /* Callback op. */
+#define RD_KAFKA_OP_REPLY (int)(1 << 30) /* Reply op. */
#define RD_KAFKA_OP_FLAGMASK (RD_KAFKA_OP_CB | RD_KAFKA_OP_REPLY)
@@ -142,6 +153,11 @@
typedef enum {
RD_KAFKA_OP_RES_PASS, /* Not handled, pass to caller */
RD_KAFKA_OP_RES_HANDLED, /* Op was handled (through callbacks) */
+ RD_KAFKA_OP_RES_KEEP, /* Op was handled (through callbacks)
+ * but must not be destroyed by op_handle().
+ * It is NOT PERMITTED to return RES_KEEP
+ * from a callback handling a ERR__DESTROY
+ * event. */
RD_KAFKA_OP_RES_YIELD /* Callback called yield */
} rd_kafka_op_res_t;
@@ -177,6 +193,9 @@
struct rd_kafka_op_s *rko)
RD_WARN_UNUSED_RESULT;
+/* Forward declaration */
+struct rd_kafka_admin_worker_cbs;
+
#define RD_KAFKA_OP_TYPE_ASSERT(rko,type) \
rd_kafka_assert(NULL, (rko)->rko_type == (type) && # type)
@@ -320,6 +339,79 @@
int level;
char *str;
} log;
+
+ struct {
+ rd_kafka_AdminOptions_t options; /**< Copy of user's
+ * options, or NULL */
+ rd_ts_t abs_timeout; /**< Absolute timeout
+ * for this request. */
+ rd_kafka_timer_t tmr; /**< Timeout timer */
+ struct rd_kafka_enq_once_s *eonce; /**< Enqueue op
+ * only once,
+ * used to
+ * (re)trigger
+ * the request op
+ * upon broker state
+ * changes while
+ * waiting for the
+ * controller, or
+ * due to .tmr
+ * timeout. */
+ rd_list_t args;/**< Type depends on request, e.g.
+ * rd_kafka_NewTopic_t for CreateTopics
+ */
+
+ rd_kafka_buf_t *reply_buf; /**< Protocol reply,
+ * temporary reference not
+ * owned by this rko */
+
+ /**< Worker callbacks, see rdkafka_admin.c */
+ struct rd_kafka_admin_worker_cbs *cbs;
+
+ /** Worker state */
+ enum {
+ RD_KAFKA_ADMIN_STATE_INIT,
+ RD_KAFKA_ADMIN_STATE_WAIT_BROKER,
+ RD_KAFKA_ADMIN_STATE_WAIT_CONTROLLER,
+ RD_KAFKA_ADMIN_STATE_CONSTRUCT_REQUEST,
+ RD_KAFKA_ADMIN_STATE_WAIT_RESPONSE,
+ } state;
+
+ int32_t broker_id; /**< Requested broker id to
+ * communicate with.
+ * Used for AlterConfigs, et.al,
+ * that needs to speak to a
+ * specific broker rather than
+ * the controller.
+ * Defaults to -1:
+ * look up and use controller. */
+
+ /** Application's reply queue */
+ rd_kafka_replyq_t replyq;
+ rd_kafka_event_type_t reply_event_type;
+ } admin_request;
+
+ struct {
+ rd_kafka_op_type_t reqtype; /**< Request op type,
+ * used for logging. */
+
+ char *errstr; /**< Error string, if rko_err
+ * is set, else NULL. */
+
+ rd_list_t results; /**< Type depends on request type:
+ *
+ * (rd_kafka_topic_result_t *):
+ * CreateTopics, DeleteTopics,
+ * CreatePartitions.
+ *
+ * (rd_kafka_ConfigResource_t *):
+ * AlterConfigs, DescribeConfigs
+ */
+
+ void *opaque; /**< Application's opaque as set by
+ * rd_kafka_AdminOptions_set_opaque
+ */
+ } admin_result;
} rko_u;
};
@@ -350,7 +442,7 @@
#define rd_kafka_op_err(rk,err,...) do { \
- if (!(rk)->rk_conf.error_cb) { \
+ if (!((rk)->rk_conf.enabled_events & RD_KAFKA_EVENT_ERROR)) { \
rd_kafka_log(rk, LOG_ERR, "ERROR", __VA_ARGS__); \
break; \
} \
@@ -398,3 +490,5 @@
void rd_kafka_op_offset_store (rd_kafka_t *rk, rd_kafka_op_t *rko,
const rd_kafka_message_t *rkmessage);
+
+#endif /* _RDKAFKA_OP_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_partition.c librdkafka-0.11.6/src/rdkafka_partition.c
--- librdkafka-0.11.3/src/rdkafka_partition.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_partition.c 2018-10-10 06:54:38.000000000 +0000
@@ -48,9 +48,10 @@
rd_kafka_toppar_op_serve (rd_kafka_t *rk,
rd_kafka_q_t *rkq, rd_kafka_op_t *rko,
rd_kafka_q_cb_type_t cb_type, void *opaque);
-static RD_INLINE void rd_kafka_broker_fetch_toppar_del (rd_kafka_broker_t *rkb,
- rd_kafka_toppar_t *rktp);
+static void rd_kafka_toppar_offset_retry (rd_kafka_toppar_t *rktp,
+ int backoff_ms,
+ const char *reason);
static RD_INLINE int32_t
@@ -88,6 +89,12 @@
/* Parse and return Offset */
err = rd_kafka_handle_Offset(rkb->rkb_rk, rkb, err,
rkbuf, request, offsets);
+
+ if (err == RD_KAFKA_RESP_ERR__IN_PROGRESS) {
+ rd_kafka_topic_partition_list_destroy(offsets);
+ return; /* Retrying */
+ }
+
if (!err && !(rktpar = rd_kafka_topic_partition_list_find(
offsets,
rktp->rktp_rkt->rkt_topic->str,
@@ -175,6 +182,10 @@
rktp->rktp_partition = partition;
rktp->rktp_rkt = rkt;
rktp->rktp_leader_id = -1;
+ /* Mark partition as unknown (does not exist) until we see the
+ * partition in topic metadata. */
+ if (partition != RD_KAFKA_PARTITION_UA)
+ rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_UNKNOWN;
rktp->rktp_fetch_state = RD_KAFKA_TOPPAR_FETCH_NONE;
rktp->rktp_fetch_msg_max_bytes
= rkt->rkt_rk->rk_conf.fetch_msg_max_bytes;
@@ -183,8 +194,12 @@
rd_kafka_offset_stats_reset(&rktp->rktp_offsets_fin);
rktp->rktp_hi_offset = RD_KAFKA_OFFSET_INVALID;
rktp->rktp_lo_offset = RD_KAFKA_OFFSET_INVALID;
+ rktp->rktp_query_offset = RD_KAFKA_OFFSET_INVALID;
+ rktp->rktp_next_offset = RD_KAFKA_OFFSET_INVALID;
+ rktp->rktp_last_next_offset = RD_KAFKA_OFFSET_INVALID;
rktp->rktp_app_offset = RD_KAFKA_OFFSET_INVALID;
rktp->rktp_stored_offset = RD_KAFKA_OFFSET_INVALID;
+ rktp->rktp_committing_offset = RD_KAFKA_OFFSET_INVALID;
rktp->rktp_committed_offset = RD_KAFKA_OFFSET_INVALID;
rd_kafka_msgq_init(&rktp->rktp_msgq);
rktp->rktp_msgq_wakeup_fd = -1;
@@ -304,6 +319,16 @@
rd_kafka_fetch_states[fetch_state]);
rktp->rktp_fetch_state = fetch_state;
+
+ if (fetch_state == RD_KAFKA_TOPPAR_FETCH_ACTIVE)
+ rd_kafka_dbg(rktp->rktp_rkt->rkt_rk,
+ CONSUMER|RD_KAFKA_DBG_TOPIC,
+ "FETCH",
+ "Partition %.*s [%"PRId32"] start fetching "
+ "at offset %s",
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
+ rktp->rktp_partition,
+ rd_kafka_offset2str(rktp->rktp_next_offset));
}
@@ -556,6 +581,9 @@
rkt->rkt_topic->str, rktp->rktp_partition);
rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_DESIRED;
}
+ /* If toppar was marked for removal this is no longer
+ * the case since the partition is now desired. */
+ rktp->rktp_flags &= ~RD_KAFKA_TOPPAR_F_REMOVE;
rd_kafka_toppar_unlock(rktp);
return s_rktp;
}
@@ -567,7 +595,6 @@
rktp = rd_kafka_toppar_s2i(s_rktp);
rd_kafka_toppar_lock(rktp);
- rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_UNKNOWN;
rd_kafka_toppar_desired_add0(rktp);
rd_kafka_toppar_unlock(rktp);
@@ -594,13 +621,15 @@
rktp->rktp_flags &= ~RD_KAFKA_TOPPAR_F_DESIRED;
rd_kafka_toppar_desired_unlink(rktp);
- if (rktp->rktp_flags & RD_KAFKA_TOPPAR_F_UNKNOWN)
- rktp->rktp_flags &= ~RD_KAFKA_TOPPAR_F_UNKNOWN;
-
-
rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "DESP",
"Removing (un)desired topic %s [%"PRId32"]",
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition);
+
+ if (rktp->rktp_flags & RD_KAFKA_TOPPAR_F_UNKNOWN) {
+ /* If this partition does not exist in the cluster
+ * and is no longer desired, remove it. */
+ rd_kafka_toppar_broker_leave_for_remove(rktp);
+ }
}
@@ -609,26 +638,39 @@
* Append message at tail of 'rktp' message queue.
*/
void rd_kafka_toppar_enq_msg (rd_kafka_toppar_t *rktp, rd_kafka_msg_t *rkm) {
+ int wakeup_fd, queue_len;
- rd_kafka_toppar_lock(rktp);
- rd_kafka_msgq_enq(&rktp->rktp_msgq, rkm);
-#ifndef _MSC_VER
- if (rktp->rktp_msgq_wakeup_fd != -1 &&
- rd_kafka_msgq_len(&rktp->rktp_msgq) == 1) {
+ rd_kafka_toppar_lock(rktp);
+
+ if (!rkm->rkm_u.producer.msgseq &&
+ rktp->rktp_partition != RD_KAFKA_PARTITION_UA)
+ rkm->rkm_u.producer.msgseq = ++rktp->rktp_msgseq;
+
+ if (rktp->rktp_partition == RD_KAFKA_PARTITION_UA ||
+ rktp->rktp_rkt->rkt_conf.queuing_strategy == RD_KAFKA_QUEUE_FIFO) {
+ /* No need for enq_sorted(), this is the oldest message. */
+ queue_len = rd_kafka_msgq_enq(&rktp->rktp_msgq, rkm);
+ } else {
+ queue_len = rd_kafka_msgq_enq_sorted(rktp->rktp_rkt,
+ &rktp->rktp_msgq, rkm);
+ }
+
+ wakeup_fd = rktp->rktp_msgq_wakeup_fd;
+ rd_kafka_toppar_unlock(rktp);
+
+ if (wakeup_fd != -1 && queue_len == 1) {
char one = 1;
int r;
- r = rd_write(rktp->rktp_msgq_wakeup_fd, &one, sizeof(one));
+ r = rd_write(wakeup_fd, &one, sizeof(one));
if (r == -1)
rd_kafka_log(rktp->rktp_rkt->rkt_rk, LOG_ERR, "PARTENQ",
"%s [%"PRId32"]: write to "
"wake-up fd %d failed: %s",
rktp->rktp_rkt->rkt_topic->str,
rktp->rktp_partition,
- rktp->rktp_msgq_wakeup_fd,
+ wakeup_fd,
rd_strerror(errno));
}
-#endif
- rd_kafka_toppar_unlock(rktp);
}
@@ -641,52 +683,170 @@
rd_kafka_toppar_unlock(rktp);
}
-/**
- * Inserts all messages from 'rkmq' at head of toppar 'rktp's queue.
- * 'rkmq' will be cleared.
- */
-void rd_kafka_toppar_insert_msgq (rd_kafka_toppar_t *rktp,
- rd_kafka_msgq_t *rkmq) {
- rd_kafka_toppar_lock(rktp);
- rd_kafka_msgq_concat(rkmq, &rktp->rktp_msgq);
- rd_kafka_msgq_move(&rktp->rktp_msgq, rkmq);
- rd_kafka_toppar_unlock(rktp);
+
+void rd_kafka_msgq_insert_msgq (rd_kafka_msgq_t *destq,
+ rd_kafka_msgq_t *srcq,
+ int (*cmp) (const void *a, const void *b)) {
+ rd_kafka_msg_t *first, *dest_first;
+
+ first = TAILQ_FIRST(&srcq->rkmq_msgs);
+ if (unlikely(!first)) {
+ /* srcq is empty */
+ return;
+ }
+
+ dest_first = TAILQ_FIRST(&destq->rkmq_msgs);
+
+ /*
+ * Try to optimize insertion of source list.
+ */
+
+ if (unlikely(!dest_first)) {
+ /* Dest queue is empty, simply move the srcq. */
+ rd_kafka_msgq_move(destq, srcq);
+
+ return;
+ }
+
+ /* See if we can optimize the insertion by bulk-loading
+ * the messages in place.
+ * We know that:
+ * - destq is sorted
+ * - srcq is sorted
+ * - there is no overlap between the two.
+ */
+
+ if (cmp(first, dest_first) < 0) {
+ /* Prepend src to dest queue.
+ * First append existing dest queue to src queue,
+ * then move src queue to now-empty dest queue,
+ * effectively prepending src queue to dest queue. */
+ rd_kafka_msgq_concat(srcq, destq);
+ rd_kafka_msgq_move(destq, srcq);
+
+ } else if (cmp(first,
+ TAILQ_LAST(&destq->rkmq_msgs,
+ rd_kafka_msgs_head_s)) > 0) {
+ /* Append src to dest queue */
+ rd_kafka_msgq_concat(destq, srcq);
+
+ } else {
+ /* Source queue messages reside somewhere
+ * in the dest queue range, find the insert position. */
+ rd_kafka_msg_t *at;
+
+ at = rd_kafka_msgq_find_pos(destq, first, cmp);
+ rd_assert(at &&
+ *"Bug in msg_order_cmp(): "
+ "could not find insert position");
+
+ /* Insert input queue after 'at' position.
+ * We know that:
+ * - at is non-NULL
+ * - at is not the last element. */
+ TAILQ_INSERT_LIST(&destq->rkmq_msgs,
+ at, &srcq->rkmq_msgs,
+ rd_kafka_msgs_head_s,
+ rd_kafka_msg_t *, rkm_link);
+
+ destq->rkmq_msg_cnt += srcq->rkmq_msg_cnt;
+ destq->rkmq_msg_bytes += srcq->rkmq_msg_bytes;
+ rd_kafka_msgq_init(srcq);
+ }
}
/**
- * Concats all messages from 'rkmq' at tail of toppar 'rktp's queue.
- * 'rkmq' will be cleared.
- */
-void rd_kafka_toppar_concat_msgq (rd_kafka_toppar_t *rktp,
- rd_kafka_msgq_t *rkmq) {
- rd_kafka_toppar_lock(rktp);
- rd_kafka_msgq_concat(&rktp->rktp_msgq, rkmq);
- rd_kafka_toppar_unlock(rktp);
+ * @brief Inserts messages from \p srcq according to their sorted position
+ * into \p destq, filtering out messages that can not be retried.
+ *
+ * @param incr_retry Increment retry count for messages.
+ * @param max_retries Maximum retries allowed per message.
+ * @param backoff Absolute retry backoff for retried messages.
+ *
+ * @returns 0 if all messages were retried, or 1 if some messages
+ * could not be retried.
+ */
+int rd_kafka_retry_msgq (rd_kafka_msgq_t *destq,
+ rd_kafka_msgq_t *srcq,
+ int incr_retry, int max_retries, rd_ts_t backoff,
+ int (*cmp) (const void *a, const void *b)) {
+ rd_kafka_msgq_t retryable = RD_KAFKA_MSGQ_INITIALIZER(retryable);
+ rd_kafka_msg_t *rkm, *tmp;
+
+ /* Scan through messages to see which ones are eligible for retry,
+ * move the retryable ones to temporary queue and
+ * set backoff time for first message and optionally
+ * increase retry count for each message.
+ * Sorted insert is not necessary since the original order
+ * srcq order is maintained. */
+ TAILQ_FOREACH_SAFE(rkm, &srcq->rkmq_msgs, rkm_link, tmp) {
+ if (rkm->rkm_u.producer.retries + incr_retry > max_retries)
+ continue;
+
+ rd_kafka_msgq_deq(srcq, rkm, 1);
+ rd_kafka_msgq_enq(&retryable, rkm);
+
+ rkm->rkm_u.producer.ts_backoff = backoff;
+ rkm->rkm_u.producer.retries += incr_retry;
+ }
+
+ /* No messages are retryable */
+ if (RD_KAFKA_MSGQ_EMPTY(&retryable))
+ return 0;
+
+ /* Insert retryable list at sorted position */
+ rd_kafka_msgq_insert_msgq(destq, &retryable, cmp);
+
+ return 1;
}
/**
- * Move all messages in 'rkmq' to the unassigned partition, if any.
- * Returns 0 on success or -1 if there was no UA partition.
+ * @brief Inserts messages from \p rkmq according to their sorted position
+ * into the partition's message queue.
+ *
+ * @param incr_retry Increment retry count for messages.
+ *
+ * @returns 0 if all messages were retried, or 1 if some messages
+ * could not be retried.
+ *
+ * @locality Broker thread
*/
-int rd_kafka_toppar_ua_move (rd_kafka_itopic_t *rkt, rd_kafka_msgq_t *rkmq) {
- shptr_rd_kafka_toppar_t *s_rktp_ua;
- rd_kafka_topic_rdlock(rkt);
- s_rktp_ua = rd_kafka_toppar_get(rkt, RD_KAFKA_PARTITION_UA, 0);
- rd_kafka_topic_rdunlock(rkt);
+int rd_kafka_toppar_retry_msgq (rd_kafka_toppar_t *rktp, rd_kafka_msgq_t *rkmq,
+ int incr_retry) {
+ rd_kafka_t *rk = rktp->rktp_rkt->rkt_rk;
+ rd_ts_t backoff = rd_clock() + (rk->rk_conf.retry_backoff_ms * 1000);
+ int r;
- if (unlikely(s_rktp_ua == NULL))
- return -1;
+ if (rd_kafka_terminating(rk))
+ return 1;
- rd_kafka_msgq_concat(&rd_kafka_toppar_s2i(s_rktp_ua)->rktp_msgq, rkmq);
+ rd_kafka_toppar_lock(rktp);
+ r = rd_kafka_retry_msgq(&rktp->rktp_msgq, rkmq,
+ incr_retry, rk->rk_conf.max_retries,
+ backoff,
+ rktp->rktp_rkt->rkt_conf.msg_order_cmp);
+ rd_kafka_toppar_unlock(rktp);
- rd_kafka_toppar_destroy(s_rktp_ua);
+ return r;
+}
- return 0;
+/**
+ * @brief Insert sorted message list \p rkmq at sorted position in \p rktp 's
+ * message queue. The queues must not overlap.
+ * @remark \p rkmq will be cleared.
+ */
+void rd_kafka_toppar_insert_msgq (rd_kafka_toppar_t *rktp,
+ rd_kafka_msgq_t *rkmq) {
+ rd_kafka_toppar_lock(rktp);
+ rd_kafka_msgq_insert_msgq(&rktp->rktp_msgq, rkmq,
+ rktp->rktp_rkt->rkt_conf.msg_order_cmp);
+ rd_kafka_toppar_unlock(rktp);
}
+
/**
* Helper method for purging queues when removing a toppar.
* Locks: rd_kafka_toppar_lock() MUST be held
@@ -725,20 +885,14 @@
if (had_next_leader)
return;
- /* Revert from offset-wait state back to offset-query
- * prior to leaving the broker to avoid stalling
- * on the new broker waiting for a offset reply from
- * this old broker (that might not come and thus need
- * to time out..slowly) */
- if (rktp->rktp_fetch_state == RD_KAFKA_TOPPAR_FETCH_OFFSET_WAIT) {
- rd_kafka_toppar_set_fetch_state(
- rktp, RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY);
- rd_kafka_timer_start(&rktp->rktp_rkt->rkt_rk->rk_timers,
- &rktp->rktp_offset_query_tmr,
- 500*1000,
- rd_kafka_offset_query_tmr_cb,
- rktp);
- }
+ /* Revert from offset-wait state back to offset-query
+ * prior to leaving the broker to avoid stalling
+ * on the new broker waiting for a offset reply from
+ * this old broker (that might not come and thus need
+ * to time out..slowly) */
+ if (rktp->rktp_fetch_state == RD_KAFKA_TOPPAR_FETCH_OFFSET_WAIT)
+ rd_kafka_toppar_offset_retry(rktp, 500,
+ "migrating to new leader");
if (old_rkb) {
/* If there is an existing broker for this toppar we let it
@@ -778,6 +932,7 @@
rd_kafka_op_t *rko;
rd_kafka_broker_t *dest_rkb;
+ rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_REMOVE;
if (rktp->rktp_next_leader)
dest_rkb = rktp->rktp_next_leader;
@@ -874,8 +1029,8 @@
RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
rktp->rktp_partition,
rd_kafka_broker_name(rkb),
- rd_atomic32_get(&rktp->rktp_msgq.rkmq_msg_cnt),
- rd_atomic64_get(&rktp->rktp_msgq.rkmq_msg_bytes));
+ rktp->rktp_msgq.rkmq_msg_cnt,
+ rktp->rktp_msgq.rkmq_msg_bytes);
} else {
@@ -1083,7 +1238,7 @@
rd_kafka_toppar_lock(rktp);
/* Drop reply from previous partition leader */
- if (rktp->rktp_leader != rkb)
+ if (err != RD_KAFKA_RESP_ERR__DESTROY && rktp->rktp_leader != rkb)
err = RD_KAFKA_RESP_ERR__OUTDATED;
rd_kafka_toppar_unlock(rktp);
@@ -1129,6 +1284,13 @@
err == RD_KAFKA_RESP_ERR__OUTDATED) {
/* Termination or outdated, quick cleanup. */
+ if (err == RD_KAFKA_RESP_ERR__OUTDATED) {
+ rd_kafka_toppar_lock(rktp);
+ rd_kafka_toppar_offset_retry(
+ rktp, 500, "outdated offset response");
+ rd_kafka_toppar_unlock(rktp);
+ }
+
/* from request.opaque */
rd_kafka_toppar_destroy(s_rktp);
return;
@@ -1180,6 +1342,52 @@
rd_kafka_toppar_destroy(s_rktp); /* from request.opaque */
}
+
+/**
+ * @brief An Offset fetch failed (for whatever reason) in
+ * the RD_KAFKA_TOPPAR_FETCH_OFFSET_WAIT state:
+ * set the state back to FETCH_OFFSET_QUERY and start the
+ * offset_query_tmr to trigger a new request eventually.
+ *
+ * @locality toppar handler thread
+ * @locks toppar_lock() MUST be held
+ */
+static void rd_kafka_toppar_offset_retry (rd_kafka_toppar_t *rktp,
+ int backoff_ms,
+ const char *reason) {
+ rd_ts_t tmr_next;
+ int restart_tmr;
+
+ /* (Re)start timer if not started or the current timeout
+ * is larger than \p backoff_ms. */
+ tmr_next = rd_kafka_timer_next(&rktp->rktp_rkt->rkt_rk->rk_timers,
+ &rktp->rktp_offset_query_tmr, 1);
+
+ restart_tmr = (tmr_next == -1 ||
+ tmr_next > rd_clock() + (backoff_ms * 1000ll));
+
+ rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
+ "%s [%"PRId32"]: %s: %s for offset %s",
+ rktp->rktp_rkt->rkt_topic->str,
+ rktp->rktp_partition,
+ reason,
+ restart_tmr ?
+ "(re)starting offset query timer" :
+ "offset query timer already scheduled",
+ rd_kafka_offset2str(rktp->rktp_query_offset));
+
+ rd_kafka_toppar_set_fetch_state(rktp,
+ RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY);
+
+ if (restart_tmr)
+ rd_kafka_timer_start(&rktp->rktp_rkt->rkt_rk->rk_timers,
+ &rktp->rktp_offset_query_tmr,
+ backoff_ms*1000ll,
+ rd_kafka_offset_query_tmr_cb, rktp);
+}
+
+
+
/**
* Send OffsetRequest for toppar.
*
@@ -1202,21 +1410,11 @@
backoff_ms = 500;
if (backoff_ms) {
- rd_kafka_dbg(rktp->rktp_rkt->rkt_rk, TOPIC, "OFFSET",
- "%s [%"PRId32"]: %s"
- "starting offset query timer for offset %s",
- rktp->rktp_rkt->rkt_topic->str,
- rktp->rktp_partition,
- !rkb ? "no current leader for partition, " : "",
- rd_kafka_offset2str(query_offset));
-
- rd_kafka_toppar_set_fetch_state(
- rktp, RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY);
- rd_kafka_timer_start(&rktp->rktp_rkt->rkt_rk->rk_timers,
- &rktp->rktp_offset_query_tmr,
- backoff_ms*1000ll,
- rd_kafka_offset_query_tmr_cb, rktp);
- return;
+ rd_kafka_toppar_offset_retry(rktp, backoff_ms,
+ !rkb ?
+ "no current leader for partition":
+ "backoff");
+ return;
}
@@ -1376,6 +1574,8 @@
rd_kafka_toppar_set_fetch_state(rktp, RD_KAFKA_TOPPAR_FETCH_STOPPED);
+ rktp->rktp_app_offset = RD_KAFKA_OFFSET_INVALID;
+
if (rktp->rktp_cgrp) {
/* Detach toppar from cgrp */
rd_kafka_cgrp_op(rktp->rktp_cgrp, rktp, RD_KAFKA_NO_REPLYQ,
@@ -1531,7 +1731,9 @@
if (rk->rk_type == RD_KAFKA_CONSUMER) {
/* Save offset of last consumed message+1 as the
* next message to fetch on resume. */
- rktp->rktp_next_offset = rktp->rktp_app_offset;
+ if (rktp->rktp_app_offset != RD_KAFKA_OFFSET_INVALID) {
+ rktp->rktp_next_offset = rktp->rktp_app_offset;
+ }
rd_kafka_dbg(rk, TOPIC, pause?"PAUSE":"RESUME",
"%s %s [%"PRId32"]: at offset %s "
@@ -1612,66 +1814,6 @@
/**
- * Add toppar to fetch list.
- *
- * Locality: broker thread
- * Locks: none
- */
-static RD_INLINE void rd_kafka_broker_fetch_toppar_add (rd_kafka_broker_t *rkb,
- rd_kafka_toppar_t *rktp){
- if (rktp->rktp_fetch)
- return; /* Already added */
-
- CIRCLEQ_INSERT_TAIL(&rkb->rkb_fetch_toppars, rktp, rktp_fetchlink);
- rkb->rkb_fetch_toppar_cnt++;
- rktp->rktp_fetch = 1;
-
- if (unlikely(rkb->rkb_fetch_toppar_cnt == 1))
- rd_kafka_broker_fetch_toppar_next(rkb, rktp);
-
- rd_rkb_dbg(rkb, TOPIC, "FETCHADD",
- "Added %.*s [%"PRId32"] to fetch list (%d entries, opv %d)",
- RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
- rktp->rktp_partition,
- rkb->rkb_fetch_toppar_cnt, rktp->rktp_fetch_version);
-}
-
-
-/**
- * Remove toppar from fetch list.
- *
- * Locality: broker thread
- * Locks: none
- */
-static RD_INLINE void rd_kafka_broker_fetch_toppar_del (rd_kafka_broker_t *rkb,
- rd_kafka_toppar_t *rktp){
- if (!rktp->rktp_fetch)
- return; /* Not added */
-
- CIRCLEQ_REMOVE(&rkb->rkb_fetch_toppars, rktp, rktp_fetchlink);
- rd_kafka_assert(NULL, rkb->rkb_fetch_toppar_cnt > 0);
- rkb->rkb_fetch_toppar_cnt--;
- rktp->rktp_fetch = 0;
-
- if (rkb->rkb_fetch_toppar_next == rktp) {
- /* Update next pointer */
- rd_kafka_broker_fetch_toppar_next(
- rkb, CIRCLEQ_LOOP_NEXT(&rkb->rkb_fetch_toppars,
- rktp, rktp_fetchlink));
- }
-
- rd_rkb_dbg(rkb, TOPIC, "FETCHADD",
- "Removed %.*s [%"PRId32"] from fetch list "
- "(%d entries, opv %d)",
- RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
- rktp->rktp_partition,
- rkb->rkb_fetch_toppar_cnt, rktp->rktp_fetch_version);
-
-}
-
-
-
-/**
* @brief Decide whether this toppar should be on the fetch list or not.
*
* Also:
@@ -1793,9 +1935,9 @@
if (should_fetch) {
rd_dassert(rktp->rktp_fetch_version > 0);
- rd_kafka_broker_fetch_toppar_add(rkb, rktp);
+ rd_kafka_broker_active_toppar_add(rkb, rktp);
} else {
- rd_kafka_broker_fetch_toppar_del(rkb, rktp);
+ rd_kafka_broker_active_toppar_del(rkb, rktp);
/* Non-fetching partitions will have an
* indefinate backoff, unless explicitly specified. */
if (!ts_backoff)
@@ -1927,16 +2069,11 @@
rktp->rktp_partition,
rd_kafka_err2str(rko->rko_err));
- /* Keep on querying until we succeed. */
- rd_kafka_toppar_set_fetch_state(rktp, RD_KAFKA_TOPPAR_FETCH_OFFSET_QUERY);
-
- rd_kafka_toppar_unlock(rktp);
+ /* Keep on querying until we succeed. */
+ rd_kafka_toppar_offset_retry(rktp, 500,
+ "failed to fetch offsets");
+ rd_kafka_toppar_unlock(rktp);
- rd_kafka_timer_start(&rktp->rktp_rkt->rkt_rk->rk_timers,
- &rktp->rktp_offset_query_tmr,
- 500*1000,
- rd_kafka_offset_query_tmr_cb,
- rktp);
/* Propagate error to application */
if (rko->rko_err != RD_KAFKA_RESP_ERR__WAIT_COORD) {
@@ -2218,13 +2355,21 @@
* Propagate error for toppar
*/
void rd_kafka_toppar_enq_error (rd_kafka_toppar_t *rktp,
- rd_kafka_resp_err_t err) {
+ rd_kafka_resp_err_t err,
+ const char *reason) {
rd_kafka_op_t *rko;
+ char buf[512];
rko = rd_kafka_op_new(RD_KAFKA_OP_ERR);
rko->rko_err = err;
rko->rko_rktp = rd_kafka_toppar_keep(rktp);
- rko->rko_u.err.errstr = rd_strdup(rd_kafka_err2str(rko->rko_err));
+
+ rd_snprintf(buf, sizeof(buf), "%.*s [%"PRId32"]: %s (%s)",
+ RD_KAFKAP_STR_PR(rktp->rktp_rkt->rkt_topic),
+ rktp->rktp_partition, reason,
+ rd_kafka_err2str(err));
+
+ rko->rko_u.err.errstr = rd_strdup(buf);
rd_kafka_q_enq(rktp->rktp_fetchq, rko);
}
@@ -3115,16 +3260,16 @@
}
void
-rd_kafka_topic_partition_list_log (rd_kafka_t *rk, const char *fac,
+rd_kafka_topic_partition_list_log (rd_kafka_t *rk, const char *fac, int dbg,
const rd_kafka_topic_partition_list_t *rktparlist) {
int i;
- rd_kafka_dbg(rk, TOPIC, fac, "List with %d partition(s):",
+ rd_kafka_dbg(rk, NONE|dbg, fac, "List with %d partition(s):",
rktparlist->cnt);
for (i = 0 ; i < rktparlist->cnt ; i++) {
const rd_kafka_topic_partition_t *rktpar =
&rktparlist->elems[i];
- rd_kafka_dbg(rk, TOPIC, fac, " %s [%"PRId32"] offset %s%s%s",
+ rd_kafka_dbg(rk, NONE|dbg, fac, " %s [%"PRId32"] offset %s%s%s",
rktpar->topic, rktpar->partition,
rd_kafka_offset2str(rktpar->offset),
rktpar->err ? ": error: " : "",
@@ -3151,8 +3296,8 @@
int r;
if (trunc) {
- if (dest_size > 3)
- rd_snprintf(&dest[dest_size-3], 3, "...");
+ if (dest_size > 4)
+ rd_snprintf(&dest[dest_size-4], 4, "...");
break;
}
@@ -3199,7 +3344,7 @@
* - offset
* - err
*
- * Will only partitions that are in both dst and src, other partitions will
+ * Will only update partitions that are in both dst and src, other partitions will
* remain unchanged.
*/
void
diff -Nru librdkafka-0.11.3/src/rdkafka_partition.h librdkafka-0.11.6/src/rdkafka_partition.h
--- librdkafka-0.11.3/src/rdkafka_partition.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_partition.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_PARTITION_H_
+#define _RDKAFKA_PARTITION_H_
#include "rdkafka_topic.h"
#include "rdkafka_cgrp.h"
@@ -60,7 +61,7 @@
struct rd_kafka_toppar_s { /* rd_kafka_toppar_t */
TAILQ_ENTRY(rd_kafka_toppar_s) rktp_rklink; /* rd_kafka_t link */
TAILQ_ENTRY(rd_kafka_toppar_s) rktp_rkblink; /* rd_kafka_broker_t link*/
- CIRCLEQ_ENTRY(rd_kafka_toppar_s) rktp_fetchlink; /* rkb_fetch_toppars */
+ CIRCLEQ_ENTRY(rd_kafka_toppar_s) rktp_activelink; /* rkb_active_toppars */
TAILQ_ENTRY(rd_kafka_toppar_s) rktp_rktlink; /* rd_kafka_itopic_t link*/
TAILQ_ENTRY(rd_kafka_toppar_s) rktp_cgrplink;/* rd_kafka_cgrp_t link */
rd_kafka_itopic_t *rktp_rkt;
@@ -82,15 +83,15 @@
rd_refcnt_t rktp_refcnt;
mtx_t rktp_lock;
- //LOCK: toppar_lock. Should move the lock inside the msgq instead
//LOCK: toppar_lock. toppar_insert_msg(), concat_msgq()
- //LOCK: toppar_lock. toppar_enq_msg(), deq_msg(), insert_msgq()
+ //LOCK: toppar_lock. toppar_enq_msg(), deq_msg(), toppar_retry_msgq()
int rktp_msgq_wakeup_fd; /* Wake-up fd */
rd_kafka_msgq_t rktp_msgq; /* application->rdkafka queue.
* protected by rktp_lock */
- rd_kafka_msgq_t rktp_xmit_msgq; /* internal broker xmit queue */
+ rd_kafka_msgq_t rktp_xmit_msgq; /* internal broker xmit queue.
+ * local to broker thread. */
- int rktp_fetch; /* On rkb_fetch_toppars list */
+ int rktp_fetch; /* On rkb_active_toppars list */
/* Consumer */
rd_kafka_q_t *rktp_fetchq; /* Queue of fetched messages
@@ -98,6 +99,16 @@
* Broker thread -> App */
rd_kafka_q_t *rktp_ops; /* * -> Main thread */
+ uint64_t rktp_msgseq; /* Current message sequence number.
+ * Each message enqueued on a
+ * non-UA partition will get a
+ * unique sequencial number assigned.
+ * This number is used to
+ * re-enqueue the message
+ * on resends but making sure
+ * the input ordering is still
+ * maintained.
+ * Starts at 1. */
/**
* rktp version barriers
@@ -172,8 +183,11 @@
int64_t rktp_last_next_offset; /* Last next_offset handled
* by fetch_decide().
* Locality: broker thread */
- int64_t rktp_app_offset; /* Last offset delivered to
- * application + 1 */
+ int64_t rktp_app_offset; /* Last offset delivered to
+ * application + 1.
+ * Is reset to INVALID_OFFSET
+ * when partition is
+ * unassigned/stopped. */
int64_t rktp_stored_offset; /* Last stored offset, but
* maybe not committed yet. */
int64_t rktp_committing_offset; /* Offset currently being
@@ -217,8 +231,8 @@
int rktp_flags;
#define RD_KAFKA_TOPPAR_F_DESIRED 0x1 /* This partition is desired
* by a consumer. */
-#define RD_KAFKA_TOPPAR_F_UNKNOWN 0x2 /* Topic is (not yet) seen on
- * a broker. */
+#define RD_KAFKA_TOPPAR_F_UNKNOWN 0x2 /* Topic is not yet or no longer
+ * seen on a broker. */
#define RD_KAFKA_TOPPAR_F_OFFSET_STORE 0x4 /* Offset store is active */
#define RD_KAFKA_TOPPAR_F_OFFSET_STORE_STOPPING 0x8 /* Offset store stopping */
#define RD_KAFKA_TOPPAR_F_APP_PAUSE 0x10 /* App pause()d consumption */
@@ -248,12 +262,15 @@
int rktp_wait_consumer_lag_resp; /* Waiting for consumer lag
* response. */
- struct {
- rd_atomic64_t tx_msgs;
- rd_atomic64_t tx_bytes;
- rd_atomic64_t msgs;
- rd_atomic64_t rx_ver_drops;
- } rktp_c;
+ struct {
+ rd_atomic64_t tx_msgs; /**< Producer: sent messages */
+ rd_atomic64_t tx_msg_bytes; /**< .. bytes */
+ rd_atomic64_t rx_msgs; /**< Consumer: received messages */
+ rd_atomic64_t rx_msg_bytes; /**< .. bytes */
+ rd_atomic64_t producer_enq_msgs; /**< Producer: enqueued msgs */
+ rd_atomic64_t rx_ver_drops; /**< Consumer: outdated message
+ * drops. */
+ } rktp_c;
};
@@ -322,12 +339,21 @@
void rd_kafka_toppar_insert_msg (rd_kafka_toppar_t *rktp, rd_kafka_msg_t *rkm);
void rd_kafka_toppar_enq_msg (rd_kafka_toppar_t *rktp, rd_kafka_msg_t *rkm);
void rd_kafka_toppar_deq_msg (rd_kafka_toppar_t *rktp, rd_kafka_msg_t *rkm);
+int rd_kafka_retry_msgq (rd_kafka_msgq_t *destq,
+ rd_kafka_msgq_t *srcq,
+ int incr_retry, int max_retries, rd_ts_t backoff,
+ int (*cmp) (const void *a, const void *b));
+void rd_kafka_msgq_insert_msgq (rd_kafka_msgq_t *destq,
+ rd_kafka_msgq_t *srcq,
+ int (*cmp) (const void *a, const void *b));
+int rd_kafka_toppar_retry_msgq (rd_kafka_toppar_t *rktp,
+ rd_kafka_msgq_t *rkmq,
+ int incr_retry);
void rd_kafka_toppar_insert_msgq (rd_kafka_toppar_t *rktp,
- rd_kafka_msgq_t *rkmq);
-void rd_kafka_toppar_concat_msgq (rd_kafka_toppar_t *rktp,
- rd_kafka_msgq_t *rkmq);
+ rd_kafka_msgq_t *rkmq);
void rd_kafka_toppar_enq_error (rd_kafka_toppar_t *rktp,
- rd_kafka_resp_err_t err);
+ rd_kafka_resp_err_t err,
+ const char *reason);
shptr_rd_kafka_toppar_t *rd_kafka_toppar_get0 (const char *func, int line,
const rd_kafka_itopic_t *rkt,
int32_t partition,
@@ -354,8 +380,6 @@
void rd_kafka_toppar_desired_unlink (rd_kafka_toppar_t *rktp);
void rd_kafka_toppar_desired_del (rd_kafka_toppar_t *rktp);
-int rd_kafka_toppar_ua_move (rd_kafka_itopic_t *rkt, rd_kafka_msgq_t *rkmq);
-
void rd_kafka_toppar_next_offset_handle (rd_kafka_toppar_t *rktp,
int64_t Offset);
@@ -385,21 +409,6 @@
void rd_kafka_toppar_fetch_stopped (rd_kafka_toppar_t *rktp,
rd_kafka_resp_err_t err);
-/**
- * Updates the current toppar fetch round-robin next pointer.
- */
-static RD_INLINE RD_UNUSED
-void rd_kafka_broker_fetch_toppar_next (rd_kafka_broker_t *rkb,
- rd_kafka_toppar_t *sugg_next) {
- if (CIRCLEQ_EMPTY(&rkb->rkb_fetch_toppars) ||
- (void *)sugg_next == CIRCLEQ_ENDC(&rkb->rkb_fetch_toppars))
- rkb->rkb_fetch_toppar_next = NULL;
- else if (sugg_next)
- rkb->rkb_fetch_toppar_next = sugg_next;
- else
- rkb->rkb_fetch_toppar_next =
- CIRCLEQ_FIRST(&rkb->rkb_fetch_toppars);
-}
rd_ts_t rd_kafka_toppar_fetch_decide (rd_kafka_toppar_t *rktp,
@@ -507,7 +516,7 @@
rd_list_t *topics, int include_regex);
void
-rd_kafka_topic_partition_list_log (rd_kafka_t *rk, const char *fac,
+rd_kafka_topic_partition_list_log (rd_kafka_t *rk, const char *fac, int dbg,
const rd_kafka_topic_partition_list_t *rktparlist);
#define RD_KAFKA_FMT_F_OFFSET 0x1 /* Print offset */
@@ -634,3 +643,5 @@
const struct rd_kafka_partition_leader *a = _a, *b = _b;
return rd_kafka_broker_cmp(a->rkb, b->rkb);
}
+
+#endif /* _RDKAFKA_PARTITION_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_pattern.h librdkafka-0.11.6/src/rdkafka_pattern.h
--- librdkafka-0.11.3/src/rdkafka_pattern.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_pattern.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_PATTERN_H_
+#define _RDKAFKA_PATTERN_H_
#include "rdregex.h"
@@ -63,3 +64,5 @@
int errstr_size);
rd_kafka_pattern_list_t *
rd_kafka_pattern_list_copy (rd_kafka_pattern_list_t *src);
+
+#endif /* _RDKAFKA_PATTERN_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_proto.h librdkafka-0.11.6/src/rdkafka_proto.h
--- librdkafka-0.11.3/src/rdkafka_proto.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_proto.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_PROTO_H_
+#define _RDKAFKA_PROTO_H_
#include "rdendian.h"
@@ -81,7 +82,16 @@
#define RD_KAFKAP_DeleteAcls 31
#define RD_KAFKAP_DescribeConfigs 32
#define RD_KAFKAP_AlterConfigs 33
-#define RD_KAFKAP__NUM 34
+#define RD_KAFKAP_AlterReplicaLogDirs 34
+#define RD_KAFKAP_DescribeLogDirs 35
+#define RD_KAFKAP_SaslAuthenticate 36
+#define RD_KAFKAP_CreatePartitions 37
+#define RD_KAFKAP_CreateDelegationToken 38
+#define RD_KAFKAP_RenewDelegationToken 39
+#define RD_KAFKAP_ExpireDelegationToken 40
+#define RD_KAFKAP_DescribeDelegationToken 41
+#define RD_KAFKAP_DeleteGroups 42
+#define RD_KAFKAP__NUM 43
int16_t ApiVersion;
int32_t CorrId;
/* ClientId follows */
@@ -134,11 +144,22 @@
[RD_KAFKAP_CreateAcls] = "CreateAcls",
[RD_KAFKAP_DeleteAcls] = "DeleteAcls",
[RD_KAFKAP_DescribeConfigs] = "DescribeConfigs",
- [RD_KAFKAP_AlterConfigs] = "AlterConfigs"
+ [RD_KAFKAP_AlterConfigs] = "AlterConfigs",
+ [RD_KAFKAP_AlterReplicaLogDirs] = "AlterReplicaLogDirs",
+ [RD_KAFKAP_DescribeLogDirs] = "DescribeLogDirs",
+ [RD_KAFKAP_SaslAuthenticate] = "SaslAuthenticate",
+ [RD_KAFKAP_CreatePartitions] = "CreatePartitions",
+ [RD_KAFKAP_CreateDelegationToken] = "CreateDelegationToken",
+ [RD_KAFKAP_RenewDelegationToken] = "RenewDelegationToken",
+ [RD_KAFKAP_ExpireDelegationToken] = "ExpireDelegationToken",
+ [RD_KAFKAP_DescribeDelegationToken] = "DescribeDelegationToken",
+ [RD_KAFKAP_DeleteGroups] = "DeleteGroups"
+
};
static RD_TLS char ret[32];
- if (ApiKey < 0 || ApiKey >= (int)RD_ARRAYSIZE(names)) {
+ if (ApiKey < 0 || ApiKey >= (int)RD_ARRAYSIZE(names) ||
+ !names[ApiKey]) {
rd_snprintf(ret, sizeof(ret), "Unknown-%hd?", ApiKey);
return ret;
}
@@ -220,6 +241,8 @@
/* strndup() a Kafka string */
#define RD_KAFKAP_STR_DUP(kstr) rd_strndup((kstr)->str, RD_KAFKAP_STR_LEN(kstr))
+#define RD_KAFKAP_STR_INITIALIZER { .len = RD_KAFKAP_STR_LEN_NULL, .str = NULL }
+
/**
* Frees a Kafka string previously allocated with `rd_kafkap_str_new()`
*/
@@ -496,3 +519,5 @@
#define RD_KAFKAP_MSGSET_V2_OF_BaseTimestamp (8+4+4+1+4+2+4)
#define RD_KAFKAP_MSGSET_V2_OF_MaxTimestamp (8+4+4+1+4+2+4+8)
#define RD_KAFKAP_MSGSET_V2_OF_RecordCount (8+4+4+1+4+2+4+8+8+8+2+4)
+
+#endif /* _RDKAFKA_PROTO_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_queue.c librdkafka-0.11.6/src/rdkafka_queue.c
--- librdkafka-0.11.3/src/rdkafka_queue.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_queue.c 2018-10-10 06:54:38.000000000 +0000
@@ -69,7 +69,8 @@
/**
* Initialize a queue.
*/
-void rd_kafka_q_init (rd_kafka_q_t *rkq, rd_kafka_t *rk) {
+void rd_kafka_q_init0 (rd_kafka_q_t *rkq, rd_kafka_t *rk,
+ const char *func, int line) {
rd_kafka_q_reset(rkq);
rkq->rkq_fwdq = NULL;
rkq->rkq_refcnt = 1;
@@ -80,6 +81,11 @@
rkq->rkq_opaque = NULL;
mtx_init(&rkq->rkq_lock, mtx_plain);
cnd_init(&rkq->rkq_cond);
+#if ENABLE_DEVEL
+ rd_snprintf(rkq->rkq_name, sizeof(rkq->rkq_name), "%s:%d", func, line);
+#else
+ rkq->rkq_name = func;
+#endif
}
@@ -325,16 +331,16 @@
rd_dassert(cb_type);
- if (timeout_ms == RD_POLL_INFINITE)
- timeout_ms = INT_MAX;
-
mtx_lock(&rkq->rkq_lock);
rd_kafka_yield_thread = 0;
if (!(fwdq = rd_kafka_q_fwd_get(rkq, 0))) {
- do {
+ struct timespec timeout_tspec;
+
+ rd_timeout_init_timespec(&timeout_tspec, timeout_ms);
+
+ while (1) {
rd_kafka_op_res_t res;
- rd_ts_t pre;
/* Filter out outdated ops */
retry:
@@ -352,7 +358,8 @@
res = rd_kafka_op_handle(rkq->rkq_rk, rkq, rko,
cb_type, opaque,
callback);
- if (res == RD_KAFKA_OP_RES_HANDLED)
+ if (res == RD_KAFKA_OP_RES_HANDLED ||
+ res == RD_KAFKA_OP_RES_KEEP)
goto retry; /* Next op */
else if (unlikely(res ==
RD_KAFKA_OP_RES_YIELD)) {
@@ -363,24 +370,14 @@
break; /* Proper op, handle below. */
}
- /* No op, wait for one if we have time left */
- if (timeout_ms == RD_POLL_NOWAIT)
- break;
-
- pre = rd_clock();
- if (cnd_timedwait_ms(&rkq->rkq_cond,
- &rkq->rkq_lock,
- timeout_ms) ==
- thrd_timedout) {
+ if (cnd_timedwait_abs(&rkq->rkq_cond,
+ &rkq->rkq_lock,
+ &timeout_tspec) ==
+ thrd_timedout) {
mtx_unlock(&rkq->rkq_lock);
return NULL;
}
- /* Remove spent time */
- timeout_ms -= (int) (rd_clock()-pre) / 1000;
- if (timeout_ms < 0)
- timeout_ms = RD_POLL_NOWAIT;
-
- } while (timeout_ms != RD_POLL_NOWAIT);
+ }
mtx_unlock(&rkq->rkq_lock);
@@ -422,6 +419,7 @@
rd_kafka_q_t localq;
rd_kafka_q_t *fwdq;
int cnt = 0;
+ struct timespec timeout_tspec;
rd_dassert(cb_type);
@@ -439,18 +437,13 @@
return ret;
}
- if (timeout_ms == RD_POLL_INFINITE)
- timeout_ms = INT_MAX;
-
- /* Wait for op */
- while (!(rko = TAILQ_FIRST(&rkq->rkq_q)) && timeout_ms != 0) {
- if (cnd_timedwait_ms(&rkq->rkq_cond,
- &rkq->rkq_lock,
- timeout_ms) != thrd_success)
- break;
+ rd_timeout_init_timespec(&timeout_tspec, timeout_ms);
- timeout_ms = 0;
- }
+ /* Wait for op */
+ while (!(rko = TAILQ_FIRST(&rkq->rkq_q)) &&
+ cnd_timedwait_abs(&rkq->rkq_cond, &rkq->rkq_lock,
+ &timeout_tspec) == thrd_success)
+ ;
if (!rko) {
mtx_unlock(&rkq->rkq_lock);
@@ -513,6 +506,7 @@
rd_kafka_op_t *rko, *next;
rd_kafka_t *rk = rkq->rkq_rk;
rd_kafka_q_t *fwdq;
+ struct timespec timeout_tspec;
mtx_lock(&rkq->rkq_lock);
if ((fwdq = rd_kafka_q_fwd_get(rkq, 0))) {
@@ -526,17 +520,18 @@
}
mtx_unlock(&rkq->rkq_lock);
+ rd_timeout_init_timespec(&timeout_tspec, timeout_ms);
+
rd_kafka_yield_thread = 0;
while (cnt < rkmessages_size) {
rd_kafka_op_res_t res;
mtx_lock(&rkq->rkq_lock);
- while (!(rko = TAILQ_FIRST(&rkq->rkq_q))) {
- if (cnd_timedwait_ms(&rkq->rkq_cond, &rkq->rkq_lock,
- timeout_ms) == thrd_timedout)
- break;
- }
+ while (!(rko = TAILQ_FIRST(&rkq->rkq_q)) &&
+ cnd_timedwait_abs(&rkq->rkq_cond, &rkq->rkq_lock,
+ &timeout_tspec) != thrd_timedout)
+ ;
if (!rko) {
mtx_unlock(&rkq->rkq_lock);
@@ -556,8 +551,9 @@
/* Serve non-FETCH callbacks */
res = rd_kafka_poll_cb(rk, rkq, rko,
RD_KAFKA_Q_CB_RETURN, NULL);
- if (res == RD_KAFKA_OP_RES_HANDLED) {
- /* Callback served, rko is destroyed. */
+ if (res == RD_KAFKA_OP_RES_KEEP ||
+ res == RD_KAFKA_OP_RES_HANDLED) {
+ /* Callback served, rko is destroyed (if HANDLED). */
continue;
} else if (unlikely(res == RD_KAFKA_OP_RES_YIELD ||
rd_kafka_yield_thread)) {
@@ -599,7 +595,10 @@
void rd_kafka_queue_destroy (rd_kafka_queue_t *rkqu) {
- rd_kafka_q_destroy_owner(rkqu->rkqu_q);
+ if (rkqu->rkqu_is_owner)
+ rd_kafka_q_destroy_owner(rkqu->rkqu_q);
+ else
+ rd_kafka_q_destroy(rkqu->rkqu_q);
rd_free(rkqu);
}
@@ -625,6 +624,7 @@
rkqu = rd_kafka_queue_new0(rk, rkq);
rd_kafka_q_destroy(rkq); /* Loose refcount from q_new, one is held
* by queue_new0 */
+ rkqu->rkqu_is_owner = 1;
return rkqu;
}
@@ -665,6 +665,14 @@
return result;
}
+rd_kafka_queue_t *rd_kafka_queue_get_background (rd_kafka_t *rk) {
+ if (rk->rk_background.q)
+ return rd_kafka_queue_new0(rk, rk->rk_background.q);
+ else
+ return NULL;
+}
+
+
rd_kafka_resp_err_t rd_kafka_set_log_queue (rd_kafka_t *rk,
rd_kafka_queue_t *rkqu) {
rd_kafka_q_t *rkq;
@@ -699,6 +707,8 @@
qio->fd = fd;
qio->size = size;
qio->payload = (void *)(qio+1);
+ qio->event_cb = NULL;
+ qio->event_cb_opaque = NULL;
memcpy(qio->payload, payload, size);
}
@@ -723,6 +733,46 @@
/**
+ * @brief Enable or disable(event_cb==NULL) callback-based wake-ups for queue
+ */
+void rd_kafka_q_cb_event_enable (rd_kafka_q_t *rkq,
+ void (*event_cb) (rd_kafka_t *rk,
+ void *opaque),
+ void *opaque) {
+ struct rd_kafka_q_io *qio = NULL;
+
+ if (event_cb) {
+ qio = rd_malloc(sizeof(*qio));
+ qio->fd = -1;
+ qio->size = 0;
+ qio->payload = NULL;
+ qio->event_cb = event_cb;
+ qio->event_cb_opaque = opaque;
+ }
+
+ mtx_lock(&rkq->rkq_lock);
+ if (rkq->rkq_qio) {
+ rd_free(rkq->rkq_qio);
+ rkq->rkq_qio = NULL;
+ }
+
+ if (event_cb) {
+ rkq->rkq_qio = qio;
+ }
+
+ mtx_unlock(&rkq->rkq_lock);
+
+}
+
+void rd_kafka_queue_cb_event_enable (rd_kafka_queue_t *rkqu,
+ void (*event_cb) (rd_kafka_t *rk,
+ void *opaque),
+ void *opaque) {
+ rd_kafka_q_cb_event_enable (rkqu->rkqu_q, event_cb, opaque);
+}
+
+
+/**
* Helper: wait for single op on 'rkq', and return its error,
* or .._TIMED_OUT on timeout.
*/
@@ -864,3 +914,10 @@
mtx_unlock(&rkq->rkq_lock);
}
+
+
+void rd_kafka_enq_once_trigger_destroy (void *ptr) {
+ rd_kafka_enq_once_t *eonce = ptr;
+
+ rd_kafka_enq_once_trigger(eonce, RD_KAFKA_RESP_ERR__DESTROY, "destroy");
+}
diff -Nru librdkafka-0.11.3/src/rdkafka_queue.h librdkafka-0.11.6/src/rdkafka_queue.h
--- librdkafka-0.11.3/src/rdkafka_queue.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_queue.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_QUEUE_H_
+#define _RDKAFKA_QUEUE_H_
#include "rdkafka_op.h"
#include "rdkafka_int.h"
@@ -35,6 +36,9 @@
#include /* for _write() */
#endif
+/** @brief Queueing strategy */
+#define RD_KAFKA_QUEUE_FIFO 0
+#define RD_KAFKA_QUEUE_LIFO 1
TAILQ_HEAD(rd_kafka_op_tailq, rd_kafka_op_s);
@@ -74,11 +78,15 @@
};
-/* FD-based application signalling state holder. */
+/* Application signalling state holder. */
struct rd_kafka_q_io {
+ /* For FD-based signalling */
int fd;
void *payload;
size_t size;
+ /* For callback-based signalling */
+ void (*event_cb) (rd_kafka_t *rk, void *opaque);
+ void *event_cb_opaque;
};
@@ -95,7 +103,9 @@
-void rd_kafka_q_init (rd_kafka_q_t *rkq, rd_kafka_t *rk);
+void rd_kafka_q_init0 (rd_kafka_q_t *rkq, rd_kafka_t *rk,
+ const char *func, int line);
+#define rd_kafka_q_init(rkq,rk) rd_kafka_q_init0(rkq,rk,__FUNCTION__,__LINE__)
rd_kafka_q_t *rd_kafka_q_new0 (rd_kafka_t *rk, const char *func, int line);
#define rd_kafka_q_new(rk) rd_kafka_q_new0(rk,__FUNCTION__,__LINE__)
void rd_kafka_q_destroy_final (rd_kafka_q_t *rkq);
@@ -277,11 +287,12 @@
if (likely(!rkq->rkq_qio))
return;
-#ifdef _MSC_VER
- r = _write(rkq->rkq_qio->fd, rkq->rkq_qio->payload, (int)rkq->rkq_qio->size);
-#else
- r = write(rkq->rkq_qio->fd, rkq->rkq_qio->payload, rkq->rkq_qio->size);
-#endif
+ if (rkq->rkq_qio->event_cb) {
+ rkq->rkq_qio->event_cb(rkq->rkq_rk, rkq->rkq_qio->event_cb_opaque);
+ return;
+ }
+
+ r = rd_write(rkq->rkq_qio->fd, rkq->rkq_qio->payload, (int)rkq->rkq_qio->size);
if (r == -1) {
fprintf(stderr,
"[ERROR:librdkafka:rd_kafka_q_io_event: "
@@ -328,53 +339,80 @@
/**
- * @brief Enqueue the 'rko' op at the tail of the queue 'rkq'.
+ * @brief Enqueue \p rko either at head or tail of \p rkq.
*
- * The provided 'rko' is either enqueued or destroyed.
+ * The provided \p rko is either enqueued or destroyed.
+ *
+ * \p orig_destq is the original (outermost) dest queue for which
+ * this op was enqueued, before any queue forwarding has kicked in.
+ * The rko_serve callback from the orig_destq will be set on the rko
+ * if there is no rko_serve callback already set, and the \p rko isn't
+ * failed because the final queue is disabled.
*
* @returns 1 if op was enqueued or 0 if queue is disabled and
* there was no replyq to enqueue on in which case the rko is destroyed.
*
- * Locality: any thread.
+ * @locality any thread.
*/
static RD_INLINE RD_UNUSED
-int rd_kafka_q_enq (rd_kafka_q_t *rkq, rd_kafka_op_t *rko) {
- rd_kafka_q_t *fwdq;
+int rd_kafka_q_enq1 (rd_kafka_q_t *rkq, rd_kafka_op_t *rko,
+ rd_kafka_q_t *orig_destq, int at_head, int do_lock) {
+ rd_kafka_q_t *fwdq;
- mtx_lock(&rkq->rkq_lock);
+ if (do_lock)
+ mtx_lock(&rkq->rkq_lock);
rd_dassert(rkq->rkq_refcnt > 0);
if (unlikely(!(rkq->rkq_flags & RD_KAFKA_Q_F_READY))) {
-
/* Queue has been disabled, reply to and fail the rko. */
- mtx_unlock(&rkq->rkq_lock);
+ if (do_lock)
+ mtx_unlock(&rkq->rkq_lock);
return rd_kafka_op_reply(rko, RD_KAFKA_RESP_ERR__DESTROY);
}
- if (!rko->rko_serve && rkq->rkq_serve) {
- /* Store original queue's serve callback and opaque
- * prior to forwarding. */
- rko->rko_serve = rkq->rkq_serve;
- rko->rko_serve_opaque = rkq->rkq_opaque;
- }
+ if (!(fwdq = rd_kafka_q_fwd_get(rkq, 0))) {
+ if (!rko->rko_serve && orig_destq->rkq_serve) {
+ /* Store original queue's serve callback and opaque
+ * prior to forwarding. */
+ rko->rko_serve = orig_destq->rkq_serve;
+ rko->rko_serve_opaque = orig_destq->rkq_opaque;
+ }
- if (!(fwdq = rd_kafka_q_fwd_get(rkq, 0))) {
- rd_kafka_q_enq0(rkq, rko, 0);
- cnd_signal(&rkq->rkq_cond);
- if (rkq->rkq_qlen == 1)
- rd_kafka_q_io_event(rkq);
- mtx_unlock(&rkq->rkq_lock);
- } else {
- mtx_unlock(&rkq->rkq_lock);
- rd_kafka_q_enq(fwdq, rko);
- rd_kafka_q_destroy(fwdq);
- }
+ rd_kafka_q_enq0(rkq, rko, at_head);
+ cnd_signal(&rkq->rkq_cond);
+ if (rkq->rkq_qlen == 1)
+ rd_kafka_q_io_event(rkq);
+
+ if (do_lock)
+ mtx_unlock(&rkq->rkq_lock);
+ } else {
+ if (do_lock)
+ mtx_unlock(&rkq->rkq_lock);
+ rd_kafka_q_enq1(fwdq, rko, orig_destq, at_head, 1/*do lock*/);
+ rd_kafka_q_destroy(fwdq);
+ }
return 1;
}
+/**
+ * @brief Enqueue the 'rko' op at the tail of the queue 'rkq'.
+ *
+ * The provided 'rko' is either enqueued or destroyed.
+ *
+ * @returns 1 if op was enqueued or 0 if queue is disabled and
+ * there was no replyq to enqueue on in which case the rko is destroyed.
+ *
+ * @locality any thread.
+ * @locks rkq MUST NOT be locked
+ */
+static RD_INLINE RD_UNUSED
+int rd_kafka_q_enq (rd_kafka_q_t *rkq, rd_kafka_op_t *rko) {
+ return rd_kafka_q_enq1(rkq, rko, rkq, 0/*at tail*/, 1/*do lock*/);
+}
+
/**
* @brief Re-enqueue rko at head of rkq.
@@ -384,37 +422,12 @@
* @returns 1 if op was enqueued or 0 if queue is disabled and
* there was no replyq to enqueue on in which case the rko is destroyed.
*
- * @locks rkq MUST BE LOCKED
- *
- * Locality: any thread.
+ * @locality any thread
+ * @locks rkq MUST BE locked
*/
static RD_INLINE RD_UNUSED
int rd_kafka_q_reenq (rd_kafka_q_t *rkq, rd_kafka_op_t *rko) {
- rd_kafka_q_t *fwdq;
-
- rd_dassert(rkq->rkq_refcnt > 0);
-
- if (unlikely(!(rkq->rkq_flags & RD_KAFKA_Q_F_READY)))
- return rd_kafka_op_reply(rko, RD_KAFKA_RESP_ERR__DESTROY);
-
- if (!rko->rko_serve && rkq->rkq_serve) {
- /* Store original queue's serve callback and opaque
- * prior to forwarding. */
- rko->rko_serve = rkq->rkq_serve;
- rko->rko_serve_opaque = rkq->rkq_opaque;
- }
-
- if (!(fwdq = rd_kafka_q_fwd_get(rkq, 0))) {
- rd_kafka_q_enq0(rkq, rko, 1/*at_head*/);
- cnd_signal(&rkq->rkq_cond);
- if (rkq->rkq_qlen == 1)
- rd_kafka_q_io_event(rkq);
- } else {
- rd_kafka_q_enq(fwdq, rko);
- rd_kafka_q_destroy(fwdq);
- }
-
- return 1;
+ return rd_kafka_q_enq1(rkq, rko, rkq, 1/*at head*/, 0/*don't lock*/);
}
@@ -426,7 +439,6 @@
*/
static RD_INLINE RD_UNUSED
void rd_kafka_q_deq0 (rd_kafka_q_t *rkq, rd_kafka_op_t *rko) {
- rd_dassert(rkq->rkq_flags & RD_KAFKA_Q_F_READY);
rd_dassert(rkq->rkq_qlen > 0 &&
rkq->rkq_qsize >= (int64_t)rko->rko_len);
@@ -680,7 +692,7 @@
/* The replyq queue reference is done after we've enqueued the rko
* so clear it here. */
- replyq->q = NULL;
+ replyq->q = NULL; /* destroyed separately below */
#if ENABLE_DEVEL
if (replyq->_id) {
@@ -756,9 +768,254 @@
struct rd_kafka_queue_s {
rd_kafka_q_t *rkqu_q;
rd_kafka_t *rkqu_rk;
+ int rkqu_is_owner; /**< Is owner/creator of rkqu_q */
};
void rd_kafka_q_dump (FILE *fp, rd_kafka_q_t *rkq);
extern int RD_TLS rd_kafka_yield_thread;
+
+
+
+/**
+ * @name Enqueue op once
+ * @{
+ */
+
+/**
+ * @brief Minimal rd_kafka_op_t wrapper that ensures that
+ * the op is only enqueued on the provided queue once.
+ *
+ * Typical use-case is for an op to be triggered from multiple sources,
+ * but at most once, such as from a timer and some other source.
+ */
+typedef struct rd_kafka_enq_once_s {
+ mtx_t lock;
+ int refcnt;
+ rd_kafka_op_t *rko;
+ rd_kafka_replyq_t replyq;
+} rd_kafka_enq_once_t;
+
+
+/**
+ * @brief Allocate and set up a new eonce and set the initial refcount to 1.
+ * @remark This is to be called by the owner of the rko.
+ */
+static RD_INLINE RD_UNUSED
+rd_kafka_enq_once_t *
+rd_kafka_enq_once_new (rd_kafka_op_t *rko, rd_kafka_replyq_t replyq) {
+ rd_kafka_enq_once_t *eonce = rd_calloc(1, sizeof(*eonce));
+ mtx_init(&eonce->lock, mtx_plain);
+ eonce->rko = rko;
+ eonce->replyq = replyq; /* struct copy */
+ eonce->refcnt = 1;
+ return eonce;
+}
+
+/**
+ * @brief Re-enable triggering of a eonce even after it has been triggered
+ * once.
+ *
+ * @remark This is to be called by the owner.
+ */
+static RD_INLINE RD_UNUSED
+void
+rd_kafka_enq_once_reenable (rd_kafka_enq_once_t *eonce,
+ rd_kafka_op_t *rko, rd_kafka_replyq_t replyq) {
+ mtx_lock(&eonce->lock);
+ eonce->rko = rko;
+ rd_kafka_replyq_destroy(&eonce->replyq);
+ eonce->replyq = replyq; /* struct copy */
+ mtx_unlock(&eonce->lock);
+}
+
+
+/**
+ * @brief Free eonce and its resources. Must only be called with refcnt==0
+ * and eonce->lock NOT held.
+ */
+static RD_INLINE RD_UNUSED
+void rd_kafka_enq_once_destroy0 (rd_kafka_enq_once_t *eonce) {
+ /* This must not be called with the rko or replyq still set, which would
+ * indicate that no enqueueing was performed and that the owner
+ * did not clean up, which is a bug. */
+ rd_assert(!eonce->rko);
+ rd_assert(!eonce->replyq.q);
+#if ENABLE_DEVEL
+ rd_assert(!eonce->replyq._id);
+#endif
+ rd_assert(eonce->refcnt == 0);
+
+ mtx_destroy(&eonce->lock);
+ rd_free(eonce);
+}
+
+
+/**
+ * @brief Increment refcount for source (non-owner), such as a timer.
+ *
+ * @param srcdesc a human-readable descriptive string of the source.
+ * May be used for future debugging.
+ */
+static RD_INLINE RD_UNUSED
+void rd_kafka_enq_once_add_source (rd_kafka_enq_once_t *eonce,
+ const char *srcdesc) {
+ mtx_lock(&eonce->lock);
+ eonce->refcnt++;
+ mtx_unlock(&eonce->lock);
+}
+
+
+/**
+ * @brief Decrement refcount for source (non-owner), such as a timer.
+ *
+ * @param srcdesc a human-readable descriptive string of the source.
+ * May be used for future debugging.
+ *
+ * @remark Must only be called from the owner with the owner
+ * still holding its own refcount.
+ * This API is used to undo an add_source() from the
+ * same code.
+ */
+static RD_INLINE RD_UNUSED
+void rd_kafka_enq_once_del_source (rd_kafka_enq_once_t *eonce,
+ const char *srcdesc) {
+ int do_destroy;
+
+ mtx_lock(&eonce->lock);
+ rd_assert(eonce->refcnt > 1);
+ eonce->refcnt--;
+ do_destroy = eonce->refcnt == 0;
+ mtx_unlock(&eonce->lock);
+
+ if (do_destroy) {
+ /* We're the last refcount holder, clean up eonce. */
+ rd_kafka_enq_once_destroy0(eonce);
+ }
+}
+
+/**
+ * @brief Trigger a source's reference where the eonce resides on
+ * an rd_list_t. This is typically used as a free_cb for
+ * rd_list_destroy() and the trigger error code is
+ * always RD_KAFKA_RESP_ERR__DESTROY.
+ */
+void rd_kafka_enq_once_trigger_destroy (void *ptr);
+
+
+/**
+ * @brief Trigger enqueuing of the rko (unless already enqueued)
+ * and drops the source's refcount.
+ *
+ * @remark Must only be called by sources (non-owner).
+ */
+static RD_INLINE RD_UNUSED
+void rd_kafka_enq_once_trigger (rd_kafka_enq_once_t *eonce,
+ rd_kafka_resp_err_t err,
+ const char *srcdesc) {
+ int do_destroy;
+ rd_kafka_op_t *rko = NULL;
+ rd_kafka_replyq_t replyq = RD_ZERO_INIT;
+
+ mtx_lock(&eonce->lock);
+
+ rd_assert(eonce->refcnt > 0);
+ eonce->refcnt--;
+ do_destroy = eonce->refcnt == 0;
+
+ if (eonce->rko) {
+ /* Not already enqueued, do it.
+ * Detach the rko and replyq from the eonce and unlock the eonce
+ * before enqueuing rko on reply to avoid recursive locks
+ * if the replyq has been disabled and the ops
+ * destructor is called (which might then access the eonce
+ * to clean up). */
+ rko = eonce->rko;
+ replyq = eonce->replyq;
+
+ eonce->rko = NULL;
+ rd_kafka_replyq_clear(&eonce->replyq);
+
+ /* Reply is enqueued at the end of this function */
+ }
+ mtx_unlock(&eonce->lock);
+
+ if (do_destroy) {
+ /* We're the last refcount holder, clean up eonce. */
+ rd_kafka_enq_once_destroy0(eonce);
+ }
+
+ if (rko) {
+ rd_kafka_replyq_enq(&replyq, rko, replyq.version);
+ rd_kafka_replyq_destroy(&replyq);
+ }
+}
+
+/**
+ * @brief Destroy eonce, must only be called by the owner.
+ * There may be outstanding refcounts by non-owners after this call
+ */
+static RD_INLINE RD_UNUSED
+void rd_kafka_enq_once_destroy (rd_kafka_enq_once_t *eonce) {
+ int do_destroy;
+
+ mtx_lock(&eonce->lock);
+ rd_assert(eonce->refcnt > 0);
+ eonce->refcnt--;
+ do_destroy = eonce->refcnt == 0;
+
+ eonce->rko = NULL;
+ rd_kafka_replyq_destroy(&eonce->replyq);
+
+ mtx_unlock(&eonce->lock);
+
+ if (do_destroy) {
+ /* We're the last refcount holder, clean up eonce. */
+ rd_kafka_enq_once_destroy0(eonce);
+ }
+}
+
+
+/**
+ * @brief Disable the owner's eonce, extracting, resetting and returning
+ * the \c rko object.
+ *
+ * This is the same as rd_kafka_enq_once_destroy() but returning
+ * the rko.
+ *
+ * Use this for owner-thread triggering where the enqueuing of the
+ * rko on the replyq is not necessary.
+ *
+ * @returns the eonce's rko object, if still available, else NULL.
+ */
+static RD_INLINE RD_UNUSED
+rd_kafka_op_t *rd_kafka_enq_once_disable (rd_kafka_enq_once_t *eonce) {
+ int do_destroy;
+ rd_kafka_op_t *rko;
+
+ mtx_lock(&eonce->lock);
+ rd_assert(eonce->refcnt > 0);
+ eonce->refcnt--;
+ do_destroy = eonce->refcnt == 0;
+
+ /* May be NULL */
+ rko = eonce->rko;
+ eonce->rko = NULL;
+ rd_kafka_replyq_destroy(&eonce->replyq);
+
+ mtx_unlock(&eonce->lock);
+
+ if (do_destroy) {
+ /* We're the last refcount holder, clean up eonce. */
+ rd_kafka_enq_once_destroy0(eonce);
+ }
+
+ return rko;
+}
+
+
+/**@}*/
+
+
+#endif /* _RDKAFKA_QUEUE_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_request.c librdkafka-0.11.6/src/rdkafka_request.c
--- librdkafka-0.11.3/src/rdkafka_request.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_request.c 2018-10-10 06:54:38.000000000 +0000
@@ -38,6 +38,7 @@
#include "rdkafka_msgset.h"
#include "rdrand.h"
+#include "rdstring.h"
/**
* Kafka protocol request and response handling.
@@ -47,12 +48,26 @@
*/
+/* RD_KAFKA_ERR_ACTION_.. to string map */
+static const char *rd_kafka_actions_descs[] = {
+ "Permanent",
+ "Ignore",
+ "Refresh",
+ "Retry",
+ "Inform",
+ "Special",
+ NULL,
+};
+
+
/**
* @brief Decide action(s) to take based on the returned error code.
*
* The optional var-args is a .._ACTION_END terminated list
* of action,error tuples which overrides the general behaviour.
* It is to be read as: for \p error, return \p action(s).
+ *
+ * @warning \p request, \p rkbuf and \p rkb may be NULL.
*/
int rd_kafka_err_action (rd_kafka_broker_t *rkb,
rd_kafka_resp_err_t err,
@@ -61,6 +76,10 @@
va_list ap;
int actions = 0;
int exp_act;
+ char actstr[64];
+
+ if (!err)
+ return 0;
/* Match explicitly defined error mappings first. */
va_start(ap, request);
@@ -72,15 +91,20 @@
}
va_end(ap);
- if (err && rkb && request)
- rd_rkb_dbg(rkb, BROKER, "REQERR",
- "%sRequest failed: %s: explicit actions 0x%x",
- rd_kafka_ApiKey2str(request->rkbuf_reqhdr.ApiKey),
- rd_kafka_err2str(err), actions);
+ /* Explicit error match. */
+ if (actions) {
+ if (err && rkb && request)
+ rd_rkb_dbg(rkb, BROKER, "REQERR",
+ "%sRequest failed: %s: explicit actions %s",
+ rd_kafka_ApiKey2str(request->rkbuf_reqhdr.
+ ApiKey),
+ rd_kafka_err2str(err),
+ rd_flags2str(actstr, sizeof(actstr),
+ rd_kafka_actions_descs,
+ actions));
- /* Explicit error match. */
- if (actions)
- return actions;
+ return actions;
+ }
/* Default error matching */
switch (err)
@@ -98,8 +122,13 @@
actions |= RD_KAFKA_ERR_ACTION_REFRESH;
break;
case RD_KAFKA_RESP_ERR__TIMED_OUT:
+ case RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE:
+ /* Client-side wait-response/in-queue timeout */
case RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT:
/* Broker-side request handling timeout */
+ case RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS:
+ case RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS_AFTER_APPEND:
+ /* Temporary broker-side problem */
case RD_KAFKA_RESP_ERR__TRANSPORT:
/* Broker connection down */
actions |= RD_KAFKA_ERR_ACTION_RETRY;
@@ -112,6 +141,20 @@
break;
}
+ /* If no request buffer was specified, which might be the case
+ * in certain error call chains, mask out the retry action. */
+ if (!request)
+ actions &= ~RD_KAFKA_ERR_ACTION_RETRY;
+
+
+ if (err && actions && rkb && request)
+ rd_rkb_dbg(rkb, BROKER, "REQERR",
+ "%sRequest failed: %s: actions %s",
+ rd_kafka_ApiKey2str(request->rkbuf_reqhdr.ApiKey),
+ rd_kafka_err2str(err),
+ rd_flags2str(actstr, sizeof(actstr),
+ rd_kafka_actions_descs, actions));
+
return actions;
}
@@ -457,11 +500,12 @@
rd_kafka_cgrp_op(rkb->rkb_rk->rk_cgrp, NULL,
RD_KAFKA_NO_REPLYQ,
RD_KAFKA_OP_COORD_QUERY, err);
- if (request) {
- /* Schedule a retry */
- rd_kafka_buf_keep(request);
- rd_kafka_broker_buf_retry(request->rkbuf_rkb, request);
- }
+ }
+
+ if (actions & RD_KAFKA_ERR_ACTION_RETRY) {
+ if (rd_kafka_buf_retry(rkb, request))
+ return RD_KAFKA_RESP_ERR__IN_PROGRESS;
+ /* FALLTHRU */
}
return err;
@@ -474,13 +518,21 @@
/**
- * opaque=rko wrapper for handle_OffsetFetch.
- * rko->rko_payload MUST be a `rd_kafka_topic_partition_list_t *` which will
- * be filled in with fetch offsets.
+ * @brief Handle OffsetFetch response based on an RD_KAFKA_OP_OFFSET_FETCH
+ * rko in \p opaque.
+ *
+ * @param opaque rko wrapper for handle_OffsetFetch.
+ *
+ * The \c rko->rko_u.offset_fetch.partitions list will be filled in with
+ * the fetched offsets.
*
* A reply will be sent on 'rko->rko_replyq' with type RD_KAFKA_OP_OFFSET_FETCH.
*
- * Locality: cgrp's broker thread
+ * @remark \p rkb, \p rkbuf and \p request are optional.
+ *
+ * @remark The \p request buffer may be retried on error.
+ *
+ * @locality cgrp's broker thread
*/
void rd_kafka_op_handle_OffsetFetch (rd_kafka_t *rk,
rd_kafka_broker_t *rkb,
@@ -503,6 +555,19 @@
offsets = rd_kafka_topic_partition_list_copy(
rko->rko_u.offset_fetch.partitions);
+ /* If all partitions already had usable offsets then there
+ * was no request sent and thus no reply, the offsets list is
+ * good to go.. */
+ if (rkbuf) {
+ /* ..else parse the response (or perror) */
+ err = rd_kafka_handle_OffsetFetch(rkb->rkb_rk, rkb, err, rkbuf,
+ request, offsets, 0);
+ if (err == RD_KAFKA_RESP_ERR__IN_PROGRESS) {
+ rd_kafka_topic_partition_list_destroy(offsets);
+ return; /* Retrying */
+ }
+ }
+
rko_reply = rd_kafka_op_new(RD_KAFKA_OP_OFFSET_FETCH|RD_KAFKA_OP_REPLY);
rko_reply->rko_err = err;
rko_reply->rko_u.offset_fetch.partitions = offsets;
@@ -511,13 +576,6 @@
rko_reply->rko_rktp = rd_kafka_toppar_keep(
rd_kafka_toppar_s2i(rko->rko_rktp));
- /* If all partitions already had usable offsets then there
- * was no request sent and thus no reply, the offsets list is
- * good to go. */
- if (rkbuf)
- rd_kafka_handle_OffsetFetch(rkb->rkb_rk, rkb, err, rkbuf,
- request, offsets, 0);
-
rd_kafka_replyq_enq(&rko->rko_replyq, rko_reply, 0);
rd_kafka_op_destroy(rko);
@@ -626,6 +684,9 @@
return;
}
+ rd_rkb_dbg(rkb, CGRP|RD_KAFKA_DBG_CONSUMER, "OFFSET",
+ "Fetch committed offsets for %d/%d partition(s)",
+ tot_PartCnt, parts->cnt);
rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque);
@@ -974,9 +1035,11 @@
/* This is a blocking request */
rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_BLOCKING;
- rkbuf->rkbuf_ts_timeout = rd_clock() +
- (rkb->rkb_rk->rk_conf.group_session_timeout_ms * 1000) +
- (3*1000*1000/* 3s grace period*/);
+ rd_kafka_buf_set_abs_timeout(
+ rkbuf,
+ rkb->rkb_rk->rk_conf.group_session_timeout_ms +
+ 3000/* 3s grace period*/,
+ 0);
rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque);
}
@@ -1026,6 +1089,12 @@
/* FALLTHRU */
}
+ if (actions & RD_KAFKA_ERR_ACTION_RETRY) {
+ if (rd_kafka_buf_retry(rkb, request))
+ return;
+ /* FALLTHRU */
+ }
+
rd_kafka_dbg(rkb->rkb_rk, CGRP, "SYNCGROUP",
"SyncGroup response: %s (%d bytes of MemberState data)",
rd_kafka_err2str(ErrorCode),
@@ -1086,9 +1155,11 @@
/* This is a blocking request */
rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_BLOCKING;
- rkbuf->rkbuf_ts_timeout = rd_clock() +
- (rk->rk_conf.group_session_timeout_ms * 1000) +
- (3*1000*1000/* 3s grace period*/);
+ rd_kafka_buf_set_abs_timeout(
+ rkbuf,
+ rk->rk_conf.group_session_timeout_ms +
+ 3000/* 3s grace period*/,
+ 0);
rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque);
}
@@ -1142,7 +1213,6 @@
rd_kafka_buf_read_i16(rkbuf, &ErrorCode);
-
err:
actions = rd_kafka_err_action(rkb, ErrorCode, rkbuf, request,
RD_KAFKA_ERR_ACTION_END);
@@ -1151,10 +1221,12 @@
/* Re-query for coordinator */
rd_kafka_cgrp_op(rkcg, NULL, RD_KAFKA_NO_REPLYQ,
RD_KAFKA_OP_COORD_QUERY, ErrorCode);
- /* Schedule a retry */
- rd_kafka_buf_keep(request);
- rd_kafka_broker_buf_retry(request->rkbuf_rkb, request);
- return;
+ }
+
+ if (actions & RD_KAFKA_ERR_ACTION_RETRY) {
+ if (rd_kafka_buf_retry(rkb, request))
+ return;
+ /* FALLTHRU */
}
if (ErrorCode)
@@ -1162,6 +1234,8 @@
"LeaveGroup response: %s",
rd_kafka_err2str(ErrorCode));
+ return;
+
err_parse:
ErrorCode = rkbuf->rkbuf_err;
goto err;
@@ -1198,8 +1272,10 @@
rd_kafka_buf_write_i32(rkbuf, generation_id);
rd_kafka_buf_write_kstr(rkbuf, member_id);
- rkbuf->rkbuf_ts_timeout = rd_clock() +
- (rkb->rkb_rk->rk_conf.group_session_timeout_ms * 1000);
+ rd_kafka_buf_set_abs_timeout(
+ rkbuf,
+ rkb->rkb_rk->rk_conf.group_session_timeout_ms,
+ 0);
rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque);
}
@@ -1259,45 +1335,36 @@
rd_kafka_op_t *rko = opaque; /* Possibly NULL */
struct rd_kafka_metadata *md = NULL;
const rd_list_t *topics = request->rkbuf_u.Metadata.topics;
+ char actstr[64];
+ int actions;
rd_kafka_assert(NULL, err == RD_KAFKA_RESP_ERR__DESTROY ||
thrd_is_current(rk->rk_thread));
- /* Avoid metadata updates when we're terminating. */
- if (rd_kafka_terminating(rkb->rkb_rk))
- err = RD_KAFKA_RESP_ERR__DESTROY;
-
- if (unlikely(err)) {
- if (err == RD_KAFKA_RESP_ERR__DESTROY) {
- /* Terminating */
- goto done;
- }
+ /* Avoid metadata updates when we're terminating. */
+ if (rd_kafka_terminating(rkb->rkb_rk) ||
+ err == RD_KAFKA_RESP_ERR__DESTROY) {
+ /* Terminating */
+ goto done;
+ }
- /* FIXME: handle errors */
- rd_rkb_log(rkb, LOG_WARNING, "METADATA",
- "Metadata request failed: %s (%dms)",
- rd_kafka_err2str(err),
- (int)(request->rkbuf_ts_sent/1000));
- } else {
+ if (err)
+ goto err;
- if (!topics)
- rd_rkb_dbg(rkb, METADATA, "METADATA",
- "===== Received metadata: %s =====",
- request->rkbuf_u.Metadata.reason);
- else
- rd_rkb_dbg(rkb, METADATA, "METADATA",
- "===== Received metadata "
- "(for %d requested topics): %s =====",
- rd_list_cnt(topics),
- request->rkbuf_u.Metadata.reason);
-
- md = rd_kafka_parse_Metadata(rkb, request, rkbuf);
- if (!md) {
- if (rd_kafka_buf_retry(rkb, request))
- return;
- err = RD_KAFKA_RESP_ERR__BAD_MSG;
- }
- }
+ if (!topics)
+ rd_rkb_dbg(rkb, METADATA, "METADATA",
+ "===== Received metadata: %s =====",
+ request->rkbuf_u.Metadata.reason);
+ else
+ rd_rkb_dbg(rkb, METADATA, "METADATA",
+ "===== Received metadata "
+ "(for %d requested topics): %s =====",
+ rd_list_cnt(topics),
+ request->rkbuf_u.Metadata.reason);
+
+ err = rd_kafka_parse_Metadata(rkb, request, rkbuf, &md);
+ if (err)
+ goto err;
if (rko && rko->rko_replyq.q) {
/* Reply to metadata requester, passing on the metadata.
@@ -1312,13 +1379,42 @@
rd_free(md);
}
+ goto done;
+
+ err:
+ actions = rd_kafka_err_action(
+ rkb, err, rkbuf, request,
+
+ RD_KAFKA_ERR_ACTION_RETRY,
+ RD_KAFKA_RESP_ERR__PARTIAL,
+
+ RD_KAFKA_ERR_ACTION_END);
+
+ if (actions & RD_KAFKA_ERR_ACTION_RETRY) {
+ if (rd_kafka_buf_retry(rkb, request))
+ return;
+ /* FALLTHRU */
+ } else {
+ rd_rkb_log(rkb, LOG_WARNING, "METADATA",
+ "Metadata request failed: %s: %s (%dms): %s",
+ request->rkbuf_u.Metadata.reason,
+ rd_kafka_err2str(err),
+ (int)(request->rkbuf_ts_sent/1000),
+ rd_flags2str(actstr, sizeof(actstr),
+ rd_kafka_actions_descs,
+ actions));
+ }
+
+
+
+ /* FALLTHRU */
+
done:
if (rko)
rd_kafka_op_destroy(rko);
}
-
/**
* @brief Construct MetadataRequest (does not send)
*
@@ -1477,7 +1573,6 @@
struct rd_kafka_ApiVersion **apis,
size_t *api_cnt) {
const int log_decode_errors = LOG_ERR;
- int actions;
int32_t ApiArrayCnt;
int16_t ErrorCode;
int i = 0;
@@ -1526,17 +1621,9 @@
if (*apis)
rd_free(*apis);
- actions = rd_kafka_err_action(
- rkb, err, rkbuf, request,
- RD_KAFKA_ERR_ACTION_END);
-
- if (actions & RD_KAFKA_ERR_ACTION_RETRY) {
- if (rd_kafka_buf_retry(rkb, request))
- return RD_KAFKA_RESP_ERR__IN_PROGRESS;
- /* FALLTHRU */
- }
+ /* There are no retryable errors. */
-done:
+ done:
return err;
}
@@ -1562,7 +1649,10 @@
/* 0.9.0.x brokers will not close the connection on unsupported
* API requests, so we minimize the timeout for the request.
* This is a regression on the broker part. */
- rkbuf->rkbuf_ts_timeout = rd_clock() + (rkb->rkb_rk->rk_conf.api_version_request_timeout_ms * 1000);
+ rd_kafka_buf_set_abs_timeout(
+ rkbuf,
+ rkb->rkb_rk->rk_conf.api_version_request_timeout_ms,
+ 0);
if (replyq.q)
rd_kafka_broker_buf_enq_replyq(rkb,
@@ -1596,9 +1686,9 @@
/* 0.9.0.x brokers will not close the connection on unsupported
* API requests, so we minimize the timeout of the request.
* This is a regression on the broker part. */
- if (!rkb->rkb_rk->rk_conf.api_version_request &&
+ if (!rkb->rkb_rk->rk_conf.api_version_request &&
rkb->rkb_rk->rk_conf.socket_timeout_ms > 10*1000)
- rkbuf->rkbuf_ts_timeout = rd_clock() + (10 * 1000 * 1000);
+ rd_kafka_buf_set_abs_timeout(rkbuf, 10*1000 /*10s*/, 0);
if (replyq.q)
rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq,
@@ -1702,11 +1792,12 @@
"%s [%"PRId32"]: MessageSet with %i message(s) "
"delivered",
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
- rd_atomic32_get(&request->rkbuf_msgq.rkmq_msg_cnt));
+ request->rkbuf_msgq.rkmq_msg_cnt);
} else {
/* Error */
int actions;
+ char actstr[64];
if (err == RD_KAFKA_RESP_ERR__DESTROY)
goto done; /* Terminating */
@@ -1720,51 +1811,90 @@
RD_KAFKA_ERR_ACTION_REFRESH,
RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART,
+ RD_KAFKA_ERR_ACTION_RETRY,
+ RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS,
+
+ RD_KAFKA_ERR_ACTION_RETRY,
+ RD_KAFKA_RESP_ERR_NOT_ENOUGH_REPLICAS_AFTER_APPEND,
+
+ RD_KAFKA_ERR_ACTION_RETRY,
+ RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE,
+
+ RD_KAFKA_ERR_ACTION_RETRY,
+ RD_KAFKA_RESP_ERR__TIMED_OUT,
+
+ RD_KAFKA_ERR_ACTION_PERMANENT,
+ RD_KAFKA_RESP_ERR__MSG_TIMED_OUT,
+
RD_KAFKA_ERR_ACTION_END);
rd_rkb_dbg(rkb, MSG, "MSGSET",
"%s [%"PRId32"]: MessageSet with %i message(s) "
- "encountered error: %s (actions 0x%x)",
+ "encountered error: %s (actions %s)",
rktp->rktp_rkt->rkt_topic->str, rktp->rktp_partition,
- rd_atomic32_get(&request->rkbuf_msgq.rkmq_msg_cnt),
- rd_kafka_err2str(err), actions);
+ request->rkbuf_msgq.rkmq_msg_cnt,
+ rd_kafka_err2str(err),
+ rd_flags2str(actstr, sizeof(actstr),
+ rd_kafka_actions_descs,
+ actions));
+
+
+ if (actions & (RD_KAFKA_ERR_ACTION_REFRESH |
+ RD_KAFKA_ERR_ACTION_RETRY)) {
+ /* Retry */
+ int incr_retry = 1; /* Increase per-message retry cnt */
+
+ if (actions & RD_KAFKA_ERR_ACTION_REFRESH) {
+ /* Request metadata information update.
+ * These errors imply that we have stale
+ * information and the request was
+ * either rejected or not sent -
+ * we don't need to increment the retry count
+ * when we perform a retry since:
+ * - it is a temporary error (hopefully)
+ * - there is no chance of duplicate delivery
+ */
+ rd_kafka_toppar_leader_unavailable(
+ rktp, "produce", err);
+
+ /* We can't be certain the request wasn't
+ * sent in case of transport failure,
+ * so the ERR__TRANSPORT case will need
+ * the retry count to be increased */
+ if (err != RD_KAFKA_RESP_ERR__TRANSPORT)
+ incr_retry = 0;
+ }
- /* NOTE: REFRESH implies a later retry, which does NOT affect
- * the retry count since refresh-errors are considered
- * to be stale metadata rather than temporary errors.
- *
- * This is somewhat problematic since it may cause
- * duplicate messages even with retries=0 if the
- * ProduceRequest made it to the broker but only the
- * response was lost due to network connectivity issues.
- * That problem will be sorted when EoS is implemented.
- */
- if (actions & RD_KAFKA_ERR_ACTION_REFRESH) {
- /* Request metadata information update */
- rd_kafka_toppar_leader_unavailable(rktp,
- "produce", err);
-
- /* Move messages (in the rkbuf) back to the partition's
- * queue head. They will be resent when a new leader
- * is delegated. */
- rd_kafka_toppar_insert_msgq(rktp, &request->rkbuf_msgq);
-
- /* No need for fallthru here since the request
- * no longer has any messages associated with it. */
- goto done;
+ /* If message timed out in queue, not in transit,
+ * we will retry at a later time but not increment
+ * the retry count since there is no risk
+ * of duplicates. */
+ if (!rd_kafka_buf_was_sent(request))
+ incr_retry = 0;
+
+ /* Since requests are specific to a broker
+ * we move the retryable messages from the request
+ * back to the partition queue (prepend) and then
+ * let the new broker construct a new request.
+ * While doing this we also make sure the retry count
+ * for each message is honoured, any messages that
+ * would exceeded the retry count will not be
+ * moved but instead fail below. */
+ rd_kafka_toppar_retry_msgq(rktp, &request->rkbuf_msgq,
+ incr_retry);
+
+ if (rd_kafka_msgq_len(&request->rkbuf_msgq) == 0) {
+ /* No need do anything more with the request
+ * here since the request no longer has any
+ messages associated with it. */
+ goto done;
+ }
}
- if ((actions & RD_KAFKA_ERR_ACTION_RETRY) &&
- rd_kafka_buf_retry(rkb, request))
- return; /* Scheduled for retry */
-
- /* Refresh implies a later retry through other means */
- if (actions & RD_KAFKA_ERR_ACTION_REFRESH)
- goto done;
-
/* Translate request-level timeout error code
* to message-level timeout error code. */
- if (err == RD_KAFKA_RESP_ERR__TIMED_OUT)
+ if (err == RD_KAFKA_RESP_ERR__TIMED_OUT ||
+ err == RD_KAFKA_RESP_ERR__TIMED_OUT_QUEUE)
err = RD_KAFKA_RESP_ERR__MSG_TIMED_OUT;
/* Fatal errors: no message transmission retries */
@@ -1772,7 +1902,7 @@
}
/* Propagate assigned offset and timestamp back to app. */
- if (likely(offset != RD_KAFKA_OFFSET_INVALID)) {
+ if (likely(!err && offset != RD_KAFKA_OFFSET_INVALID)) {
rd_kafka_msg_t *rkm;
if (rktp->rktp_rkt->rkt_conf.produce_offset_report) {
/* produce.offset.report: each message */
@@ -1789,8 +1919,7 @@
rkm = TAILQ_LAST(&request->rkbuf_msgq.rkmq_msgs,
rd_kafka_msg_head_s);
rkm->rkm_offset = offset +
- rd_atomic32_get(&request->rkbuf_msgq.
- rkmq_msg_cnt) - 1;
+ request->rkbuf_msgq.rkmq_msg_cnt - 1;
if (timestamp != -1) {
rkm->rkm_timestamp = timestamp;
rkm->rkm_tstype = RD_KAFKA_MSG_ATTR_LOG_APPEND_TIME;
@@ -1818,6 +1947,9 @@
rd_kafka_itopic_t *rkt = rktp->rktp_rkt;
size_t MessageSetSize = 0;
int cnt;
+ rd_ts_t now;
+ int64_t first_msg_timeout;
+ int tmout;
/**
* Create ProduceRequest with as many messages from the toppar
@@ -1828,18 +1960,32 @@
if (unlikely(!rkbuf))
return 0;
- cnt = rd_atomic32_get(&rkbuf->rkbuf_msgq.rkmq_msg_cnt);
+ cnt = rkbuf->rkbuf_msgq.rkmq_msg_cnt;
rd_dassert(cnt > 0);
- rd_atomic64_add(&rktp->rktp_c.tx_msgs, cnt);
- rd_atomic64_add(&rktp->rktp_c.tx_bytes, MessageSetSize);
+ rd_avg_add(&rktp->rktp_rkt->rkt_avg_batchcnt, (int64_t)cnt);
+ rd_avg_add(&rktp->rktp_rkt->rkt_avg_batchsize, (int64_t)MessageSetSize);
if (!rkt->rkt_conf.required_acks)
rkbuf->rkbuf_flags |= RD_KAFKA_OP_F_NO_RESPONSE;
- /* Use timeout from first message. */
- rkbuf->rkbuf_ts_timeout =
- TAILQ_FIRST(&rkbuf->rkbuf_msgq.rkmq_msgs)->rkm_ts_timeout;
+ /* Use timeout from first message in batch */
+ now = rd_clock();
+ first_msg_timeout = (TAILQ_FIRST(&rkbuf->rkbuf_msgq.rkmq_msgs)->
+ rkm_ts_timeout - now) / 1000;
+
+ if (unlikely(first_msg_timeout <= 0)) {
+ /* Message has already timed out, allow 100 ms
+ * to produce anyway */
+ tmout = 100;
+ } else {
+ tmout = (int)first_msg_timeout;
+ }
+
+ /* Set absolute timeout (including retries), the
+ * effective timeout for this specific request will be
+ * capped by socket.timeout.ms */
+ rd_kafka_buf_set_abs_timeout(rkbuf, tmout, now);
rd_kafka_broker_buf_enq_replyq(rkb, rkbuf,
RD_KAFKA_NO_REPLYQ,
@@ -1849,3 +1995,518 @@
return cnt;
}
+
+
+/**
+ * @brief Construct and send CreateTopicsRequest to \p rkb
+ * with the topics (NewTopic_t*) in \p new_topics, using
+ * \p options.
+ *
+ * The response (unparsed) will be enqueued on \p replyq
+ * for handling by \p resp_cb (with \p opaque passed).
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR if the request was enqueued for
+ * transmission, otherwise an error code and errstr will be
+ * updated with a human readable error string.
+ */
+rd_kafka_resp_err_t
+rd_kafka_CreateTopicsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *new_topics /*(NewTopic_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque) {
+ rd_kafka_buf_t *rkbuf;
+ int16_t ApiVersion = 0;
+ int features;
+ int i = 0;
+ rd_kafka_NewTopic_t *newt;
+ int op_timeout;
+
+ if (rd_list_cnt(new_topics) == 0) {
+ rd_snprintf(errstr, errstr_size, "No topics to create");
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ ApiVersion = rd_kafka_broker_ApiVersion_supported(
+ rkb, RD_KAFKAP_CreateTopics, 0, 2, &features);
+ if (ApiVersion == -1) {
+ rd_snprintf(errstr, errstr_size,
+ "Topic Admin API (KIP-4) not supported "
+ "by broker, requires broker version >= 0.10.2.0");
+ return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE;
+ }
+
+ if (rd_kafka_confval_get_int(&options->validate_only) &&
+ ApiVersion < 1) {
+ rd_snprintf(errstr, errstr_size,
+ "CreateTopics.validate_only=true not "
+ "supported by broker");
+ return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE;
+ }
+
+
+
+ rkbuf = rd_kafka_buf_new_request(rkb, RD_KAFKAP_CreateTopics, 1,
+ 4 +
+ (rd_list_cnt(new_topics) * 200) +
+ 4 + 1);
+
+ /* #topics */
+ rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(new_topics));
+
+ while ((newt = rd_list_elem(new_topics, i++))) {
+ int partition;
+ int ei = 0;
+ const rd_kafka_ConfigEntry_t *entry;
+
+ /* topic */
+ rd_kafka_buf_write_str(rkbuf, newt->topic, -1);
+
+ if (rd_list_cnt(&newt->replicas)) {
+ /* num_partitions and replication_factor must be
+ * set to -1 if a replica assignment is sent. */
+ /* num_partitions */
+ rd_kafka_buf_write_i32(rkbuf, -1);
+ /* replication_factor */
+ rd_kafka_buf_write_i16(rkbuf, -1);
+ } else {
+ /* num_partitions */
+ rd_kafka_buf_write_i32(rkbuf, newt->num_partitions);
+ /* replication_factor */
+ rd_kafka_buf_write_i16(rkbuf,
+ (int16_t)newt->
+ replication_factor);
+ }
+
+ /* #replica_assignment */
+ rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(&newt->replicas));
+
+ /* Replicas per partition, see rdkafka_admin.[ch]
+ * for how these are constructed. */
+ for (partition = 0 ; partition < rd_list_cnt(&newt->replicas);
+ partition++) {
+ const rd_list_t *replicas;
+ int ri = 0;
+
+ replicas = rd_list_elem(&newt->replicas, partition);
+ if (!replicas)
+ continue;
+
+ /* partition */
+ rd_kafka_buf_write_i32(rkbuf, partition);
+ /* #replicas */
+ rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(replicas));
+
+ for (ri = 0 ; ri < rd_list_cnt(replicas) ; ri++) {
+ /* replica */
+ rd_kafka_buf_write_i32(
+ rkbuf, rd_list_get_int32(replicas, ri));
+ }
+ }
+
+ /* #config_entries */
+ rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(&newt->config));
+
+ RD_LIST_FOREACH(entry, &newt->config, ei) {
+ /* config_name */
+ rd_kafka_buf_write_str(rkbuf, entry->kv->name, -1);
+ /* config_value (nullable) */
+ rd_kafka_buf_write_str(rkbuf, entry->kv->value, -1);
+ }
+ }
+
+ /* timeout */
+ op_timeout = rd_kafka_confval_get_int(&options->operation_timeout);
+ rd_kafka_buf_write_i32(rkbuf, op_timeout);
+
+ if (op_timeout > rkb->rkb_rk->rk_conf.socket_timeout_ms)
+ rd_kafka_buf_set_abs_timeout(rkbuf, op_timeout+1000, 0);
+
+ if (ApiVersion >= 1) {
+ /* validate_only */
+ rd_kafka_buf_write_i8(rkbuf,
+ rd_kafka_confval_get_int(&options->
+ validate_only));
+ }
+
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, 0);
+
+ rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+/**
+ * @brief Construct and send DeleteTopicsRequest to \p rkb
+ * with the topics (DeleteTopic_t *) in \p del_topics, using
+ * \p options.
+ *
+ * The response (unparsed) will be enqueued on \p replyq
+ * for handling by \p resp_cb (with \p opaque passed).
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR if the request was enqueued for
+ * transmission, otherwise an error code and errstr will be
+ * updated with a human readable error string.
+ */
+rd_kafka_resp_err_t
+rd_kafka_DeleteTopicsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *del_topics /*(DeleteTopic_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque) {
+ rd_kafka_buf_t *rkbuf;
+ int16_t ApiVersion = 0;
+ int features;
+ int i = 0;
+ rd_kafka_DeleteTopic_t *delt;
+ int op_timeout;
+
+ if (rd_list_cnt(del_topics) == 0) {
+ rd_snprintf(errstr, errstr_size, "No topics to delete");
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ ApiVersion = rd_kafka_broker_ApiVersion_supported(
+ rkb, RD_KAFKAP_DeleteTopics, 0, 1, &features);
+ if (ApiVersion == -1) {
+ rd_snprintf(errstr, errstr_size,
+ "Topic Admin API (KIP-4) not supported "
+ "by broker, requires broker version >= 0.10.2.0");
+ return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE;
+ }
+
+ rkbuf = rd_kafka_buf_new_request(rkb, RD_KAFKAP_DeleteTopics, 1,
+ /* FIXME */
+ 4 +
+ (rd_list_cnt(del_topics) * 100) +
+ 4);
+
+ /* #topics */
+ rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(del_topics));
+
+ while ((delt = rd_list_elem(del_topics, i++)))
+ rd_kafka_buf_write_str(rkbuf, delt->topic, -1);
+
+ /* timeout */
+ op_timeout = rd_kafka_confval_get_int(&options->operation_timeout);
+ rd_kafka_buf_write_i32(rkbuf, op_timeout);
+
+ if (op_timeout > rkb->rkb_rk->rk_conf.socket_timeout_ms)
+ rd_kafka_buf_set_abs_timeout(rkbuf, op_timeout+1000, 0);
+
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, 0);
+
+ rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+
+/**
+ * @brief Construct and send CreatePartitionsRequest to \p rkb
+ * with the topics (NewPartitions_t*) in \p new_parts, using
+ * \p options.
+ *
+ * The response (unparsed) will be enqueued on \p replyq
+ * for handling by \p resp_cb (with \p opaque passed).
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR if the request was enqueued for
+ * transmission, otherwise an error code and errstr will be
+ * updated with a human readable error string.
+ */
+rd_kafka_resp_err_t
+rd_kafka_CreatePartitionsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *new_parts /*(NewPartitions_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque) {
+ rd_kafka_buf_t *rkbuf;
+ int16_t ApiVersion = 0;
+ int i = 0;
+ rd_kafka_NewPartitions_t *newp;
+ int op_timeout;
+
+ if (rd_list_cnt(new_parts) == 0) {
+ rd_snprintf(errstr, errstr_size, "No partitions to create");
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ ApiVersion = rd_kafka_broker_ApiVersion_supported(
+ rkb, RD_KAFKAP_CreatePartitions, 0, 0, NULL);
+ if (ApiVersion == -1) {
+ rd_snprintf(errstr, errstr_size,
+ "CreatePartitions (KIP-195) not supported "
+ "by broker, requires broker version >= 1.0.0");
+ return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE;
+ }
+
+ rkbuf = rd_kafka_buf_new_request(rkb, RD_KAFKAP_CreatePartitions, 1,
+ 4 +
+ (rd_list_cnt(new_parts) * 200) +
+ 4 + 1);
+
+ /* #topics */
+ rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(new_parts));
+
+ while ((newp = rd_list_elem(new_parts, i++))) {
+ /* topic */
+ rd_kafka_buf_write_str(rkbuf, newp->topic, -1);
+
+ /* New partition count */
+ rd_kafka_buf_write_i32(rkbuf, (int32_t)newp->total_cnt);
+
+ /* #replica_assignment */
+ if (rd_list_empty(&newp->replicas)) {
+ rd_kafka_buf_write_i32(rkbuf, -1);
+ } else {
+ const rd_list_t *replicas;
+ int pi = -1;
+
+ rd_kafka_buf_write_i32(rkbuf,
+ rd_list_cnt(&newp->replicas));
+
+ while ((replicas = rd_list_elem(&newp->replicas,
+ ++pi))) {
+ int ri = 0;
+
+ /* replica count */
+ rd_kafka_buf_write_i32(rkbuf,
+ rd_list_cnt(replicas));
+
+ /* replica */
+ for (ri = 0 ; ri < rd_list_cnt(replicas) ;
+ ri++) {
+ rd_kafka_buf_write_i32(
+ rkbuf,
+ rd_list_get_int32(replicas,
+ ri));
+ }
+ }
+ }
+ }
+
+ /* timeout */
+ op_timeout = rd_kafka_confval_get_int(&options->operation_timeout);
+ rd_kafka_buf_write_i32(rkbuf, op_timeout);
+
+ if (op_timeout > rkb->rkb_rk->rk_conf.socket_timeout_ms)
+ rd_kafka_buf_set_abs_timeout(rkbuf, op_timeout+1000, 0);
+
+ /* validate_only */
+ rd_kafka_buf_write_i8(
+ rkbuf, rd_kafka_confval_get_int(&options->validate_only));
+
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, 0);
+
+ rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+/**
+ * @brief Construct and send AlterConfigsRequest to \p rkb
+ * with the configs (ConfigResource_t*) in \p configs, using
+ * \p options.
+ *
+ * The response (unparsed) will be enqueued on \p replyq
+ * for handling by \p resp_cb (with \p opaque passed).
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR if the request was enqueued for
+ * transmission, otherwise an error code and errstr will be
+ * updated with a human readable error string.
+ */
+rd_kafka_resp_err_t
+rd_kafka_AlterConfigsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *configs /*(ConfigResource_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque) {
+ rd_kafka_buf_t *rkbuf;
+ int16_t ApiVersion = 0;
+ int i;
+ const rd_kafka_ConfigResource_t *config;
+ int op_timeout;
+
+ if (rd_list_cnt(configs) == 0) {
+ rd_snprintf(errstr, errstr_size,
+ "No config resources specified");
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ ApiVersion = rd_kafka_broker_ApiVersion_supported(
+ rkb, RD_KAFKAP_AlterConfigs, 0, 0, NULL);
+ if (ApiVersion == -1) {
+ rd_snprintf(errstr, errstr_size,
+ "AlterConfigs (KIP-133) not supported "
+ "by broker, requires broker version >= 0.11.0");
+ return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE;
+ }
+
+ /* incremental requires ApiVersion > FIXME */
+ if (ApiVersion < 1 /* FIXME */ &&
+ rd_kafka_confval_get_int(&options->incremental)) {
+ rd_snprintf(errstr, errstr_size,
+ "AlterConfigs.incremental=true (KIP-248) "
+ "not supported by broker, "
+ "requires broker version >= 2.0.0");
+ return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE;
+ }
+
+ rkbuf = rd_kafka_buf_new_request(rkb, RD_KAFKAP_AlterConfigs, 1,
+ rd_list_cnt(configs) * 200);
+
+ /* #resources */
+ rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(configs));
+
+ RD_LIST_FOREACH(config, configs, i) {
+ const rd_kafka_ConfigEntry_t *entry;
+ int ei;
+
+ /* resource_type */
+ rd_kafka_buf_write_i8(rkbuf, config->restype);
+
+ /* resource_name */
+ rd_kafka_buf_write_str(rkbuf, config->name, -1);
+
+ /* #config */
+ rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(&config->config));
+
+ RD_LIST_FOREACH(entry, &config->config, ei) {
+ /* config_name */
+ rd_kafka_buf_write_str(rkbuf, entry->kv->name, -1);
+ /* config_value (nullable) */
+ rd_kafka_buf_write_str(rkbuf, entry->kv->value, -1);
+
+ if (ApiVersion == 1)
+ rd_kafka_buf_write_i8(rkbuf,
+ entry->a.operation);
+ else if (entry->a.operation != RD_KAFKA_ALTER_OP_SET) {
+ rd_snprintf(errstr, errstr_size,
+ "Broker version >= 2.0.0 required "
+ "for add/delete config "
+ "entries: only set supported "
+ "by this broker");
+ rd_kafka_buf_destroy(rkbuf);
+ return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE;
+ }
+ }
+ }
+
+ /* timeout */
+ op_timeout = rd_kafka_confval_get_int(&options->operation_timeout);
+ if (op_timeout > rkb->rkb_rk->rk_conf.socket_timeout_ms)
+ rd_kafka_buf_set_abs_timeout(rkbuf, op_timeout+1000, 0);
+
+ /* validate_only */
+ rd_kafka_buf_write_i8(
+ rkbuf, rd_kafka_confval_get_int(&options->validate_only));
+
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, 0);
+
+ rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+/**
+ * @brief Construct and send DescribeConfigsRequest to \p rkb
+ * with the configs (ConfigResource_t*) in \p configs, using
+ * \p options.
+ *
+ * The response (unparsed) will be enqueued on \p replyq
+ * for handling by \p resp_cb (with \p opaque passed).
+ *
+ * @returns RD_KAFKA_RESP_ERR_NO_ERROR if the request was enqueued for
+ * transmission, otherwise an error code and errstr will be
+ * updated with a human readable error string.
+ */
+rd_kafka_resp_err_t
+rd_kafka_DescribeConfigsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *configs /*(ConfigResource_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque) {
+ rd_kafka_buf_t *rkbuf;
+ int16_t ApiVersion = 0;
+ int i;
+ const rd_kafka_ConfigResource_t *config;
+ int op_timeout;
+
+ if (rd_list_cnt(configs) == 0) {
+ rd_snprintf(errstr, errstr_size,
+ "No config resources specified");
+ return RD_KAFKA_RESP_ERR__INVALID_ARG;
+ }
+
+ ApiVersion = rd_kafka_broker_ApiVersion_supported(
+ rkb, RD_KAFKAP_DescribeConfigs, 0, 1, NULL);
+ if (ApiVersion == -1) {
+ rd_snprintf(errstr, errstr_size,
+ "DescribeConfigs (KIP-133) not supported "
+ "by broker, requires broker version >= 0.11.0");
+ return RD_KAFKA_RESP_ERR__UNSUPPORTED_FEATURE;
+ }
+
+ rkbuf = rd_kafka_buf_new_request(rkb, RD_KAFKAP_DescribeConfigs, 1,
+ rd_list_cnt(configs) * 200);
+
+ /* #resources */
+ rd_kafka_buf_write_i32(rkbuf, rd_list_cnt(configs));
+
+ RD_LIST_FOREACH(config, configs, i) {
+ const rd_kafka_ConfigEntry_t *entry;
+ int ei;
+
+ /* resource_type */
+ rd_kafka_buf_write_i8(rkbuf, config->restype);
+
+ /* resource_name */
+ rd_kafka_buf_write_str(rkbuf, config->name, -1);
+
+ /* #config */
+ if (rd_list_empty(&config->config)) {
+ /* Get all configs */
+ rd_kafka_buf_write_i32(rkbuf, -1);
+ } else {
+ /* Get requested configs only */
+ rd_kafka_buf_write_i32(rkbuf,
+ rd_list_cnt(&config->config));
+ }
+
+ RD_LIST_FOREACH(entry, &config->config, ei) {
+ /* config_name */
+ rd_kafka_buf_write_str(rkbuf, entry->kv->name, -1);
+ }
+ }
+
+
+ if (ApiVersion == 1) {
+ /* include_synonyms */
+ rd_kafka_buf_write_i8(rkbuf, 1);
+ }
+
+ /* timeout */
+ op_timeout = rd_kafka_confval_get_int(&options->operation_timeout);
+ if (op_timeout > rkb->rkb_rk->rk_conf.socket_timeout_ms)
+ rd_kafka_buf_set_abs_timeout(rkbuf, op_timeout+1000, 0);
+
+ rd_kafka_buf_ApiVersion_set(rkbuf, ApiVersion, 0);
+
+ rd_kafka_broker_buf_enq_replyq(rkb, rkbuf, replyq, resp_cb, opaque);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
diff -Nru librdkafka-0.11.3/src/rdkafka_request.h librdkafka-0.11.6/src/rdkafka_request.h
--- librdkafka-0.11.3/src/rdkafka_request.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_request.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,8 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-
-#pragma once
+#ifndef _RDKAFKA_REQUEST_H_
+#define _RDKAFKA_REQUEST_H_
#include "rdkafka_cgrp.h"
#include "rdkafka_feature.h"
@@ -194,3 +194,50 @@
void *opaque, int flash_msg);
int rd_kafka_ProduceRequest (rd_kafka_broker_t *rkb, rd_kafka_toppar_t *rktp);
+
+rd_kafka_resp_err_t
+rd_kafka_CreateTopicsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *new_topics /*(NewTopic_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque);
+
+rd_kafka_resp_err_t
+rd_kafka_DeleteTopicsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *del_topics /*(DeleteTopic_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque);
+
+rd_kafka_resp_err_t
+rd_kafka_CreatePartitionsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *new_parts /*(NewPartitions_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque);
+
+rd_kafka_resp_err_t
+rd_kafka_AlterConfigsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *configs /*(ConfigResource_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque);
+
+rd_kafka_resp_err_t
+rd_kafka_DescribeConfigsRequest (rd_kafka_broker_t *rkb,
+ const rd_list_t *configs /*(ConfigResource_t*)*/,
+ rd_kafka_AdminOptions_t *options,
+ char *errstr, size_t errstr_size,
+ rd_kafka_replyq_t replyq,
+ rd_kafka_resp_cb_t *resp_cb,
+ void *opaque);
+
+#endif /* _RDKAFKA_REQUEST_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_sasl.h librdkafka-0.11.6/src/rdkafka_sasl.h
--- librdkafka-0.11.3/src/rdkafka_sasl.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_sasl.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_SASL_H_
+#define _RDKAFKA_SASL_H_
@@ -44,3 +45,5 @@
int rd_kafka_sasl_select_provider (rd_kafka_t *rk,
char *errstr, size_t errstr_size);
+
+#endif /* _RDKAFKA_SASL_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_sasl_int.h librdkafka-0.11.6/src/rdkafka_sasl_int.h
--- librdkafka-0.11.3/src/rdkafka_sasl_int.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_sasl_int.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_SASL_INT_H_
+#define _RDKAFKA_SASL_INT_H_
struct rd_kafka_sasl_provider {
const char *name;
@@ -68,3 +69,4 @@
const void *payload, int len,
char *errstr, size_t errstr_size);
+#endif /* _RDKAFKA_SASL_INT_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_sasl_plain.c librdkafka-0.11.6/src/rdkafka_sasl_plain.c
--- librdkafka-0.11.3/src/rdkafka_sasl_plain.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_sasl_plain.c 2018-10-10 06:54:38.000000000 +0000
@@ -67,28 +67,28 @@
rd_kafka_broker_t *rkb = rktrans->rktrans_rkb;
rd_kafka_t *rk = rkb->rkb_rk;
/* [authzid] UTF8NUL authcid UTF8NUL passwd */
- char buf[255+1+255+1+255+1];
+ char *buf;
int of = 0;
+ int zidlen = 0;
+ int cidlen = rk->rk_conf.sasl.username ?
+ (int)strlen(rk->rk_conf.sasl.username) : 0;
+ int pwlen = rk->rk_conf.sasl.password ?
+ (int)strlen(rk->rk_conf.sasl.password) : 0;
+
+
+ buf = rd_alloca(zidlen + 1 + cidlen + 1 + pwlen + 1);
/* authzid: none (empty) */
/* UTF8NUL */
buf[of++] = 0;
/* authcid */
- if (rk->rk_conf.sasl.username) {
- int r = (int)strlen(rk->rk_conf.sasl.username);
- r = RD_MIN(255, r);
- memcpy(&buf[of], rk->rk_conf.sasl.username, r);
- of += r;
- }
+ memcpy(&buf[of], rk->rk_conf.sasl.username, cidlen);
+ of += cidlen;
/* UTF8NUL */
buf[of++] = 0;
/* passwd */
- if (rk->rk_conf.sasl.password) {
- int r = (int)strlen(rk->rk_conf.sasl.password);
- r = RD_MIN(255, r);
- memcpy(&buf[of], rk->rk_conf.sasl.password, r);
- of += r;
- }
+ memcpy(&buf[of], rk->rk_conf.sasl.password, pwlen);
+ of += pwlen;
rd_rkb_dbg(rkb, SECURITY, "SASLPLAIN",
"Sending SASL PLAIN (builtin) authentication token");
diff -Nru librdkafka-0.11.3/src/rdkafka_sasl_scram.c librdkafka-0.11.6/src/rdkafka_sasl_scram.c
--- librdkafka-0.11.3/src/rdkafka_sasl_scram.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_sasl_scram.c 2018-10-10 06:54:38.000000000 +0000
@@ -140,66 +140,69 @@
/**
* @brief Base64 encode binary input \p in
- * @returns a newly allocated base64 string
+ * @returns a newly allocated, base64-encoded string or NULL on error.
*/
static char *rd_base64_encode (const rd_chariov_t *in) {
- BIO *buf, *b64f;
- BUF_MEM *ptr;
- char *out;
-
- b64f = BIO_new(BIO_f_base64());
- buf = BIO_new(BIO_s_mem());
- buf = BIO_push(b64f, buf);
-
- BIO_set_flags(buf, BIO_FLAGS_BASE64_NO_NL);
- BIO_set_close(buf, BIO_CLOSE);
- BIO_write(buf, in->ptr, (int)in->size);
- BIO_flush(buf);
-
- BIO_get_mem_ptr(buf, &ptr);
- out = malloc(ptr->length + 1);
- memcpy(out, ptr->data, ptr->length);
- out[ptr->length] = '\0';
+ char *ret;
+ size_t ret_len, max_len;
- BIO_free_all(buf);
+ /* OpenSSL takes an |int| argument so the input cannot exceed that. */
+ if (in->size > INT_MAX) {
+ return NULL;
+ }
+
+ /* This does not overflow given the |INT_MAX| bound, above. */
+ max_len = (((in->size + 2) / 3) * 4) + 1;
+ ret = rd_malloc(max_len);
+ if (ret == NULL) {
+ return NULL;
+ }
- return out;
+ ret_len = EVP_EncodeBlock((uint8_t*)ret, (uint8_t*)in->ptr, (int)in->size);
+ assert(ret_len < max_len);
+ ret[ret_len] = 0;
+
+ return ret;
}
+
/**
- * @brief Base64 decode input string \p in of size \p insize.
+ * @brief Base64 decode input string \p in. Ignores leading and trailing
+ * whitespace.
* @returns -1 on invalid Base64, or 0 on successes in which case a
* newly allocated binary string is set in out (and size).
*/
static int rd_base64_decode (const rd_chariov_t *in, rd_chariov_t *out) {
- size_t asize;
- BIO *b64, *bmem;
+ size_t ret_len;
- if (in->size == 0 || (in->size % 4) != 0)
+ /* OpenSSL takes an |int| argument, so |in->size| must not exceed
+ * that. */
+ if (in->size % 4 != 0 || in->size > INT_MAX) {
return -1;
+ }
- asize = (in->size * 3) / 4; /* allocation size */
- out->ptr = rd_malloc(asize+1);
-
- b64 = BIO_new(BIO_f_base64());
- BIO_set_flags(b64, BIO_FLAGS_BASE64_NO_NL);
-
- bmem = BIO_new_mem_buf(in->ptr, (int)in->size);
- bmem = BIO_push(b64, bmem);
+ ret_len = ((in->size / 4) * 3);
+ out->ptr = rd_malloc(ret_len+1);
- out->size = BIO_read(bmem, out->ptr, (int)asize+1);
- assert(out->size <= asize);
- BIO_free_all(bmem);
+ if (EVP_DecodeBlock((uint8_t*)out->ptr, (uint8_t*)in->ptr,
+ (int)in->size) == -1) {
+ free(out->ptr);
+ out->ptr = NULL;
+ return -1;
+ }
-#if ENABLE_DEVEL
- /* Verify that decode==encode */
- {
- char *encoded = rd_base64_encode(out);
- assert(strlen(encoded) == in->size);
- assert(!strncmp(encoded, in->ptr, in->size));
- rd_free(encoded);
+ /* EVP_DecodeBlock will pad the output with trailing NULs and count
+ * them in the return value. */
+ if (in->size > 1 && in->ptr[in->size-1] == '=') {
+ if (in->size > 2 && in->ptr[in->size-2] == '=') {
+ ret_len -= 2;
+ } else {
+ ret_len -= 1;
+ }
}
-#endif
+
+ out->ptr[ret_len] = 0;
+ out->size = ret_len;
return 0;
}
@@ -499,7 +502,10 @@
/* Store the Base64 encoded ServerSignature for quick comparison */
state->ServerSignatureB64 = rd_base64_encode(&ServerSignature);
-
+ if (state->ServerSignatureB64 == NULL) {
+ rd_free(client_final_msg_wo_proof.ptr);
+ return -1;
+ }
/*
* Continue with client-final-message
@@ -521,6 +527,10 @@
/* Base64 encoded ClientProof */
ClientProofB64 = rd_base64_encode(&ClientProof);
+ if (ClientProofB64 == NULL) {
+ rd_free(client_final_msg_wo_proof.ptr);
+ return -1;
+ }
/* Construct client-final-message */
out->size = client_final_msg_wo_proof.size +
@@ -602,6 +612,7 @@
"Invalid Base64 Salt in server-first-message");
rd_free(server_nonce);
rd_free(salt_b64.ptr);
+ return -1;
}
rd_free(salt_b64.ptr);
diff -Nru librdkafka-0.11.3/src/rdkafka_subscription.c librdkafka-0.11.6/src/rdkafka_subscription.c
--- librdkafka-0.11.3/src/rdkafka_subscription.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_subscription.c 2018-10-10 06:54:38.000000000 +0000
@@ -34,7 +34,6 @@
*/
#include "rdkafka_int.h"
-#include "rdkafka_subscription.h"
rd_kafka_resp_err_t rd_kafka_unsubscribe (rd_kafka_t *rk) {
diff -Nru librdkafka-0.11.3/src/rdkafka_subscription.h librdkafka-0.11.6/src/rdkafka_subscription.h
--- librdkafka-0.11.3/src/rdkafka_subscription.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_subscription.h 1970-01-01 00:00:00.000000000 +0000
@@ -1,31 +0,0 @@
-/*
- * librdkafka - Apache Kafka C library
- *
- * Copyright (c) 2012-2015, Magnus Edenhill
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#pragma once
-
-
diff -Nru librdkafka-0.11.3/src/rdkafka_timer.c librdkafka-0.11.6/src/rdkafka_timer.c
--- librdkafka-0.11.3/src/rdkafka_timer.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_timer.c 2018-10-10 06:54:38.000000000 +0000
@@ -87,10 +87,12 @@
}
/**
- * Stop a timer that may be started.
- * If called from inside a timer callback 'lock' must be 0, else 1.
+ * @brief Stop a timer that may be started.
+ * If called from inside a timer callback 'lock' must be 0, else 1.
+ *
+ * @returns 1 if the timer was started (before being stopped), else 0.
*/
-void rd_kafka_timer_stop (rd_kafka_timers_t *rkts, rd_kafka_timer_t *rtmr,
+int rd_kafka_timer_stop (rd_kafka_timers_t *rkts, rd_kafka_timer_t *rtmr,
int lock) {
if (lock)
rd_kafka_timers_lock(rkts);
@@ -98,7 +100,7 @@
if (!rd_kafka_timer_started(rtmr)) {
if (lock)
rd_kafka_timers_unlock(rkts);
- return;
+ return 0;
}
if (rd_kafka_timer_scheduled(rtmr))
@@ -108,6 +110,8 @@
if (lock)
rd_kafka_timers_unlock(rkts);
+
+ return 1;
}
@@ -118,10 +122,12 @@
*
* Use rd_kafka_timer_stop() to stop a timer.
*/
-void rd_kafka_timer_start (rd_kafka_timers_t *rkts,
- rd_kafka_timer_t *rtmr, rd_ts_t interval,
- void (*callback) (rd_kafka_timers_t *rkts, void *arg),
- void *arg) {
+void rd_kafka_timer_start0 (rd_kafka_timers_t *rkts,
+ rd_kafka_timer_t *rtmr, rd_ts_t interval,
+ rd_bool_t oneshot,
+ void (*callback) (rd_kafka_timers_t *rkts,
+ void *arg),
+ void *arg) {
rd_kafka_timers_lock(rkts);
rd_kafka_timer_stop(rkts, rtmr, 0/*!lock*/);
@@ -129,13 +135,13 @@
rtmr->rtmr_interval = interval;
rtmr->rtmr_callback = callback;
rtmr->rtmr_arg = arg;
+ rtmr->rtmr_oneshot = oneshot;
rd_kafka_timer_schedule(rkts, rtmr, 0);
rd_kafka_timers_unlock(rkts);
}
-
/**
* Delay the next timer invocation by 'backoff_us'
*/
@@ -223,7 +229,7 @@
rd_kafka_timers_lock(rkts);
- while (!rd_atomic32_get(&rkts->rkts_rk->rk_terminate) && now <= end) {
+ while (!rd_kafka_terminating(rkts->rkts_rk) && now <= end) {
int64_t sleeptime;
rd_kafka_timer_t *rtmr;
@@ -246,11 +252,18 @@
rtmr->rtmr_next <= now) {
rd_kafka_timer_unschedule(rkts, rtmr);
+
+ /* If timer must only be fired once,
+ * disable it now prior to callback. */
+ if (rtmr->rtmr_oneshot)
+ rtmr->rtmr_interval = 0;
+
rd_kafka_timers_unlock(rkts);
rtmr->rtmr_callback(rkts, rtmr->rtmr_arg);
rd_kafka_timers_lock(rkts);
+
/* Restart timer, unless it has been stopped, or
* already reschedueld (start()ed) from callback. */
if (rd_kafka_timer_started(rtmr) &&
diff -Nru librdkafka-0.11.3/src/rdkafka_timer.h librdkafka-0.11.6/src/rdkafka_timer.h
--- librdkafka-0.11.3/src/rdkafka_timer.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_timer.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_TIMER_H_
+#define _RDKAFKA_TIMER_H_
#include "rd.h"
@@ -49,6 +50,7 @@
rd_ts_t rtmr_next;
rd_ts_t rtmr_interval; /* interval in microseconds */
+ rd_bool_t rtmr_oneshot; /**< Only fire once. */
void (*rtmr_callback) (rd_kafka_timers_t *rkts, void *arg);
void *rtmr_arg;
@@ -56,13 +58,18 @@
-void rd_kafka_timer_stop (rd_kafka_timers_t *rkts,
- rd_kafka_timer_t *rtmr, int lock);
-void rd_kafka_timer_start (rd_kafka_timers_t *rkts,
- rd_kafka_timer_t *rtmr, rd_ts_t interval,
- void (*callback) (rd_kafka_timers_t *rkts,
- void *arg),
- void *arg);
+int rd_kafka_timer_stop (rd_kafka_timers_t *rkts,
+ rd_kafka_timer_t *rtmr, int lock);
+void rd_kafka_timer_start0 (rd_kafka_timers_t *rkts,
+ rd_kafka_timer_t *rtmr, rd_ts_t interval,
+ rd_bool_t oneshot,
+ void (*callback) (rd_kafka_timers_t *rkts,
+ void *arg),
+ void *arg);
+#define rd_kafka_timer_start(rkts,rtmr,interval,callback,arg) \
+ rd_kafka_timer_start0(rkts,rtmr,interval,rd_false,callback,arg)
+#define rd_kafka_timer_start_oneshot(rkts,rtmr,interval,callback,arg) \
+ rd_kafka_timer_start0(rkts,rtmr,interval,rd_true,callback,arg)
void rd_kafka_timer_backoff (rd_kafka_timers_t *rkts,
rd_kafka_timer_t *rtmr, int backoff_us);
@@ -75,3 +82,5 @@
void rd_kafka_timers_run (rd_kafka_timers_t *rkts, int timeout_us);
void rd_kafka_timers_destroy (rd_kafka_timers_t *rkts);
void rd_kafka_timers_init (rd_kafka_timers_t *rkte, rd_kafka_t *rk);
+
+#endif /* _RDKAFKA_TIMER_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_topic.c librdkafka-0.11.6/src/rdkafka_topic.c
--- librdkafka-0.11.3/src/rdkafka_topic.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_topic.c 2018-10-10 06:54:38.000000000 +0000
@@ -117,6 +117,9 @@
rd_kafka_assert(rkt->rkt_rk, rd_list_empty(&rkt->rkt_desp));
rd_list_destroy(&rkt->rkt_desp);
+ rd_avg_destroy(&rkt->rkt_avg_batchsize);
+ rd_avg_destroy(&rkt->rkt_avg_batchcnt);
+
if (rkt->rkt_topic)
rd_kafkap_str_destroy(rkt->rkt_topic);
@@ -260,13 +263,90 @@
* just the placeholder. The internal members
* were copied on the line above. */
- /* Default partitioner: consistent_random */
- if (!rkt->rkt_conf.partitioner)
- rkt->rkt_conf.partitioner = rd_kafka_msg_partitioner_consistent_random;
+ /* Partitioner */
+ if (!rkt->rkt_conf.partitioner) {
+ const struct {
+ const char *str;
+ void *part;
+ } part_map[] = {
+ { "random",
+ (void *)rd_kafka_msg_partitioner_random },
+ { "consistent",
+ (void *)rd_kafka_msg_partitioner_consistent },
+ { "consistent_random",
+ (void *)rd_kafka_msg_partitioner_consistent_random },
+ { "murmur2",
+ (void *)rd_kafka_msg_partitioner_murmur2 },
+ { "murmur2_random",
+ (void *)rd_kafka_msg_partitioner_murmur2_random },
+ { NULL }
+ };
+ int i;
+
+ /* Use "partitioner" configuration property string, if set */
+ for (i = 0 ; rkt->rkt_conf.partitioner_str && part_map[i].str ;
+ i++) {
+ if (!strcmp(rkt->rkt_conf.partitioner_str,
+ part_map[i].str)) {
+ rkt->rkt_conf.partitioner = part_map[i].part;
+ break;
+ }
+ }
+
+ /* Default partitioner: consistent_random */
+ if (!rkt->rkt_conf.partitioner) {
+ /* Make sure part_map matched something, otherwise
+ * there is a discreprency between this code
+ * and the validator in rdkafka_conf.c */
+ assert(!rkt->rkt_conf.partitioner_str);
+
+ rkt->rkt_conf.partitioner =
+ rd_kafka_msg_partitioner_consistent_random;
+ }
+ }
+
+ if (rkt->rkt_conf.queuing_strategy == RD_KAFKA_QUEUE_FIFO)
+ rkt->rkt_conf.msg_order_cmp = rd_kafka_msg_cmp_msgseq;
+ else
+ rkt->rkt_conf.msg_order_cmp = rd_kafka_msg_cmp_msgseq_lifo;
if (rkt->rkt_conf.compression_codec == RD_KAFKA_COMPRESSION_INHERIT)
rkt->rkt_conf.compression_codec = rk->rk_conf.compression_codec;
+ /* Translate compression level to library-specific level and check
+ * upper bound */
+ switch (rkt->rkt_conf.compression_codec) {
+#if WITH_ZLIB
+ case RD_KAFKA_COMPRESSION_GZIP:
+ if (rkt->rkt_conf.compression_level == RD_KAFKA_COMPLEVEL_DEFAULT)
+ rkt->rkt_conf.compression_level = Z_DEFAULT_COMPRESSION;
+ else if (rkt->rkt_conf.compression_level > RD_KAFKA_COMPLEVEL_GZIP_MAX)
+ rkt->rkt_conf.compression_level =
+ RD_KAFKA_COMPLEVEL_GZIP_MAX;
+ break;
+#endif
+ case RD_KAFKA_COMPRESSION_LZ4:
+ if (rkt->rkt_conf.compression_level == RD_KAFKA_COMPLEVEL_DEFAULT)
+ /* LZ4 has no notion of system-wide default compression
+ * level, use zero in this case */
+ rkt->rkt_conf.compression_level = 0;
+ else if (rkt->rkt_conf.compression_level > RD_KAFKA_COMPLEVEL_LZ4_MAX)
+ rkt->rkt_conf.compression_level =
+ RD_KAFKA_COMPLEVEL_LZ4_MAX;
+ break;
+ case RD_KAFKA_COMPRESSION_SNAPPY:
+ default:
+ /* Compression level has no effect in this case */
+ rkt->rkt_conf.compression_level = RD_KAFKA_COMPLEVEL_DEFAULT;
+ }
+
+ rd_avg_init(&rkt->rkt_avg_batchsize, RD_AVG_GAUGE, 0,
+ rk->rk_conf.max_msg_size, 2,
+ rk->rk_conf.stats_interval_ms ? 1 : 0);
+ rd_avg_init(&rkt->rkt_avg_batchcnt, RD_AVG_GAUGE, 0,
+ rk->rk_conf.batch_num_messages, 2,
+ rk->rk_conf.stats_interval_ms ? 1 : 0);
+
rd_kafka_dbg(rk, TOPIC, "TOPIC", "New local topic: %.*s",
RD_KAFKAP_STR_PR(rkt->rkt_topic));
@@ -459,10 +539,8 @@
int32_t partition_cnt) {
rd_kafka_t *rk = rkt->rkt_rk;
shptr_rd_kafka_toppar_t **rktps;
- shptr_rd_kafka_toppar_t *rktp_ua;
shptr_rd_kafka_toppar_t *s_rktp;
rd_kafka_toppar_t *rktp;
- rd_kafka_msgq_t tmpq = RD_KAFKA_MSGQ_INITIALIZER(tmpq);
int32_t i;
if (likely(rkt->rkt_partition_cnt == partition_cnt))
@@ -499,14 +577,24 @@
rktp = s_rktp ? rd_kafka_toppar_s2i(s_rktp) : NULL;
if (rktp) {
rd_kafka_toppar_lock(rktp);
- rktp->rktp_flags &= ~RD_KAFKA_TOPPAR_F_UNKNOWN;
+ rktp->rktp_flags &=
+ ~(RD_KAFKA_TOPPAR_F_UNKNOWN |
+ RD_KAFKA_TOPPAR_F_REMOVE);
/* Remove from desp list since the
* partition is now known. */
rd_kafka_toppar_desired_unlink(rktp);
rd_kafka_toppar_unlock(rktp);
- } else
+ } else {
s_rktp = rd_kafka_toppar_new(rkt, i);
+ rktp = rd_kafka_toppar_s2i(s_rktp);
+
+ rd_kafka_toppar_lock(rktp);
+ rktp->rktp_flags &=
+ ~(RD_KAFKA_TOPPAR_F_UNKNOWN |
+ RD_KAFKA_TOPPAR_F_REMOVE);
+ rd_kafka_toppar_unlock(rktp);
+ }
rktps[i] = s_rktp;
} else {
/* Existing partition, grab our own reference. */
@@ -517,8 +605,6 @@
}
}
- rktp_ua = rd_kafka_toppar_get(rkt, RD_KAFKA_PARTITION_UA, 0);
-
/* Propagate notexist errors for desired partitions */
RD_LIST_FOREACH(s_rktp, &rkt->rkt_desp, i) {
rd_kafka_dbg(rkt->rkt_rk, TOPIC, "DESIRED",
@@ -527,7 +613,10 @@
rkt->rkt_topic->str,
rd_kafka_toppar_s2i(s_rktp)->rktp_partition);
rd_kafka_toppar_enq_error(rd_kafka_toppar_s2i(s_rktp),
- RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION);
+ RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION,
+ "desired partition does not exist "
+ "in cluster");
+
}
/* Remove excessive partitions */
@@ -541,6 +630,8 @@
rd_kafka_toppar_lock(rktp);
+ rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_UNKNOWN;
+
if (rktp->rktp_flags & RD_KAFKA_TOPPAR_F_DESIRED) {
rd_kafka_dbg(rkt->rkt_rk, TOPIC, "DESIRED",
"Topic %s [%"PRId32"] is desired "
@@ -550,22 +641,18 @@
/* If this is a desired partition move it back on to
* the desired list since partition is no longer known*/
- rd_kafka_assert(rkt->rkt_rk,
- !(rktp->rktp_flags &
- RD_KAFKA_TOPPAR_F_UNKNOWN));
- rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_UNKNOWN;
rd_kafka_toppar_desired_link(rktp);
if (!rd_kafka_terminating(rkt->rkt_rk))
rd_kafka_toppar_enq_error(
rktp,
- RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION);
+ RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION,
+ "desired partition no longer exists");
rd_kafka_toppar_broker_delegate(rktp, NULL, 0);
} else {
/* Tell handling broker to let go of the toppar */
- rktp->rktp_flags |= RD_KAFKA_TOPPAR_F_REMOVE;
rd_kafka_toppar_broker_leave_for_remove(rktp);
}
@@ -574,42 +661,6 @@
rd_kafka_toppar_destroy(s_rktp);
}
- if (likely(rktp_ua != NULL)) {
- /* Move messages from removed partitions to UA for
- * further processing. */
- rktp = rd_kafka_toppar_s2i(rktp_ua);
-
- // FIXME: tmpq not used
- if (rd_kafka_msgq_len(&tmpq) > 0) {
- rd_kafka_dbg(rkt->rkt_rk, TOPIC, "TOPPARMOVE",
- "Moving %d messages (%zd bytes) from "
- "%d removed partitions to UA partition",
- rd_kafka_msgq_len(&tmpq),
- rd_kafka_msgq_size(&tmpq),
- i - partition_cnt);
-
-
- rd_kafka_toppar_lock(rktp);
- rd_kafka_msgq_concat(&rktp->rktp_msgq, &tmpq);
- rd_kafka_toppar_unlock(rktp);
- }
-
- rd_kafka_toppar_destroy(rktp_ua); /* .._get() above */
- } else {
- /* No UA, fail messages from removed partitions. */
- if (rd_kafka_msgq_len(&tmpq) > 0) {
- rd_kafka_dbg(rkt->rkt_rk, TOPIC, "TOPPARMOVE",
- "Failing %d messages (%zd bytes) from "
- "%d removed partitions",
- rd_kafka_msgq_len(&tmpq),
- rd_kafka_msgq_size(&tmpq),
- i - partition_cnt);
-
- rd_kafka_dr_msgq(rkt, &tmpq,
- RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION);
- }
- }
-
if (rkt->rkt_p)
rd_free(rkt->rkt_p);
@@ -641,7 +692,8 @@
/* Notify consumers that the topic doesn't exist. */
RD_LIST_FOREACH(s_rktp, &rkt->rkt_desp, i)
- rd_kafka_toppar_enq_error(rd_kafka_toppar_s2i(s_rktp), err);
+ rd_kafka_toppar_enq_error(rd_kafka_toppar_s2i(s_rktp), err,
+ "topic does not exist");
}
@@ -673,16 +725,17 @@
rktp_ua = rd_kafka_toppar_s2i(s_rktp_ua);
/* Assign all unassigned messages to new topics. */
- rd_kafka_dbg(rk, TOPIC, "PARTCNT",
- "Partitioning %i unassigned messages in topic %.*s to "
- "%"PRId32" partitions",
- rd_atomic32_get(&rktp_ua->rktp_msgq.rkmq_msg_cnt),
- RD_KAFKAP_STR_PR(rkt->rkt_topic),
- rkt->rkt_partition_cnt);
+ rd_kafka_toppar_lock(rktp_ua);
+
+ rd_kafka_dbg(rk, TOPIC, "PARTCNT",
+ "Partitioning %i unassigned messages in topic %.*s to "
+ "%"PRId32" partitions",
+ rktp_ua->rktp_msgq.rkmq_msg_cnt,
+ RD_KAFKAP_STR_PR(rkt->rkt_topic),
+ rkt->rkt_partition_cnt);
- rd_kafka_toppar_lock(rktp_ua);
rd_kafka_msgq_move(&uas, &rktp_ua->rktp_msgq);
- cnt = rd_atomic32_get(&uas.rkmq_msg_cnt);
+ cnt = uas.rkmq_msg_cnt;
rd_kafka_toppar_unlock(rktp_ua);
TAILQ_FOREACH_SAFE(rkm, &uas.rkmq_msgs, rkm_link, tmp) {
@@ -700,18 +753,16 @@
}
}
- rd_kafka_dbg(rk, TOPIC, "UAS",
- "%i/%i messages were partitioned in topic %s",
- cnt - rd_atomic32_get(&failed.rkmq_msg_cnt),
- cnt, rkt->rkt_topic->str);
-
- if (rd_atomic32_get(&failed.rkmq_msg_cnt) > 0) {
- /* Fail the messages */
- rd_kafka_dbg(rk, TOPIC, "UAS",
- "%"PRId32"/%i messages failed partitioning "
- "in topic %s",
- rd_atomic32_get(&uas.rkmq_msg_cnt), cnt,
- rkt->rkt_topic->str);
+ rd_kafka_dbg(rk, TOPIC, "UAS",
+ "%i/%i messages were partitioned in topic %s",
+ cnt - failed.rkmq_msg_cnt, cnt, rkt->rkt_topic->str);
+
+ if (failed.rkmq_msg_cnt > 0) {
+ /* Fail the messages */
+ rd_kafka_dbg(rk, TOPIC, "UAS",
+ "%"PRId32"/%i messages failed partitioning "
+ "in topic %s",
+ failed.rkmq_msg_cnt, cnt, rkt->rkt_topic->str);
rd_kafka_dr_msgq(rkt, &failed,
rkt->rkt_state == RD_KAFKA_TOPIC_S_NOTEXISTS ?
err :
@@ -729,7 +780,7 @@
void rd_kafka_topic_metadata_none (rd_kafka_itopic_t *rkt) {
rd_kafka_topic_wrlock(rkt);
- if (unlikely(rd_atomic32_get(&rkt->rkt_rk->rk_terminate))) {
+ if (unlikely(rd_kafka_terminating(rkt->rkt_rk))) {
/* Dont update metadata while terminating, do this
* after acquiring lock for proper synchronisation */
rd_kafka_topic_wrunlock(rkt);
@@ -811,9 +862,9 @@
old_state = rkt->rkt_state;
rkt->rkt_ts_metadata = ts_age;
- /* Set topic state */
+ /* Set topic state.
+ * UNKNOWN_TOPIC_OR_PART may indicate that auto.create.topics failed */
if (mdt->err == RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART ||
- mdt->err == RD_KAFKA_RESP_ERR_UNKNOWN/*auto.create.topics fails*/||
mdt->err == RD_KAFKA_RESP_ERR_TOPIC_EXCEPTION/*invalid topic*/)
rd_kafka_topic_set_state(rkt, RD_KAFKA_TOPIC_S_NOTEXISTS);
else if (mdt->partition_cnt > 0)
@@ -821,10 +872,21 @@
/* Update number of partitions, but not if there are
* (possibly intermittent) errors (e.g., "Leader not available"). */
- if (mdt->err == RD_KAFKA_RESP_ERR_NO_ERROR)
+ if (mdt->err == RD_KAFKA_RESP_ERR_NO_ERROR) {
upd += rd_kafka_topic_partition_cnt_update(rkt,
mdt->partition_cnt);
+ /* If the metadata times out for a topic (because all brokers
+ * are down) the state will transition to S_UNKNOWN.
+ * When updated metadata is eventually received there might
+ * not be any change to partition count or leader,
+ * but there may still be messages in the UA partition that
+ * needs to be assigned, so trigger an update for this case too.
+ * Issue #1985. */
+ if (old_state == RD_KAFKA_TOPIC_S_UNKNOWN)
+ upd++;
+ }
+
/* Update leader for each partition */
for (j = 0 ; j < mdt->partition_cnt ; j++) {
int r;
@@ -1033,10 +1095,12 @@
/**
- * Scan all topics and partitions for:
+ * @brief Scan all topics and partitions for:
* - timed out messages.
* - topics that needs to be created on the broker.
* - topics who's metadata is too old.
+ *
+ * @locality rdkafka main thread
*/
int rd_kafka_topic_scan_all (rd_kafka_t *rk, rd_ts_t now) {
rd_kafka_itopic_t *rkt;
@@ -1121,11 +1185,6 @@
query_this = 1;
}
- /* Scan toppar's message queues for timeouts */
- if (rd_kafka_msgq_age_scan(&rktp->rktp_xmit_msgq,
- &timedout, now) > 0)
- did_tmout = 1;
-
if (rd_kafka_msgq_age_scan(&rktp->rktp_msgq,
&timedout, now) > 0)
did_tmout = 1;
@@ -1138,7 +1197,7 @@
rd_kafka_topic_rdunlock(rkt);
- if ((cnt = rd_atomic32_get(&timedout.rkmq_msg_cnt)) > 0) {
+ if ((cnt = timedout.rkmq_msg_cnt) > 0) {
totcnt += cnt;
rd_kafka_dbg(rk, MSG, "TIMEOUT",
"%s: %"PRId32" message(s) "
diff -Nru librdkafka-0.11.3/src/rdkafka_topic.h librdkafka-0.11.6/src/rdkafka_topic.h
--- librdkafka-0.11.3/src/rdkafka_topic.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_topic.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_TOPIC_H_
+#define _RDKAFKA_TOPIC_H_
#include "rdlist.h"
@@ -73,6 +74,9 @@
rd_kafka_t *rkt_rk;
+ rd_avg_t rkt_avg_batchsize; /**< Average batch size */
+ rd_avg_t rkt_avg_batchcnt; /**< Average batch message count */
+
shptr_rd_kafka_itopic_t *rkt_shptr_app; /* Application's topic_new() */
rd_kafka_topic_conf_t rkt_conf;
@@ -183,3 +187,5 @@
rd_kafka_metadata_fast_leader_query(rk)
void rd_kafka_local_topics_to_list (rd_kafka_t *rk, rd_list_t *topics);
+
+#endif /* _RDKAFKA_TOPIC_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_transport.c librdkafka-0.11.6/src/rdkafka_transport.c
--- librdkafka-0.11.3/src/rdkafka_transport.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_transport.c 2018-10-10 06:54:38.000000000 +0000
@@ -38,6 +38,7 @@
#include "rdkafka_transport.h"
#include "rdkafka_transport_int.h"
#include "rdkafka_broker.h"
+#include "rdkafka_interceptor.h"
#include
@@ -54,6 +55,7 @@
#define socket_errno WSAGetLastError()
#else
#include
+#include
#define socket_errno errno
#define SOCKET_ERROR -1
#endif
@@ -140,7 +142,7 @@
rd_slice_get_iov(slice, msg.msg_iov, &iovlen, IOV_MAX,
/* FIXME: Measure the effects of this */
rktrans->rktrans_sndbuf_size);
- msg.msg_iovlen = (typeof(msg.msg_iovlen))iovlen;
+ msg.msg_iovlen = (int)iovlen;
#ifdef sun
/* See recvmsg() comment. Setting it here to be safe. */
@@ -258,7 +260,7 @@
rd_buf_get_write_iov(rbuf, msg.msg_iov, &iovlen, IOV_MAX,
/* FIXME: Measure the effects of this */
rktrans->rktrans_rcvbuf_size);
- msg.msg_iovlen = (typeof(msg.msg_iovlen))iovlen;
+ msg.msg_iovlen = (int)iovlen;
#ifdef sun
/* SunOS doesn't seem to set errno when recvmsg() fails
@@ -273,10 +275,13 @@
/* Receive 0 after POLLIN event means
* connection closed. */
rd_snprintf(errstr, errstr_size, "Disconnected");
+ errno = ECONNRESET;
return -1;
} else if (r == -1) {
+ int errno_save = errno;
rd_snprintf(errstr, errstr_size, "%s",
rd_strerror(errno));
+ errno = errno_save;
return -1;
}
}
@@ -310,31 +315,33 @@
len,
0);
-#ifdef _MSC_VER
if (unlikely(r == SOCKET_ERROR)) {
+#ifdef _MSC_VER
if (WSAGetLastError() == WSAEWOULDBLOCK)
return sum;
rd_snprintf(errstr, errstr_size, "%s",
socket_strerror(WSAGetLastError()));
- return -1;
- }
#else
- if (unlikely(r <= 0)) {
- if (r == -1 && socket_errno == EAGAIN)
- return 0;
- else if (r == 0) {
- /* Receive 0 after POLLIN event means
- * connection closed. */
- rd_snprintf(errstr, errstr_size,
- "Disconnected");
- return -1;
- } else if (r == -1) {
+ if (socket_errno == EAGAIN)
+ return sum;
+ else {
+ int errno_save = errno;
rd_snprintf(errstr, errstr_size, "%s",
rd_strerror(errno));
+ errno = errno_save;
return -1;
}
- }
#endif
+ } else if (unlikely(r == 0)) {
+ /* Receive 0 after POLLIN event means
+ * connection closed. */
+ rd_snprintf(errstr, errstr_size,
+ "Disconnected");
+#ifndef _MSC_VER
+ errno = ECONNRESET;
+#endif
+ return -1;
+ }
/* Update buffer write position */
rd_buf_write(rbuf, NULL, (size_t)r);
@@ -387,6 +394,21 @@
/**
+ * @brief Clear OpenSSL error queue to get a proper error reporting in case
+ * the next SSL_*() operation fails.
+ */
+static RD_INLINE void
+rd_kafka_transport_ssl_clear_error (rd_kafka_transport_t *rktrans) {
+ ERR_clear_error();
+#ifdef _MSC_VER
+ WSASetLastError(0);
+#else
+ rd_set_errno(0);
+#endif
+}
+
+
+/**
* Serves the entire OpenSSL error queue and logs each error.
* The last error is not logged but returned in 'errstr'.
*
@@ -425,15 +447,15 @@
}
-static void rd_kafka_transport_ssl_lock_cb (int mode, int i,
- const char *file, int line) {
+static RD_UNUSED void
+rd_kafka_transport_ssl_lock_cb (int mode, int i, const char *file, int line) {
if (mode & CRYPTO_LOCK)
mtx_lock(&rd_kafka_ssl_locks[i]);
else
mtx_unlock(&rd_kafka_ssl_locks[i]);
}
-static unsigned long rd_kafka_transport_ssl_threadid_cb (void) {
+static RD_UNUSED unsigned long rd_kafka_transport_ssl_threadid_cb (void) {
#ifdef _MSC_VER
/* Windows makes a distinction between thread handle
* and thread id, which means we can't use the
@@ -495,7 +517,7 @@
rd_kafka_transport_ssl_io_update (rd_kafka_transport_t *rktrans, int ret,
char *errstr, size_t errstr_size) {
int serr = SSL_get_error(rktrans->rktrans_ssl, ret);
- int serr2;
+ int serr2;
switch (serr)
{
@@ -508,17 +530,18 @@
rd_kafka_transport_poll_set(rktrans, POLLOUT);
break;
- case SSL_ERROR_SYSCALL:
- if (!(serr2 = SSL_get_error(rktrans->rktrans_ssl, ret))) {
- if (ret == 0)
- errno = ECONNRESET;
- rd_snprintf(errstr, errstr_size,
- "SSL syscall error: %s", rd_strerror(errno));
- } else
- rd_snprintf(errstr, errstr_size,
- "SSL syscall error number: %d: %s", serr2,
- rd_strerror(errno));
- return -1;
+ case SSL_ERROR_SYSCALL:
+ serr2 = ERR_peek_error();
+ if (!serr2 && !socket_errno)
+ rd_snprintf(errstr, errstr_size, "Disconnected");
+ else if (serr2)
+ rd_kafka_ssl_error(NULL, rktrans->rktrans_rkb,
+ errstr, errstr_size);
+ else
+ rd_snprintf(errstr, errstr_size,
+ "SSL transport error: %s",
+ rd_strerror(socket_errno));
+ return -1;
case SSL_ERROR_ZERO_RETURN:
rd_snprintf(errstr, errstr_size, "Disconnected");
@@ -541,6 +564,8 @@
const void *p;
size_t rlen;
+ rd_kafka_transport_ssl_clear_error(rktrans);
+
while ((rlen = rd_slice_peeker(slice, &p))) {
int r;
@@ -578,6 +603,8 @@
while ((len = rd_buf_get_writable(rbuf, &p))) {
int r;
+ rd_kafka_transport_ssl_clear_error(rktrans);
+
r = SSL_read(rktrans->rktrans_ssl, p, (int)len);
if (unlikely(r <= 0)) {
@@ -667,6 +694,8 @@
goto fail;
#endif
+ rd_kafka_transport_ssl_clear_error(rktrans);
+
r = SSL_connect(rktrans->rktrans_ssl);
if (r == 1) {
/* Connected, highly unlikely since this is a
@@ -694,6 +723,8 @@
char errstr[512];
if (events & POLLOUT) {
+ rd_kafka_transport_ssl_clear_error(rktrans);
+
r = SSL_write(rktrans->rktrans_ssl, NULL, 0);
if (rd_kafka_transport_ssl_io_update(rktrans, r,
errstr,
@@ -800,6 +831,17 @@
int r;
SSL_CTX *ctx;
+#if OPENSSL_VERSION_NUMBER >= 0x10100000
+ rd_kafka_dbg(rk, SECURITY, "OPENSSL", "Using OpenSSL version %s "
+ "(0x%lx, librdkafka built with 0x%lx)",
+ OpenSSL_version(OPENSSL_VERSION),
+ OpenSSL_version_num(),
+ OPENSSL_VERSION_NUMBER);
+#else
+ rd_kafka_dbg(rk, SECURITY, "OPENSSL", "librdkafka built with OpenSSL "
+ "version 0x%lx", OPENSSL_VERSION_NUMBER);
+#endif
+
if (errstr_size > 0)
errstr[0] = '\0';
@@ -835,6 +877,33 @@
}
}
+#if OPENSSL_VERSION_NUMBER >= 0x1000200fL && !defined(LIBRESSL_VERSION_NUMBER)
+ /* Curves */
+ if (rk->rk_conf.ssl.curves_list) {
+ rd_kafka_dbg(rk, SECURITY, "SSL",
+ "Setting curves list: %s",
+ rk->rk_conf.ssl.curves_list);
+ if (!SSL_CTX_set1_curves_list(ctx,
+ rk->rk_conf.ssl.curves_list)) {
+ rd_snprintf(errstr, errstr_size,
+ "ssl.curves.list failed: ");
+ goto fail;
+ }
+ }
+
+ /* Certificate signature algorithms */
+ if (rk->rk_conf.ssl.sigalgs_list) {
+ rd_kafka_dbg(rk, SECURITY, "SSL",
+ "Setting signature algorithms list: %s",
+ rk->rk_conf.ssl.sigalgs_list);
+ if (!SSL_CTX_set1_sigalgs_list(ctx,
+ rk->rk_conf.ssl.sigalgs_list)) {
+ rd_snprintf(errstr, errstr_size,
+ "ssl.sigalgs.list failed: ");
+ goto fail;
+ }
+ }
+#endif
if (rk->rk_conf.ssl.ca_location) {
/* CA certificate location, either file or directory. */
@@ -920,6 +989,69 @@
}
}
+ if (rk->rk_conf.ssl.keystore_location) {
+ FILE *fp;
+ EVP_PKEY *pkey;
+ X509 *cert;
+ STACK_OF(X509) *ca = NULL;
+ PKCS12 *p12;
+
+ rd_kafka_dbg(rk, SECURITY, "SSL",
+ "Loading client's keystore file from %s",
+ rk->rk_conf.ssl.keystore_location);
+
+ if (!(fp = fopen(rk->rk_conf.ssl.keystore_location, "rb"))) {
+ rd_snprintf(errstr, errstr_size,
+ "Failed to open ssl.keystore.location: %s: %s",
+ rk->rk_conf.ssl.keystore_location,
+ rd_strerror(errno));
+ goto fail;
+ }
+
+ p12 = d2i_PKCS12_fp(fp, NULL);
+ fclose(fp);
+ if (!p12) {
+ rd_snprintf(errstr, errstr_size,
+ "Error reading PKCS#12 file: ");
+ goto fail;
+ }
+
+ pkey = EVP_PKEY_new();
+ cert = X509_new();
+ if (!PKCS12_parse(p12, rk->rk_conf.ssl.keystore_password, &pkey, &cert, &ca)) {
+ EVP_PKEY_free(pkey);
+ X509_free(cert);
+ PKCS12_free(p12);
+ if (ca != NULL)
+ sk_X509_pop_free(ca, X509_free);
+ rd_snprintf(errstr, errstr_size,
+ "Failed to parse PKCS#12 file: %s: ",
+ rk->rk_conf.ssl.keystore_location);
+ goto fail;
+ }
+
+ if (ca != NULL)
+ sk_X509_pop_free(ca, X509_free);
+
+ PKCS12_free(p12);
+
+ r = SSL_CTX_use_certificate(ctx, cert);
+ X509_free(cert);
+ if (r != 1) {
+ EVP_PKEY_free(pkey);
+ rd_snprintf(errstr, errstr_size,
+ "Failed to use ssl.keystore.location certificate: ");
+ goto fail;
+ }
+
+ r = SSL_CTX_use_PrivateKey(ctx, pkey);
+ EVP_PKEY_free(pkey);
+ if (r != 1) {
+ rd_snprintf(errstr, errstr_size,
+ "Failed to use ssl.keystore.location private key: ");
+ goto fail;
+ }
+ }
SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
@@ -969,6 +1101,26 @@
+/**
+ * @brief Notify transport layer of full request sent.
+ */
+void rd_kafka_transport_request_sent (rd_kafka_broker_t *rkb,
+ rd_kafka_buf_t *rkbuf) {
+ rd_kafka_transport_t *rktrans = rkb->rkb_transport;
+
+ /* Call on_request_sent interceptors */
+ rd_kafka_interceptors_on_request_sent(
+ rkb->rkb_rk,
+ rktrans->rktrans_s,
+ rkb->rkb_name, rkb->rkb_nodeid,
+ rkbuf->rkbuf_reqhdr.ApiKey,
+ rkbuf->rkbuf_reqhdr.ApiVersion,
+ rkbuf->rkbuf_corrid,
+ rd_slice_size(&rkbuf->rkbuf_reader));
+}
+
+
+
/**
* Length framed receive handling.
@@ -1137,15 +1289,15 @@
#ifdef TCP_NODELAY
- if (rkb->rkb_rk->rk_conf.socket_nagle_disable) {
- int one = 1;
- if (setsockopt(rktrans->rktrans_s, IPPROTO_TCP, TCP_NODELAY,
- (void *)&one, sizeof(one)) == SOCKET_ERROR)
- rd_rkb_log(rkb, LOG_WARNING, "NAGLE",
- "Failed to disable Nagle (TCP_NODELAY) "
- "on socket %d: %s",
- socket_strerror(socket_errno));
- }
+ if (rkb->rkb_rk->rk_conf.socket_nagle_disable) {
+ int one = 1;
+ if (setsockopt(rktrans->rktrans_s, IPPROTO_TCP, TCP_NODELAY,
+ (void *)&one, sizeof(one)) == SOCKET_ERROR)
+ rd_rkb_log(rkb, LOG_WARNING, "NAGLE",
+ "Failed to disable Nagle (TCP_NODELAY) "
+ "on socket: %s",
+ socket_strerror(socket_errno));
+ }
#endif
@@ -1260,6 +1412,16 @@
errstr);
return;
}
+
+ if (events & POLLHUP) {
+ errno = EINVAL;
+ rd_kafka_broker_fail(rkb, LOG_ERR,
+ RD_KAFKA_RESP_ERR__AUTHENTICATION,
+ "Disconnected");
+
+ return;
+ }
+
break;
case RD_KAFKA_BROKER_STATE_APIVERSION_QUERY:
@@ -1271,18 +1433,19 @@
while (rkb->rkb_state >= RD_KAFKA_BROKER_STATE_UP &&
rd_kafka_recv(rkb) > 0)
;
- }
- if (events & POLLHUP) {
- rd_kafka_broker_fail(rkb,
- rkb->rkb_rk->rk_conf.
- log_connection_close ?
- LOG_NOTICE : LOG_DEBUG,
- RD_KAFKA_RESP_ERR__TRANSPORT,
- "Connection closed");
- return;
+ /* If connection went down: bail out early */
+ if (rkb->rkb_state == RD_KAFKA_BROKER_STATE_DOWN)
+ return;
}
+ if (events & POLLHUP) {
+ rd_kafka_broker_conn_closed(
+ rkb, RD_KAFKA_RESP_ERR__TRANSPORT,
+ "Disconnected");
+ return;
+ }
+
if (events & POLLOUT) {
while (rd_kafka_send(rkb) > 0)
;
@@ -1353,20 +1516,16 @@
socket_strerror(socket_errno));
#endif
- /* Enable TCP keep-alives, if configured. */
- if (rkb->rkb_rk->rk_conf.socket_keepalive) {
#ifdef SO_KEEPALIVE
- if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE,
- (void *)&on, sizeof(on)) == SOCKET_ERROR)
- rd_rkb_dbg(rkb, BROKER, "SOCKET",
- "Failed to set SO_KEEPALIVE: %s",
- socket_strerror(socket_errno));
-#else
- rd_rkb_dbg(rkb, BROKER, "SOCKET",
- "System does not support "
- "socket.keepalive.enable (SO_KEEPALIVE)");
+ /* Enable TCP keep-alives, if configured. */
+ if (rkb->rkb_rk->rk_conf.socket_keepalive) {
+ if (setsockopt(s, SOL_SOCKET, SO_KEEPALIVE,
+ (void *)&on, sizeof(on)) == SOCKET_ERROR)
+ rd_rkb_dbg(rkb, BROKER, "SOCKET",
+ "Failed to set SO_KEEPALIVE: %s",
+ socket_strerror(socket_errno));
+ }
#endif
- }
/* Set the socket to non-blocking */
if ((r = rd_fd_set_nonblocking(s))) {
diff -Nru librdkafka-0.11.3/src/rdkafka_transport.h librdkafka-0.11.6/src/rdkafka_transport.h
--- librdkafka-0.11.3/src/rdkafka_transport.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_transport.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_TRANSPORT_H_
+#define _RDKAFKA_TRANSPORT_H_
#ifndef _MSC_VER
#include
@@ -46,6 +47,10 @@
ssize_t rd_kafka_transport_recv (rd_kafka_transport_t *rktrans,
rd_buf_t *rbuf,
char *errstr, size_t errstr_size);
+
+void rd_kafka_transport_request_sent (rd_kafka_broker_t *rkb,
+ rd_kafka_buf_t *rkbuf);
+
int rd_kafka_transport_framed_recv (rd_kafka_transport_t *rktrans,
rd_kafka_buf_t **rkbufp,
char *errstr, size_t errstr_size);
@@ -70,3 +75,5 @@
#endif
void rd_kafka_transport_term (void);
void rd_kafka_transport_init(void);
+
+#endif /* _RDKAFKA_TRANSPORT_H_ */
diff -Nru librdkafka-0.11.3/src/rdkafka_transport_int.h librdkafka-0.11.6/src/rdkafka_transport_int.h
--- librdkafka-0.11.3/src/rdkafka_transport_int.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdkafka_transport_int.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKA_TRANSPORT_INT_H_
+#define _RDKAFKA_TRANSPORT_INT_H_
/* This header file is to be used by .c files needing access to the
* rd_kafka_transport_t struct internals. */
@@ -35,6 +36,7 @@
#if WITH_SSL
#include
#include
+#include
#endif
struct rd_kafka_transport_s {
@@ -82,3 +84,4 @@
size_t rktrans_sndbuf_size; /**< Socket send buffer size */
};
+#endif /* _RDKAFKA_TRANSPORT_INT_H_ */
diff -Nru librdkafka-0.11.3/src/rdlist.c librdkafka-0.11.6/src/rdlist.c
--- librdkafka-0.11.3/src/rdlist.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdlist.c 2018-10-10 06:54:38.000000000 +0000
@@ -60,14 +60,37 @@
return rl;
}
+rd_list_t *rd_list_init_copy (rd_list_t *dst, const rd_list_t *src) {
+
+ if (src->rl_flags & RD_LIST_F_FIXED_SIZE) {
+ /* Source was preallocated, prealloc new dst list */
+ rd_list_init(dst, 0, src->rl_free_cb);
+
+ rd_list_prealloc_elems(dst, src->rl_elemsize, src->rl_size,
+ 1/*memzero*/);
+ } else {
+ /* Source is dynamic, initialize dst the same */
+ rd_list_init(dst, rd_list_cnt(src), src->rl_free_cb);
+
+ }
+
+ return dst;
+}
+
+static RD_INLINE rd_list_t *rd_list_alloc (void) {
+ return malloc(sizeof(rd_list_t));
+}
+
rd_list_t *rd_list_new (int initial_size, void (*free_cb) (void *)) {
- rd_list_t *rl = malloc(sizeof(*rl));
+ rd_list_t *rl = rd_list_alloc();
rd_list_init(rl, initial_size, free_cb);
rl->rl_flags |= RD_LIST_F_ALLOCATED;
return rl;
}
-void rd_list_prealloc_elems (rd_list_t *rl, size_t elemsize, size_t size) {
+
+void rd_list_prealloc_elems (rd_list_t *rl, size_t elemsize, size_t cnt,
+ int memzero) {
size_t allocsize;
char *p;
size_t i;
@@ -79,19 +102,33 @@
* elems[elemsize][cnt];
*/
- allocsize = (sizeof(void *) * size) + (elemsize * size);
- rl->rl_elems = rd_malloc(allocsize);
-
- /* p points to first element's memory. */
- p = (char *)&rl->rl_elems[size];
+ allocsize = (sizeof(void *) * cnt) + (elemsize * cnt);
+ if (memzero)
+ rl->rl_elems = rd_calloc(1, allocsize);
+ else
+ rl->rl_elems = rd_malloc(allocsize);
+
+ /* p points to first element's memory, unless elemsize is 0. */
+ if (elemsize > 0)
+ p = rl->rl_p = (char *)&rl->rl_elems[cnt];
+ else
+ p = rl->rl_p = NULL;
/* Pointer -> elem mapping */
- for (i = 0 ; i < size ; i++, p += elemsize)
+ for (i = 0 ; i < cnt ; i++, p += elemsize)
rl->rl_elems[i] = p;
- rl->rl_size = (int)size;
+ rl->rl_size = (int)cnt;
rl->rl_cnt = 0;
rl->rl_flags |= RD_LIST_F_FIXED_SIZE;
+ rl->rl_elemsize = (int)elemsize;
+}
+
+
+void rd_list_set_cnt (rd_list_t *rl, size_t cnt) {
+ rd_assert(rl->rl_flags & RD_LIST_F_FIXED_SIZE);
+ rd_assert((int)cnt <= rl->rl_size);
+ rl->rl_cnt = (int)cnt;
}
@@ -110,7 +147,25 @@
return rl->rl_elems[rl->rl_cnt++];
}
-static void rd_list_remove0 (rd_list_t *rl, int idx) {
+void rd_list_set (rd_list_t *rl, int idx, void *ptr) {
+ if (idx >= rl->rl_size)
+ rd_list_grow(rl, idx+1);
+
+ if (idx >= rl->rl_cnt) {
+ memset(&rl->rl_elems[rl->rl_cnt], 0,
+ sizeof(*rl->rl_elems) * (idx-rl->rl_cnt));
+ rl->rl_cnt = idx+1;
+ } else {
+ /* Not allowed to replace existing element. */
+ rd_assert(!rl->rl_elems[idx]);
+ }
+
+ rl->rl_elems[idx] = ptr;
+}
+
+
+
+void rd_list_remove_elem (rd_list_t *rl, int idx) {
rd_assert(idx < rl->rl_cnt);
if (idx + 1 < rl->rl_cnt)
@@ -126,7 +181,7 @@
RD_LIST_FOREACH(elem, rl, i) {
if (elem == match_elem) {
- rd_list_remove0(rl, i);
+ rd_list_remove_elem(rl, i);
return elem;
}
}
@@ -143,7 +198,7 @@
RD_LIST_FOREACH(elem, rl, i) {
if (elem == match_elem ||
!cmp(elem, match_elem)) {
- rd_list_remove0(rl, i);
+ rd_list_remove_elem(rl, i);
return elem;
}
}
@@ -152,6 +207,26 @@
}
+int rd_list_remove_multi_cmp (rd_list_t *rl, void *match_elem,
+ int (*cmp) (void *_a, void *_b)) {
+
+ void *elem;
+ int i;
+ int cnt = 0;
+
+ /* Scan backwards to minimize memmoves */
+ RD_LIST_FOREACH_REVERSE(elem, rl, i) {
+ if (match_elem == cmp ||
+ !cmp(elem, match_elem)) {
+ rd_list_remove_elem(rl, i);
+ cnt++;
+ }
+ }
+
+ return cnt;
+}
+
+
/**
* Trampoline to avoid the double pointers in callbacks.
*
@@ -200,6 +275,9 @@
rd_free(rl);
}
+void rd_list_destroy_free (void *rl) {
+ rd_list_destroy((rd_list_t *)rl);
+}
void *rd_list_elem (const rd_list_t *rl, int idx) {
if (likely(idx < rl->rl_cnt))
@@ -207,6 +285,20 @@
return NULL;
}
+int rd_list_index (const rd_list_t *rl, const void *match,
+ int (*cmp) (const void *, const void *)) {
+ int i;
+ const void *elem;
+
+ RD_LIST_FOREACH(elem, rl, i) {
+ if (!cmp(match, elem))
+ return i;
+ }
+
+ return -1;
+}
+
+
void *rd_list_find (const rd_list_t *rl, const void *match,
int (*cmp) (const void *, const void *)) {
int i;
@@ -267,7 +359,7 @@
RD_LIST_FOREACH(elem, rl, i) {
if (!cb(elem, opaque)) {
- rd_list_remove0(rl, i);
+ rd_list_remove_elem(rl, i);
i--;
}
}
@@ -283,7 +375,6 @@
return (void *)elem;
}
-
rd_list_t *rd_list_copy (const rd_list_t *src,
void *(*copy_cb) (const void *elem, void *opaque),
void *opaque) {
@@ -302,6 +393,8 @@
void *elem;
int i;
+ rd_assert(dst != src);
+
if (!copy_cb)
copy_cb = rd_list_nocopy_ptr;
@@ -311,3 +404,75 @@
rd_list_add(dst, celem);
}
}
+
+
+/**
+ * @brief Copy elements of preallocated \p src to preallocated \p dst.
+ *
+ * @remark \p dst will be overwritten and initialized, but its
+ * flags will be retained.
+ *
+ * @returns \p dst
+ */
+static rd_list_t *rd_list_copy_preallocated0 (rd_list_t *dst,
+ const rd_list_t *src) {
+ int dst_flags = dst->rl_flags & RD_LIST_F_ALLOCATED;
+
+ rd_assert(dst != src);
+
+ rd_list_init_copy(dst, src);
+ dst->rl_flags |= dst_flags;
+
+ rd_assert((dst->rl_flags & RD_LIST_F_FIXED_SIZE));
+ rd_assert((src->rl_flags & RD_LIST_F_FIXED_SIZE));
+ rd_assert(dst->rl_elemsize == src->rl_elemsize &&
+ dst->rl_size == src->rl_size);
+
+ memcpy(dst->rl_p, src->rl_p, src->rl_elemsize * src->rl_size);
+ dst->rl_cnt = src->rl_cnt;
+
+ return dst;
+}
+
+void *rd_list_copy_preallocated (const void *elem, void *opaque) {
+ return rd_list_copy_preallocated0(rd_list_new(0, NULL),
+ (const rd_list_t *)elem);
+}
+
+
+/**
+ * @name Misc helpers for common list types
+ * @{
+ *
+ */
+rd_list_t *rd_list_init_int32 (rd_list_t *rl, int max_size) {
+ int rl_flags = rl->rl_flags & RD_LIST_F_ALLOCATED;
+ rd_list_init(rl, 0, NULL);
+ rl->rl_flags |= rl_flags;
+ rd_list_prealloc_elems(rl, sizeof(int32_t), max_size, 1/*memzero*/);
+ return rl;
+}
+
+void rd_list_set_int32 (rd_list_t *rl, int idx, int32_t val) {
+ rd_assert((rl->rl_flags & RD_LIST_F_FIXED_SIZE) &&
+ rl->rl_elemsize == sizeof(int32_t));
+ rd_assert(idx < rl->rl_size);
+
+ memcpy(rl->rl_elems[idx], &val, sizeof(int32_t));
+
+ if (rl->rl_cnt <= idx)
+ rl->rl_cnt = idx+1;
+}
+
+int32_t rd_list_get_int32 (const rd_list_t *rl, int idx) {
+ rd_assert((rl->rl_flags & RD_LIST_F_FIXED_SIZE) &&
+ rl->rl_elemsize == sizeof(int32_t) &&
+ idx < rl->rl_cnt);
+ return *(int32_t *)rl->rl_elems[idx];
+}
+
+
+
+
+/**@}*/
+
diff -Nru librdkafka-0.11.3/src/rdlist.h librdkafka-0.11.6/src/rdlist.h
--- librdkafka-0.11.3/src/rdlist.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdlist.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDLIST_H_
+#define _RDLIST_H_
/**
@@ -46,16 +47,20 @@
#define RD_LIST_F_SORTED 0x2 /* Set by sort(), cleared by any mutations.
* When this flag is set bsearch() is used
* by find(), otherwise a linear search. */
-#define RD_LIST_F_FIXED_SIZE 0x4 /* Assert on grow */
+#define RD_LIST_F_FIXED_SIZE 0x4 /* Assert on grow, when prealloc()ed */
#define RD_LIST_F_UNIQUE 0x8 /* Don't allow duplicates:
* ONLY ENFORCED BY CALLER. */
+ int rl_elemsize; /**< Element size (when prealloc()ed) */
+ void *rl_p; /**< Start of prealloced elements,
+ * the allocation itself starts at rl_elems
+ */
} rd_list_t;
/**
- * @brief Initialize a list, preallocate space for 'initial_size' elements
- * (optional).
- * List elements will optionally be freed by \p free_cb.
+ * @brief Initialize a list, prepare for 'initial_size' elements
+ * (optional optimization).
+ * List elements will optionally be freed by \p free_cb.
*
* @returns \p rl
*/
@@ -64,7 +69,16 @@
/**
- * Allocate a new list pointer and initialize it according to rd_list_init().
+ * @brief Same as rd_list_init() but uses initial_size and free_cb
+ * from the provided \p src list.
+ */
+rd_list_t *rd_list_init_copy (rd_list_t *rl, const rd_list_t *src);
+
+/**
+ * @brief Allocate a new list pointer and initialize
+ * it according to rd_list_init().
+ *
+ * This is the same as calling \c rd_list_init(rd_list_alloc(), ..));
*
* Use rd_list_destroy() to free.
*/
@@ -82,12 +96,21 @@
* rd_list_add(), instead pass NULL to rd_list_add() and use the returned
* pointer as the element.
*
- * @param elemsize element size
+ * @param elemsize element size, or 0 if elements are allocated separately.
* @param size number of elements
+ * @param memzero initialize element memory to zeros.
*
* @remark Preallocated element lists can't grow past \p size.
*/
-void rd_list_prealloc_elems (rd_list_t *rl, size_t elemsize, size_t size);
+void rd_list_prealloc_elems (rd_list_t *rl, size_t elemsize, size_t size,
+ int memzero);
+
+/**
+ * @brief Set the number of valid elements, this must only be used
+ * with prealloc_elems() to make the preallocated elements directly
+ * usable.
+ */
+void rd_list_set_cnt (rd_list_t *rl, size_t cnt);
/**
@@ -110,6 +133,16 @@
/**
+ * @brief Set element at \p idx to \p ptr.
+ *
+ * @remark MUST NOT overwrite an existing element.
+ * @remark The list will be grown, if needed, any gaps between the current
+ * highest element and \p idx will be set to NULL.
+ */
+void rd_list_set (rd_list_t *rl, int idx, void *ptr);
+
+
+/**
* Remove element from list.
* This is a slow O(n) + memmove operation.
* Returns the removed element.
@@ -123,6 +156,26 @@
void *rd_list_remove_cmp (rd_list_t *rl, void *match_elem,
int (*cmp) (void *_a, void *_b));
+
+/**
+ * @brief Remove element at index \p idx.
+ *
+ * This is a O(1) + memmove operation
+ */
+void rd_list_remove_elem (rd_list_t *rl, int idx);
+
+
+/**
+ * @brief Remove all elements matching comparator.
+ *
+ * @returns the number of elements removed.
+ *
+ * @sa rd_list_remove()
+ */
+int rd_list_remove_multi_cmp (rd_list_t *rl, void *match_elem,
+ int (*cmp) (void *_a, void *_b));
+
+
/**
* Sort list using comparator
*/
@@ -143,6 +196,12 @@
*/
void rd_list_destroy (rd_list_t *rl);
+/**
+ * @brief Wrapper for rd_list_destroy() that has same signature as free(3),
+ * allowing it to be used as free_cb for nested lists.
+ */
+void rd_list_destroy_free (void *rl);
+
/**
* Returns the element at index 'idx', or NULL if out of range.
@@ -176,11 +235,27 @@
#define rd_list_empty(rl) (rd_list_cnt(rl) == 0)
+/**
+ * @brief Find element index using comparator.
+ *
+ * \p match is the first argument to \p cmp, and each element (up to a match)
+ * is the second argument to \p cmp.
+ *
+ * @remark this is a O(n) scan.
+ * @returns the first matching element or NULL.
+ */
+int rd_list_index (const rd_list_t *rl, const void *match,
+ int (*cmp) (const void *, const void *));
/**
- * Find element using comparator
- * 'match' will be the first argument to 'cmp', and each element (up to a match)
- * will be the second argument to 'cmp'.
+ * @brief Find element using comparator
+ *
+ * \p match is the first argument to \p cmp, and each element (up to a match)
+ * is the second argument to \p cmp.
+ *
+ * @remark if the list is sorted bsearch() is used, otherwise an O(n) scan.
+ *
+ * @returns the first matching element or NULL.
*/
void *rd_list_find (const rd_list_t *rl, const void *match,
int (*cmp) (const void *, const void *));
@@ -231,6 +306,13 @@
void *(*copy_cb) (const void *elem, void *opaque),
void *opaque);
+
+/**
+ * @brief Copy callback to copy elements that are preallocated lists.
+ */
+void *rd_list_copy_preallocated (const void *elem, void *opaque);
+
+
/**
* @brief String copier for rd_list_copy()
*/
@@ -240,7 +322,48 @@
}
+
+/**
+ * @name Misc helpers for common list types
+ * @{
+ *
+ */
+
+/**
+ * @brief Init a new list of int32_t's of maximum size \p max_size
+ * where each element is pre-allocated.
+<<<<<<< HEAD
+=======
+ *
+ * @remark The allocation flag of the original \p rl is retained,
+ * do not pass an uninitialized \p rl to this function.
+>>>>>>> 96b11b2... Added builtin support for int32_t rd_list_t
+ */
+rd_list_t *rd_list_init_int32 (rd_list_t *rl, int max_size);
+
+
/**
* Debugging: Print list to stdout.
*/
void rd_list_dump (const char *what, const rd_list_t *rl);
+
+
+
+/**
+ * @brief Set element at index \p idx to value \p val.
+ *
+ * @remark Must only be used with preallocated int32_t lists.
+ * @remark Allows values to be overwritten.
+ */
+void rd_list_set_int32 (rd_list_t *rl, int idx, int32_t val);
+
+/**
+ * @returns the int32_t element value at index \p idx
+ *
+ * @remark Must only be used with preallocated int32_t lists.
+ */
+int32_t rd_list_get_int32 (const rd_list_t *rl, int idx);
+
+/**@}*/
+
+#endif /* _RDLIST_H_ */
diff -Nru librdkafka-0.11.3/src/rdlog.h librdkafka-0.11.6/src/rdlog.h
--- librdkafka-0.11.3/src/rdlog.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdlog.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDLOG_H_
+#define _RDLOG_H_
void rd_hexdump (FILE *fp, const char *name, const void *ptr, size_t len);
@@ -35,3 +36,5 @@
struct msghdr;
void rd_msghdr_print (const char *what, const struct msghdr *msg,
int hexdump);
+
+#endif /* _RDLOG_H_ */
diff -Nru librdkafka-0.11.3/src/rdmurmur2.c librdkafka-0.11.6/src/rdmurmur2.c
--- librdkafka-0.11.3/src/rdmurmur2.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdmurmur2.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,159 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "rd.h"
+#include "rdunittest.h"
+#include "rdmurmur2.h"
+#include "rdendian.h"
+
+
+/* MurmurHash2, by Austin Appleby
+ *
+ * With librdkafka modifications combinining aligned/unaligned variants
+ * into the same function.
+ */
+
+#define MM_MIX(h,k,m) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; }
+
+/*-----------------------------------------------------------------------------
+// Based on MurmurHashNeutral2, by Austin Appleby
+//
+// Same as MurmurHash2, but endian- and alignment-neutral.
+// Half the speed though, alas.
+//
+*/
+uint32_t rd_murmur2 (const void *key, size_t len) {
+ const uint32_t seed = 0x9747b28c;
+ const uint32_t m = 0x5bd1e995;
+ const int r = 24;
+ uint32_t h = seed ^ (uint32_t)len;
+ const unsigned char *tail;
+
+ if (likely(((intptr_t)key & 0x3) == 0)) {
+ /* Input is 32-bit word aligned. */
+ const uint32_t *data = (const uint32_t *)key;
+
+ while (len >= 4) {
+ uint32_t k = htole32(*(uint32_t *)data);
+
+ MM_MIX(h,k,m);
+
+ data++;
+ len -= 4;
+ }
+
+ tail = (const unsigned char *)data;
+
+ } else {
+ /* Unaligned slower variant */
+ const unsigned char *data = (const unsigned char *)key;
+
+ while (len >= 4) {
+ uint32_t k;
+
+ k = data[0];
+ k |= data[1] << 8;
+ k |= data[2] << 16;
+ k |= data[3] << 24;
+
+ MM_MIX(h,k,m);
+
+ data += 4;
+ len -= 4;
+ }
+
+ tail = data;
+ }
+
+ /* Read remaining sub-word */
+ switch(len)
+ {
+ case 3: h ^= tail[2] << 16;
+ case 2: h ^= tail[1] << 8;
+ case 1: h ^= tail[0];
+ h *= m;
+ };
+
+ h ^= h >> 13;
+ h *= m;
+ h ^= h >> 15;
+
+ /* Last bit is set to 0 because the java implementation uses int_32
+ * and then sets to positive number flipping last bit to 1. */
+ return h;
+}
+
+
+/**
+ * @brief Unittest for rd_murmur2()
+ */
+int unittest_murmur2 (void) {
+ const char *short_unaligned = "1234";
+ const char *unaligned = "PreAmbleWillBeRemoved,ThePrePartThatIs";
+ const char *keysToTest[] = {
+ "kafka",
+ "giberish123456789",
+ short_unaligned,
+ short_unaligned+1,
+ short_unaligned+2,
+ short_unaligned+3,
+ unaligned,
+ unaligned+1,
+ unaligned+2,
+ unaligned+3,
+ "",
+ NULL,
+ };
+
+ const int32_t java_murmur2_results[] = {
+ 0xd067cf64, // kafka
+ 0x8f552b0c, // giberish123456789
+ 0x9fc97b14, // short_unaligned
+ 0xe7c009ca, // short_unaligned+1
+ 0x873930da, // short_unaligned+2
+ 0x5a4b5ca1, // short_unaligned+3
+ 0x78424f1c, // unaligned
+ 0x4a62b377, // unaligned+1
+ 0xe0e4e09e, // unaligned+2
+ 0x62b8b43f, // unaligned+3
+ 0x106e08d9, // ""
+ 0x106e08d9, // NULL
+ };
+
+ size_t i;
+ for (i = 0; i < RD_ARRAYSIZE(keysToTest); i++) {
+ uint32_t h = rd_murmur2(keysToTest[i],
+ keysToTest[i] ?
+ strlen(keysToTest[i]) : 0);
+ RD_UT_ASSERT((int32_t)h == java_murmur2_results[i],
+ "Calculated murmur2 hash 0x%x for \"%s\", "
+ "expected 0x%x",
+ h, keysToTest[i], java_murmur2_results[i]);
+ }
+ RD_UT_PASS();
+}
diff -Nru librdkafka-0.11.3/src/rdmurmur2.h librdkafka-0.11.6/src/rdmurmur2.h
--- librdkafka-0.11.3/src/rdmurmur2.h 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/rdmurmur2.h 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,7 @@
+#ifndef __RDMURMUR2___H__
+#define __RDMURMUR2___H__
+
+uint32_t rd_murmur2 (const void *key, size_t len);
+int unittest_murmur2 (void);
+
+#endif // __RDMURMUR2___H__
diff -Nru librdkafka-0.11.3/src/rdports.h librdkafka-0.11.6/src/rdports.h
--- librdkafka-0.11.3/src/rdports.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdports.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,9 +25,12 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDPORTS_H_
+#define _RDPORTS_H_
void rd_qsort_r (void *base, size_t nmemb, size_t size,
int (*compar)(const void *, const void *, void *),
void *arg);
+
+#endif /* _RDPORTS_H_ */
diff -Nru librdkafka-0.11.3/src/rdposix.h librdkafka-0.11.6/src/rdposix.h
--- librdkafka-0.11.3/src/rdposix.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdposix.h 2018-10-10 06:54:38.000000000 +0000
@@ -1,4 +1,3 @@
-#pragma once
/*
* librdkafka - Apache Kafka C library
*
@@ -30,7 +29,8 @@
/**
* POSIX system support
*/
-#pragma once
+#ifndef _RDPOSIX_H_
+#define _RDPOSIX_H_
#include
#include
@@ -38,6 +38,7 @@
#include
#include
#include
+#include
/**
* Types
@@ -90,11 +91,15 @@
/**
* Errors
*/
+
+
+#define rd_set_errno(err) (errno = (err))
+
#if HAVE_STRERROR_R
static RD_INLINE RD_UNUSED const char *rd_strerror(int err) {
static RD_TLS char ret[128];
-#if defined(__linux__) && defined(_GNU_SOURCE)
+#if defined(__GLIBC__) && defined(_GNU_SOURCE)
return strerror_r(err, ret, sizeof(ret));
#else /* XSI version */
int r;
@@ -180,3 +185,5 @@
#define rd_read(fd,buf,sz) read(fd,buf,sz)
#define rd_write(fd,buf,sz) write(fd,buf,sz)
#define rd_close(fd) close(fd)
+
+#endif /* _RDPOSIX_H_ */
diff -Nru librdkafka-0.11.3/src/rdrand.h librdkafka-0.11.6/src/rdrand.h
--- librdkafka-0.11.3/src/rdrand.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdrand.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDRAND_H_
+#define _RDRAND_H_
/**
@@ -43,3 +44,5 @@
* Shuffles (randomizes) an array using the modern Fisher-Yates algorithm.
*/
void rd_array_shuffle (void *base, size_t nmemb, size_t entry_size);
+
+#endif /* _RDRAND_H_ */
diff -Nru librdkafka-0.11.3/src/rdregex.h librdkafka-0.11.6/src/rdregex.h
--- librdkafka-0.11.3/src/rdregex.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdregex.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDREGEX_H_
+#define _RDREGEX_H_
typedef struct rd_regex_s rd_regex_t;
@@ -35,3 +36,5 @@
int rd_regex_match (const char *pattern, const char *str,
char *errstr, size_t errstr_size);
+
+#endif /* _RDREGEX_H_ */
diff -Nru librdkafka-0.11.3/src/rdsignal.h librdkafka-0.11.6/src/rdsignal.h
--- librdkafka-0.11.3/src/rdsignal.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdsignal.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDSIGNAL_H_
+#define _RDSIGNAL_H_
#include
@@ -52,3 +53,5 @@
sigprocmask(SIG_UNBLOCK, &rd_intr_sigset, NULL);
}
+
+#endif /* _RDSIGNAL_H_ */
diff -Nru librdkafka-0.11.3/src/rdstring.c librdkafka-0.11.6/src/rdstring.c
--- librdkafka-0.11.3/src/rdstring.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdstring.c 2018-10-10 06:54:38.000000000 +0000
@@ -147,16 +147,98 @@
rd_free(strtup);
}
-rd_strtup_t *rd_strtup_new (const char *name, const char *value) {
- size_t name_sz = strlen(name) + 1;
- size_t value_sz = strlen(value) + 1;
+void rd_strtup_free (void *strtup) {
+ rd_strtup_destroy((rd_strtup_t *)strtup);
+}
+
+rd_strtup_t *rd_strtup_new0 (const char *name, ssize_t name_len,
+ const char *value, ssize_t value_len) {
rd_strtup_t *strtup;
+ /* Calculate lengths, if needed, and add space for \0 nul */
+
+ if (name_len == -1)
+ name_len = strlen(name);
+
+ if (!value)
+ value_len = 0;
+ else if (value_len == -1)
+ value_len = strlen(value);
+
+
strtup = rd_malloc(sizeof(*strtup) +
- name_sz + value_sz - 1/*name[1]*/);
- memcpy(strtup->name, name, name_sz);
- strtup->value = &strtup->name[name_sz];
- memcpy(strtup->value, value, value_sz);
+ name_len + 1 + value_len + 1 - 1/*name[1]*/);
+ memcpy(strtup->name, name, name_len);
+ strtup->name[name_len] = '\0';
+ if (value) {
+ strtup->value = &strtup->name[name_len+1];
+ memcpy(strtup->value, value, value_len);
+ strtup->value[value_len] = '\0';
+ } else {
+ strtup->value = NULL;
+ }
return strtup;
}
+
+rd_strtup_t *rd_strtup_new (const char *name, const char *value) {
+ return rd_strtup_new0(name, -1, value, -1);
+}
+
+
+/**
+ * @returns a new copy of \p src
+ */
+rd_strtup_t *rd_strtup_dup (const rd_strtup_t *src) {
+ return rd_strtup_new(src->name, src->value);
+}
+
+/**
+ * @brief Wrapper for rd_strtup_dup() suitable rd_list_copy*() use
+ */
+void *rd_strtup_list_copy (const void *elem, void *opaque) {
+ const rd_strtup_t *src = elem;
+ return (void *)rd_strtup_dup(src);
+}
+
+
+
+/**
+ * @brief Convert bit-flags in \p flags to human-readable CSV string
+ * use the bit-description strings in \p desc.
+ *
+ * \p desc array element N corresponds to bit (1<= size) {
+ /* Dest buffer too small, indicate truncation */
+ if (size > 3)
+ rd_snprintf(dst+(size-3), 3, "..");
+ break;
+ }
+
+ r = rd_snprintf(dst+of, size-of, "%s%s",
+ !of ? "" : ",", *desc);
+
+ of += r;
+ }
+
+ if (of == 0 && size > 0)
+ *dst = '\0';
+
+ return dst;
+}
diff -Nru librdkafka-0.11.3/src/rdstring.h librdkafka-0.11.6/src/rdstring.h
--- librdkafka-0.11.3/src/rdstring.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdstring.h 2018-10-10 06:54:38.000000000 +0000
@@ -27,7 +27,8 @@
*/
-#pragma once
+#ifndef _RDSTRING_H_
+#define _RDSTRING_H_
@@ -42,6 +43,7 @@
/**
* @brief An immutable string tuple (name, value) in a single allocation.
+ * \p value may be NULL.
*/
typedef struct rd_strtup_s {
char *value;
@@ -49,4 +51,14 @@
} rd_strtup_t;
void rd_strtup_destroy (rd_strtup_t *strtup);
+void rd_strtup_free (void *strtup);
+rd_strtup_t *rd_strtup_new0 (const char *name, ssize_t name_len,
+ const char *value, ssize_t value_len);
rd_strtup_t *rd_strtup_new (const char *name, const char *value);
+rd_strtup_t *rd_strtup_dup (const rd_strtup_t *strtup);
+void *rd_strtup_list_copy (const void *elem, void *opaque);
+
+char *rd_flags2str (char *dst, size_t size,
+ const char **desc, int flags);
+
+#endif /* _RDSTRING_H_ */
diff -Nru librdkafka-0.11.3/src/rdsysqueue.h librdkafka-0.11.6/src/rdsysqueue.h
--- librdkafka-0.11.3/src/rdsysqueue.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdsysqueue.h 2018-10-10 06:54:38.000000000 +0000
@@ -53,7 +53,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDSYSQUEUE_H_
+#define _RDSYSQUEUE_H_
#include "queue.h"
@@ -243,6 +244,22 @@
} while (0)
#endif
+/* @brief Insert \p shead after element \p listelm in \p dhead */
+#define TAILQ_INSERT_LIST(dhead,listelm,shead,headname,elmtype,field) do { \
+ if (TAILQ_LAST(dhead, headname) == listelm) { \
+ TAILQ_CONCAT(dhead, shead, field); \
+ } else { \
+ elmtype _elm = TAILQ_FIRST(shead); \
+ elmtype _last = TAILQ_LAST(shead, headname); \
+ elmtype _aft = TAILQ_NEXT(listelm, field); \
+ (listelm)->field.tqe_next = _elm; \
+ _elm->field.tqe_prev = &(listelm)->field.tqe_next; \
+ _last->field.tqe_next = _aft; \
+ _aft->field.tqe_prev = &_last->field.tqe_next; \
+ TAILQ_INIT((shead)); \
+ } \
+ } while (0)
+
#ifndef SIMPLEQ_HEAD
#define SIMPLEQ_HEAD(name, type) \
struct name { \
@@ -328,3 +345,4 @@
+#endif /* _RDSYSQUEUE_H_ */
diff -Nru librdkafka-0.11.3/src/rdtime.h librdkafka-0.11.6/src/rdtime.h
--- librdkafka-0.11.3/src/rdtime.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdtime.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDTIME_H_
+#define _RDTIME_H_
#ifndef TIMEVAL_TO_TIMESPEC
@@ -60,6 +61,17 @@
#define RD_POLL_NOWAIT 0
+#if RD_UNITTEST_QPC_OVERRIDES
+ /* Overrides for rd_clock() unittest using QPC on Windows */
+BOOL rd_ut_QueryPerformanceFrequency(_Out_ LARGE_INTEGER * lpFrequency);
+BOOL rd_ut_QueryPerformanceCounter(_Out_ LARGE_INTEGER * lpPerformanceCount);
+#define rd_QueryPerformanceFrequency(IFREQ) rd_ut_QueryPerformanceFrequency(IFREQ)
+#define rd_QueryPerformanceCounter(PC) rd_ut_QueryPerformanceCounter(PC)
+#else
+#define rd_QueryPerformanceFrequency(IFREQ) QueryPerformanceFrequency(IFREQ)
+#define rd_QueryPerformanceCounter(PC) QueryPerformanceCounter(PC)
+#endif
+
/**
* @returns a monotonically increasing clock in microseconds.
* @remark There is no monotonic clock on OSX, the system time
@@ -73,7 +85,17 @@
gettimeofday(&tv, NULL);
return ((rd_ts_t)tv.tv_sec * 1000000LLU) + (rd_ts_t)tv.tv_usec;
#elif defined(_MSC_VER)
- return (rd_ts_t)GetTickCount64() * 1000LLU;
+ LARGE_INTEGER now;
+ static RD_TLS double freq = 0.0;
+ if (!freq) {
+ LARGE_INTEGER ifreq;
+ rd_QueryPerformanceFrequency(&ifreq);
+ /* Convert frequency to double to avoid overflow in
+ * return statement */
+ freq = (double)ifreq.QuadPart / 1000000.0;
+ }
+ rd_QueryPerformanceCounter(&now);
+ return (rd_ts_t)((double)now.QuadPart / freq);
#else
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
@@ -133,6 +155,49 @@
/**
+ * @brief Initialize an absolute timespec timeout based on the provided
+ * relative \p timeout_ms.
+ *
+ * To be used with cnd_timedwait_abs().
+ *
+ * Honours RD_POLL_INFITE and RD_POLL_NOWAIT (reflected in tspec.tv_sec).
+ */
+static RD_INLINE void rd_timeout_init_timespec (struct timespec *tspec,
+ int timeout_ms) {
+ if (timeout_ms == RD_POLL_INFINITE ||
+ timeout_ms == RD_POLL_NOWAIT) {
+ tspec->tv_sec = timeout_ms;
+ tspec->tv_nsec = 0;
+ } else {
+ timespec_get(tspec, TIME_UTC);
+ tspec->tv_sec += timeout_ms / 1000;
+ tspec->tv_nsec += (timeout_ms % 1000) * 1000000;
+ if (tspec->tv_nsec > 1000000000) {
+ tspec->tv_nsec -= 1000000000;
+ tspec->tv_sec++;
+ }
+ }
+}
+
+
+/**
+ * @brief Same as rd_timeout_remains() but with microsecond precision
+ */
+static RD_INLINE rd_ts_t rd_timeout_remains_us (rd_ts_t abs_timeout) {
+ rd_ts_t timeout_us;
+
+ if (abs_timeout == RD_POLL_INFINITE ||
+ abs_timeout == RD_POLL_NOWAIT)
+ return (rd_ts_t)abs_timeout;
+
+ timeout_us = abs_timeout - rd_clock();
+ if (timeout_us <= 0)
+ return RD_POLL_NOWAIT;
+ else
+ return timeout_us;
+}
+
+/**
* @returns the remaining timeout for timeout \p abs_timeout previously set
* up by rd_timeout_init()
*
@@ -145,17 +210,13 @@
* in a bool fashion.
*/
static RD_INLINE int rd_timeout_remains (rd_ts_t abs_timeout) {
- int timeout_ms;
+ rd_ts_t timeout_us = rd_timeout_remains_us(abs_timeout);
- if (abs_timeout == RD_POLL_INFINITE ||
- abs_timeout == RD_POLL_NOWAIT)
- return (int)abs_timeout;
-
- timeout_ms = (int)((abs_timeout - rd_clock()) / 1000);
- if (timeout_ms <= 0)
- return RD_POLL_NOWAIT;
- else
- return timeout_ms;
+ if (timeout_us == RD_POLL_INFINITE ||
+ timeout_us == RD_POLL_NOWAIT)
+ return (int)timeout_us;
+
+ return (int)(timeout_us / 1000);
}
/**
@@ -179,3 +240,5 @@
static RD_INLINE int rd_timeout_expired (int timeout_ms) {
return timeout_ms == RD_POLL_NOWAIT;
}
+
+#endif /* _RDTIME_H_ */
diff -Nru librdkafka-0.11.3/src/rdtypes.h librdkafka-0.11.6/src/rdtypes.h
--- librdkafka-0.11.3/src/rdtypes.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdtypes.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDTYPES_H_
+#define _RDTYPES_H_
#include
@@ -40,3 +41,10 @@
typedef int64_t rd_ts_t;
#define RD_TS_MAX INT64_MAX
+
+
+typedef uint8_t rd_bool_t;
+#define rd_true 1
+#define rd_false 0
+
+#endif /* _RDTYPES_H_ */
diff -Nru librdkafka-0.11.3/src/rdunittest.c librdkafka-0.11.6/src/rdunittest.c
--- librdkafka-0.11.3/src/rdunittest.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdunittest.c 2018-10-10 06:54:38.000000000 +0000
@@ -26,18 +26,390 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
+#ifdef _MSC_VER
+#define RD_UNITTEST_QPC_OVERRIDES 1
+#endif
+
#include "rd.h"
#include "rdunittest.h"
#include "rdvarint.h"
#include "rdbuf.h"
#include "crc32c.h"
+#include "rdmurmur2.h"
+#if WITH_HDRHISTOGRAM
+#include "rdhdrhistogram.h"
+#endif
+#include "rdkafka_int.h"
+
+#include "rdsysqueue.h"
+
+
+/**
+ * @name Test rdsysqueue.h / queue.h
+ * @{
+ */
+
+struct ut_tq {
+ TAILQ_ENTRY(ut_tq) link;
+ int v;
+};
+
+TAILQ_HEAD(ut_tq_head, ut_tq);
+
+struct ut_tq_args {
+ const char *name; /**< Descriptive test name */
+ struct {
+ int base; /**< Base value */
+ int cnt; /**< Number of elements to add */
+ int step; /**< Value step */
+ } q[3]; /**< Queue element definition */
+ int qcnt; /**< Number of defs in .q */
+ int exp[16]; /**< Expected value order after join */
+};
+
+/**
+ * @brief Find the previous element (insert position) for
+ * value \p val in list \p head or NULL if \p val is less than
+ * the first element in \p head.
+ * @remarks \p head must be ascending sorted.
+ */
+static struct ut_tq *ut_tq_find_prev_pos (const struct ut_tq_head *head,
+ int val) {
+ struct ut_tq *e, *prev = NULL;
+
+ TAILQ_FOREACH(e, head, link) {
+ if (e->v > val)
+ return prev;
+ prev = e;
+ }
+
+ return prev;
+}
+
+static int ut_tq_test (const struct ut_tq_args *args) {
+ int totcnt = 0;
+ int fails = 0;
+ struct ut_tq_head *tqh[3];
+ struct ut_tq *e, *insert_after;
+ int i, qi;
+
+ RD_UT_SAY("Testing TAILQ: %s", args->name);
+
+ /*
+ * Verify TAILQ_INSERT_LIST:
+ * For each insert position test:
+ * - create two lists: tqh 0 and 1
+ * - add entries to both lists
+ * - insert list 1 into 0
+ * - verify expected order and correctness
+ */
+
+ /* Use heap allocated heads to let valgrind/asan assist
+ * in detecting corruption. */
+
+ for (qi = 0 ; qi < args->qcnt ; qi++) {
+ tqh[qi] = rd_calloc(1, sizeof(*tqh[qi]));
+ TAILQ_INIT(tqh[qi]);
+
+ for (i = 0 ; i < args->q[qi].cnt ; i++) {
+ e = rd_malloc(sizeof(*e));
+ e->v = args->q[qi].base + (i * args->q[qi].step);
+ TAILQ_INSERT_TAIL(tqh[qi], e, link);
+ }
+
+ totcnt += args->q[qi].cnt;
+ }
+
+ for (qi = 1 ; qi < args->qcnt ; qi++) {
+ insert_after = ut_tq_find_prev_pos(tqh[0], args->q[qi].base);
+ if (!insert_after) {
+ /* Insert position is head of list,
+ * do two-step concat+move */
+ TAILQ_CONCAT(tqh[qi], tqh[0], link); /* append */
+ TAILQ_MOVE(tqh[0], tqh[qi], link); /* replace */
+ } else {
+ TAILQ_INSERT_LIST(tqh[0], insert_after, tqh[qi],
+ ut_tq_head,
+ struct ut_tq *, link);
+ }
+
+ RD_UT_ASSERT(TAILQ_EMPTY(tqh[qi]),
+ "expected empty tqh[%d]", qi);
+ RD_UT_ASSERT(!TAILQ_EMPTY(tqh[0]), "expected non-empty tqh[0]");
+
+ memset(tqh[qi], (int)'A', sizeof(*tqh[qi]));
+ rd_free(tqh[qi]);
+ }
+
+ RD_UT_ASSERT(TAILQ_LAST(tqh[0], ut_tq_head)->v == args->exp[totcnt-1],
+ "TAILQ_LAST val %d, expected %d",
+ TAILQ_LAST(tqh[0], ut_tq_head)->v, args->exp[totcnt-1]);
+
+ /* Add sentinel value to verify that INSERT_TAIL works
+ * after INSERT_LIST */
+ e = rd_malloc(sizeof(*e));
+ e->v = 99;
+ TAILQ_INSERT_TAIL(tqh[0], e, link);
+ totcnt++;
+
+ i = 0;
+ TAILQ_FOREACH(e, tqh[0], link) {
+ if (i >= totcnt) {
+ RD_UT_WARN("Too many elements in list tqh[0]: "
+ "idx %d > totcnt %d: element %p (value %d)",
+ i, totcnt, e, e->v);
+ fails++;
+ } else if (e->v != args->exp[i]) {
+ RD_UT_WARN("Element idx %d/%d in tqh[0] has value %d, "
+ "expected %d",
+ i, totcnt, e->v, args->exp[i]);
+ fails++;
+ } else if (i == totcnt - 1 &&
+ e != TAILQ_LAST(tqh[0], ut_tq_head)) {
+ RD_UT_WARN("TAILQ_LAST == %p, expected %p",
+ TAILQ_LAST(tqh[0], ut_tq_head), e);
+ fails++;
+ }
+ i++;
+ }
+
+ /* Then scan it in reverse */
+ i = totcnt - 1;
+ TAILQ_FOREACH_REVERSE(e, tqh[0], ut_tq_head, link) {
+ if (i < 0) {
+ RD_UT_WARN("REVERSE: Too many elements in list tqh[0]: "
+ "idx %d < 0: element %p (value %d)",
+ i, e, e->v);
+ fails++;
+ } else if (e->v != args->exp[i]) {
+ RD_UT_WARN("REVERSE: Element idx %d/%d in tqh[0] has "
+ "value %d, expected %d",
+ i, totcnt, e->v, args->exp[i]);
+ fails++;
+ } else if (i == totcnt - 1 &&
+ e != TAILQ_LAST(tqh[0], ut_tq_head)) {
+ RD_UT_WARN("REVERSE: TAILQ_LAST == %p, expected %p",
+ TAILQ_LAST(tqh[0], ut_tq_head), e);
+ fails++;
+ }
+ i--;
+ }
+
+ RD_UT_ASSERT(TAILQ_LAST(tqh[0], ut_tq_head)->v == args->exp[totcnt-1],
+ "TAILQ_LAST val %d, expected %d",
+ TAILQ_LAST(tqh[0], ut_tq_head)->v, args->exp[totcnt-1]);
+
+ while ((e = TAILQ_FIRST(tqh[0]))) {
+ TAILQ_REMOVE(tqh[0], e, link);
+ rd_free(e);
+ }
+
+ rd_free(tqh[0]);
+
+ return fails;
+}
+
+
+static int unittest_sysqueue (void) {
+ const struct ut_tq_args args[] = {
+ {
+ "empty tqh[0]",
+ {
+ { 0, 0, 0 },
+ { 0, 3, 1 }
+ },
+ 2,
+ { 0, 1, 2, 99 /*sentinel*/ }
+ },
+ {
+ "prepend 1,0",
+ {
+ { 10, 3, 1 },
+ { 0, 3, 1 }
+ },
+ 2,
+ { 0, 1, 2, 10, 11, 12, 99 }
+ },
+ {
+ "prepend 2,1,0",
+ {
+ { 10, 3, 1 }, /* 10, 11, 12 */
+ { 5, 3, 1 }, /* 5, 6, 7 */
+ { 0, 2, 1 } /* 0, 1 */
+ },
+ 3,
+ { 0, 1, 5, 6, 7, 10, 11, 12, 99 }
+ },
+ {
+ "insert 1",
+ {
+ { 0, 3, 2 },
+ { 1, 2, 2 }
+ },
+ 2,
+ { 0, 1, 3, 2, 4, 99 }
+ },
+ {
+ "insert 1,2",
+ {
+ { 0, 3, 3 }, /* 0, 3, 6 */
+ { 1, 2, 3 }, /* 1, 4 */
+ { 2, 1, 3 } /* 2 */
+ },
+ 3,
+ { 0, 1, 2, 4, 3, 6, 99 }
+ },
+ {
+ "append 1",
+ {
+ { 0, 5, 1 },
+ { 5, 5, 1 }
+ },
+ 2,
+ { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 99 }
+ },
+ {
+ "append 1,2",
+ {
+ { 0, 5, 1 }, /* 0, 1, 2, 3, 4 */
+ { 5, 5, 1 }, /* 5, 6, 7, 8, 9 */
+ { 11, 2, 1 } /* 11, 12 */
+ },
+ 3,
+ { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 99 }
+ },
+ {
+ "insert 1,0,2",
+ {
+ { 5, 3, 1 }, /* 5, 6, 7 */
+ { 0, 1, 1 }, /* 0 */
+ { 10, 2, 1 } /* 10, 11 */
+ },
+ 3,
+ { 0, 5, 6, 7, 10, 11, 99 },
+ },
+ {
+ "insert 2,0,1",
+ {
+ { 5, 3, 1 }, /* 5, 6, 7 */
+ { 10, 2, 1 }, /* 10, 11 */
+ { 0, 1, 1 } /* 0 */
+ },
+ 3,
+ { 0, 5, 6, 7, 10, 11, 99 },
+ },
+ {
+ NULL
+ }
+ };
+ int i;
+ int fails = 0;
+
+ for (i = 0 ; args[i].name != NULL; i++)
+ fails += ut_tq_test(&args[i]);
+
+ RD_UT_ASSERT(!fails, "See %d previous failure(s)", fails);
+
+ RD_UT_PASS();
+}
+
+/**@}*/
+
+
+/**
+ * @name rd_clock() unittests
+ * @{
+ */
+
+#if RD_UNITTEST_QPC_OVERRIDES
+
+/**
+ * These values are based off a machine with freq 14318180
+ * which would cause the original rd_clock() calculation to overflow
+ * after about 8 days.
+ * Details:
+ * https://github.com/confluentinc/confluent-kafka-dotnet/issues/603#issuecomment-417274540
+ */
+
+static const int64_t rd_ut_qpc_freq = 14318180;
+static int64_t rd_ut_qpc_now;
+
+BOOL rd_ut_QueryPerformanceFrequency(_Out_ LARGE_INTEGER * lpFrequency) {
+ lpFrequency->QuadPart = rd_ut_qpc_freq;
+ return TRUE;
+}
+
+BOOL rd_ut_QueryPerformanceCounter(_Out_ LARGE_INTEGER * lpPerformanceCount) {
+ lpPerformanceCount->QuadPart = rd_ut_qpc_now * rd_ut_qpc_freq;
+ return TRUE;
+}
+
+static int unittest_rdclock (void) {
+ rd_ts_t t1, t2;
+
+ /* First let "uptime" be fresh boot (0). */
+ rd_ut_qpc_now = 0;
+ t1 = rd_clock();
+ rd_ut_qpc_now++;
+ t2 = rd_clock();
+ RD_UT_ASSERT(t2 == t1 + (1 * 1000000),
+ "Expected t2 %"PRId64" to be 1s more than t1 %"PRId64,
+ t2, t1);
+
+ /* Then skip forward to 8 days, which should trigger the
+ * overflow in a faulty implementation. */
+ rd_ut_qpc_now = 8 * 86400;
+ t2 = rd_clock();
+ RD_UT_ASSERT(t2 == t1 + (8LL * 86400 * 1000000),
+ "Expected t2 %"PRId64" to be 8 days larger than t1 %"PRId64,
+ t2, t1);
+
+ /* And make sure we can run on a system with 38 years of uptime.. */
+ rd_ut_qpc_now = 38 * 365 * 86400;
+ t2 = rd_clock();
+ RD_UT_ASSERT(t2 == t1 + (38LL * 365 * 86400 * 1000000),
+ "Expected t2 %"PRId64" to be 38 years larger than t1 %"PRId64,
+ t2, t1);
+
+ RD_UT_PASS();
+}
+#endif
+
+
+
+/**@}*/
int rd_unittest (void) {
int fails = 0;
- fails += unittest_rdbuf();
- fails += unittest_rdvarint();
- fails += unittest_crc32c();
+ const struct {
+ const char *name;
+ int (*call) (void);
+ } unittests[] = {
+ { "sysqueue", unittest_sysqueue },
+ { "rdbuf", unittest_rdbuf },
+ { "rdvarint", unittest_rdvarint },
+ { "crc32c", unittest_crc32c },
+ { "msg", unittest_msg },
+ { "murmurhash", unittest_murmur2 },
+#if WITH_HDRHISTOGRAM
+ { "rdhdrhistogram", unittest_rdhdrhistogram },
+#endif
+#ifdef _MSC_VER
+ { "rdclock", unittest_rdclock },
+#endif
+ { NULL }
+ };
+ int i;
+
+ for (i = 0 ; unittests[i].name ; i++) {
+ int f = unittests[i].call();
+ RD_UT_SAY("unittest: %s: %4s\033[0m",
+ unittests[i].name,
+ f ? "\033[31mFAIL" : "\033[32mPASS");
+ fails += f;
+ }
+
return fails;
}
diff -Nru librdkafka-0.11.3/src/rdunittest.h librdkafka-0.11.6/src/rdunittest.h
--- librdkafka-0.11.3/src/rdunittest.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdunittest.h 2018-10-10 06:54:38.000000000 +0000
@@ -78,6 +78,17 @@
} while (0)
+/**
+ * @brief Warn about something from a unit-test
+ */
+#define RD_UT_WARN(...) do { \
+ fprintf(stderr, "\033[33mRDUT: WARN: %s:%d: %s: ", \
+ __FILE__, __LINE__, __FUNCTION__); \
+ fprintf(stderr, __VA_ARGS__); \
+ fprintf(stderr, "\033[0m\n"); \
+ } while (0)
+
+
int rd_unittest (void);
#endif /* _RD_UNITTEST_H */
diff -Nru librdkafka-0.11.3/src/rdwin32.h librdkafka-0.11.6/src/rdwin32.h
--- librdkafka-0.11.3/src/rdwin32.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/rdwin32.h 2018-10-10 06:54:38.000000000 +0000
@@ -29,7 +29,8 @@
/**
* Win32 (Visual Studio) support
*/
-#pragma once
+#ifndef _RDWIN32_H_
+#define _RDWIN32_H_
#include
@@ -136,6 +137,16 @@
/**
* Errors
*/
+
+/* MSVC:
+ * This is the correct way to set errno on Windows,
+ * but it is still pointless due to different errnos in
+ * in different runtimes:
+ * https://social.msdn.microsoft.com/Forums/vstudio/en-US/b4500c0d-1b69-40c7-9ef5-08da1025b5bf/setting-errno-from-within-a-dll?forum=vclanguage/
+ * errno is thus highly deprecated, and buggy, on Windows
+ * when using librdkafka as a dynamically loaded DLL. */
+#define rd_set_errno(err) _set_errno((err))
+
static RD_INLINE RD_UNUSED const char *rd_strerror(int err) {
static RD_TLS char ret[128];
@@ -212,37 +223,75 @@
* @returns 0 on success or errno on failure
*/
static RD_UNUSED int rd_pipe_nonblocking (int *fds) {
- HANDLE h[2];
- int i;
+ /* On windows, the "pipe" will be a tcp connection.
+ * This is to allow WSAPoll to be used to poll pipe events */
- if (!CreatePipe(&h[0], &h[1], NULL, 0))
- return (int)GetLastError();
- for (i = 0 ; i < 2 ; i++) {
- DWORD mode = PIPE_NOWAIT;
- /* Set non-blocking */
- if (!SetNamedPipeHandleState(h[i], &mode, NULL, NULL)) {
- CloseHandle(h[0]);
- CloseHandle(h[1]);
- return (int)GetLastError();
- }
-
- /* Open file descriptor for handle */
- fds[i] = _open_osfhandle((intptr_t)h[i],
- i == 0 ?
- O_RDONLY | O_BINARY :
- O_WRONLY | O_BINARY);
-
- if (fds[i] == -1) {
- CloseHandle(h[0]);
- CloseHandle(h[1]);
- return (int)GetLastError();
- }
- }
+ SOCKET listen_s = INVALID_SOCKET;
+ SOCKET accept_s = INVALID_SOCKET;
+ SOCKET connect_s = INVALID_SOCKET;
+
+ struct sockaddr_in listen_addr;
+ struct sockaddr_in connect_addr;
+ socklen_t sock_len = 0;
+
+ /* Create listen socket */
+ listen_s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (listen_s == INVALID_SOCKET)
+ goto err;
+
+ listen_addr.sin_family = AF_INET;
+ listen_addr.sin_addr.s_addr = ntohl(INADDR_LOOPBACK);
+ listen_addr.sin_port = 0;
+ if (bind(listen_s, (struct sockaddr*)&listen_addr, sizeof(listen_addr)) != 0)
+ goto err;
+
+ sock_len = sizeof(connect_addr);
+ if (getsockname(listen_s, (struct sockaddr*)&connect_addr, &sock_len) != 0)
+ goto err;
+
+ if (listen(listen_s, 1) != 0)
+ goto err;
+
+ /* Create connection socket */
+ connect_s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (connect_s == INVALID_SOCKET)
+ goto err;
+
+ if (connect(connect_s, (struct sockaddr*)&connect_addr, sizeof(connect_addr)) == SOCKET_ERROR)
+ goto err;
+
+ /* Wait for incoming connection */
+ accept_s = accept(listen_s, NULL, NULL);
+ if (accept_s == SOCKET_ERROR)
+ goto err;
+
+ /* Done with listening */
+ closesocket(listen_s);
+
+ if (rd_fd_set_nonblocking((int)accept_s) != 0)
+ goto err;
+
+ if (rd_fd_set_nonblocking((int)connect_s) != 0)
+ goto err;
+
+ /* Store resulting sockets. They are bidirectional, so it does not matter
+ * which is read or write side of pipe. */
+ fds[0] = (int)accept_s;
+ fds[1] = (int)connect_s;
return 0;
+
+ err:
+ if (listen_s != INVALID_SOCKET)
+ closesocket(listen_s);
+ if (accept_s != INVALID_SOCKET)
+ closesocket(accept_s);
+ if (connect_s != INVALID_SOCKET)
+ closesocket(connect_s);
+ return -1;
}
-#define rd_read(fd,buf,sz) _read(fd,buf,sz)
-#define rd_write(fd,buf,sz) _write(fd,buf,sz)
+#define rd_read(fd,buf,sz) recv(fd,buf,sz,0)
+#define rd_write(fd,buf,sz) send(fd,buf,sz,0)
#define rd_close(fd) closesocket(fd)
static RD_UNUSED char *
@@ -260,3 +309,5 @@
}
#endif /* !__cplusplus*/
+
+#endif /* _RDWIN32_H_ */
diff -Nru librdkafka-0.11.3/src/statistics_schema.json librdkafka-0.11.6/src/statistics_schema.json
--- librdkafka-0.11.3/src/statistics_schema.json 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/statistics_schema.json 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,422 @@
+{ "$schema": "http://json-schema.org/schema#",
+ "id": "https://github.com/edenhill/librdkafka/src/statistics_schema.json",
+ "title": "librdkafka statistics schema",
+ "definitions": {
+ "window": {
+ "type": "object",
+ "title": "Rolling window statistics",
+ "description": "The values are in microseconds unless otherwise stated.",
+ "properties": {
+ "type": "object",
+ "properties": {
+ "min": {
+ "type": "integer"
+ },
+ "max": {
+ "type": "integer"
+ },
+ "avg": {
+ "type": "integer"
+ },
+ "sum": {
+ "type": "integer"
+ },
+ "stddev": {
+ "type": "integer"
+ },
+ "p50": {
+ "type": "integer"
+ },
+ "p75": {
+ "type": "integer"
+ },
+ "p90": {
+ "type": "integer"
+ },
+ "p95": {
+ "type": "integer"
+ },
+ "p99": {
+ "type": "integer"
+ },
+ "p99_99": {
+ "type": "integer"
+ },
+ "outofrange": {
+ "type": "integer"
+ },
+ "hdrsize": {
+ "type": "integer"
+ },
+ "cnt": {
+ "type": "integer"
+ }
+ }
+ }
+ }
+ },
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "client_id": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string"
+ },
+ "ts": {
+ "type": "integer"
+ },
+ "time": {
+ "type": "integer"
+ },
+ "replyq": {
+ "type": "integer"
+ },
+ "msg_cnt": {
+ "type": "integer"
+ },
+ "msg_size": {
+ "type": "integer"
+ },
+ "msg_max": {
+ "type": "integer"
+ },
+ "msg_size_max": {
+ "type": "integer"
+ },
+ "simple_cnt": {
+ "type": "integer"
+ },
+ "metadata_cache_cnt": {
+ "type": "integer"
+ },
+ "brokers": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "object",
+ "title": "Broker object keyed by the broker \"name:port/id\"",
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "nodeid": {
+ "type": "integer"
+ },
+ "state": {
+ "type": "string"
+ },
+ "stateage": {
+ "type": "integer"
+ },
+ "outbuf_cnt": {
+ "type": "integer"
+ },
+ "outbuf_msg_cnt": {
+ "type": "integer"
+ },
+ "waitresp_cnt": {
+ "type": "integer"
+ },
+ "waitresp_msg_cnt": {
+ "type": "integer"
+ },
+ "tx": {
+ "type": "integer"
+ },
+ "txbytes": {
+ "type": "integer"
+ },
+ "txerrs": {
+ "type": "integer"
+ },
+ "txretries": {
+ "type": "integer"
+ },
+ "req_timeouts": {
+ "type": "integer"
+ },
+ "rx": {
+ "type": "integer"
+ },
+ "rxbytes": {
+ "type": "integer"
+ },
+ "rxerrs": {
+ "type": "integer"
+ },
+ "rxcorriderrs": {
+ "type": "integer"
+ },
+ "rxpartial": {
+ "type": "integer"
+ },
+ "zbuf_grow": {
+ "type": "integer"
+ },
+ "buf_grow": {
+ "type": "integer"
+ },
+ "wakeups": {
+ "type": "integer"
+ },
+ "int_latency": {
+ "$ref": "#/definitions/window"
+ },
+ "outbuf_latency": {
+ "$ref": "#/definitions/window"
+ },
+ "rtt": {
+ "$ref": "#/definitions/window"
+ },
+ "throttle": {
+ "$ref": "#/definitions/window"
+ },
+ "toppars": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "topic": {
+ "type": "string"
+ },
+ "partition": {
+ "type": "integer"
+ }
+ },
+ "required": [
+ "topic",
+ "partition"
+ ]
+ }
+ }
+ },
+ "required": [
+ "name",
+ "nodeid",
+ "state",
+ "stateage",
+ "outbuf_cnt",
+ "outbuf_msg_cnt",
+ "waitresp_cnt",
+ "waitresp_msg_cnt",
+ "tx",
+ "txbytes",
+ "txerrs",
+ "txretries",
+ "req_timeouts",
+ "rx",
+ "rxbytes",
+ "rxerrs",
+ "rxcorriderrs",
+ "rxpartial",
+ "zbuf_grow",
+ "buf_grow",
+ "wakeups",
+ "int_latency",
+ "rtt",
+ "throttle",
+ "toppars"
+ ]
+ }
+ },
+ "topics": {
+ "type": "object",
+ "properties": {
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "topic": {
+ "type": "string"
+ },
+ "metadata_age": {
+ "type": "integer"
+ },
+ "batchsize": {
+ "$ref": "#/definitions/window"
+ },
+ "batchcnt": {
+ "$ref": "#/definitions/window"
+ },
+ "partitions": {
+ "type": "object",
+ "properties": {
+ "^-?[0-9]+$": {
+ "type": "object",
+ "properties": {
+ "partition": {
+ "type": "integer"
+ },
+ "leader": {
+ "type": "integer"
+ },
+ "desired": {
+ "type": "boolean"
+ },
+ "unknown": {
+ "type": "boolean"
+ },
+ "msgq_cnt": {
+ "type": "integer"
+ },
+ "msgq_bytes": {
+ "type": "integer"
+ },
+ "xmit_msgq_cnt": {
+ "type": "integer"
+ },
+ "xmit_msgq_bytes": {
+ "type": "integer"
+ },
+ "fetchq_cnt": {
+ "type": "integer"
+ },
+ "fetchq_size": {
+ "type": "integer"
+ },
+ "fetch_state": {
+ "type": "string"
+ },
+ "query_offset": {
+ "type": "integer"
+ },
+ "next_offset": {
+ "type": "integer"
+ },
+ "app_offset": {
+ "type": "integer"
+ },
+ "stored_offset": {
+ "type": "integer"
+ },
+ "commited_offset": {
+ "type": "integer"
+ },
+ "committed_offset": {
+ "type": "integer"
+ },
+ "eof_offset": {
+ "type": "integer"
+ },
+ "lo_offset": {
+ "type": "integer"
+ },
+ "hi_offset": {
+ "type": "integer"
+ },
+ "consumer_lag": {
+ "type": "integer"
+ },
+ "txmsgs": {
+ "type": "integer"
+ },
+ "txbytes": {
+ "type": "integer"
+ },
+ "rxmsgs": {
+ "type": "integer"
+ },
+ "rxbytes": {
+ "type": "integer"
+ },
+ "msgs": {
+ "type": "integer"
+ },
+ "rx_ver_drops": {
+ "type": "integer"
+ }
+ },
+ "required": [
+ "partition",
+ "leader",
+ "desired",
+ "unknown",
+ "msgq_cnt",
+ "msgq_bytes",
+ "xmit_msgq_cnt",
+ "xmit_msgq_bytes",
+ "fetchq_cnt",
+ "fetchq_size",
+ "fetch_state",
+ "query_offset",
+ "next_offset",
+ "app_offset",
+ "stored_offset",
+ "commited_offset",
+ "committed_offset",
+ "eof_offset",
+ "lo_offset",
+ "hi_offset",
+ "consumer_lag",
+ "txmsgs",
+ "txbytes",
+ "rxmsgs",
+ "rxbytes",
+ "msgs",
+ "rx_ver_drops"
+ ]
+ }
+ }
+ }
+ },
+ "required": [
+ "topic",
+ "metadata_age",
+ "batchsize",
+ "partitions"
+ ]
+ }
+ }
+ },
+ "tx": {
+ "type": "integer"
+ },
+ "tx_bytes": {
+ "type": "integer"
+ },
+ "rx": {
+ "type": "integer"
+ },
+ "rx_bytes": {
+ "type": "integer"
+ },
+ "txmsgs": {
+ "type": "integer"
+ },
+ "txmsg_bytes": {
+ "type": "integer"
+ },
+ "rxmsgs": {
+ "type": "integer"
+ },
+ "rxmsg_bytes": {
+ "type": "integer"
+ }
+ },
+ "required": [
+ "name",
+ "client_id",
+ "type",
+ "ts",
+ "time",
+ "replyq",
+ "msg_cnt",
+ "msg_size",
+ "msg_max",
+ "msg_size_max",
+ "simple_cnt",
+ "metadata_cache_cnt",
+ "brokers",
+ "topics",
+ "tx",
+ "tx_bytes",
+ "rx",
+ "rx_bytes",
+ "txmsgs",
+ "txmsg_bytes",
+ "rxmsgs",
+ "rxmsg_bytes"
+ ]
+}
diff -Nru librdkafka-0.11.3/src/tinycthread.c librdkafka-0.11.6/src/tinycthread.c
--- librdkafka-0.11.3/src/tinycthread.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/tinycthread.c 2018-10-10 06:54:38.000000000 +0000
@@ -23,10 +23,10 @@
*/
#include "rd.h"
-#include "rdtime.h"
-#include "tinycthread.h"
#include
+#if !WITH_C11THREADS
+
/* Platform specific includes */
#if defined(_TTHREAD_POSIX_)
#include
@@ -387,7 +387,7 @@
}
#if defined(_TTHREAD_WIN32_)
-static int _cnd_timedwait_win32(cnd_t *cond, mtx_t *mtx, DWORD timeout)
+int _cnd_timedwait_win32(cnd_t *cond, mtx_t *mtx, DWORD timeout)
{
int result, lastWaiter;
@@ -476,47 +476,6 @@
}
-int cnd_timedwait_ms(cnd_t *cnd, mtx_t *mtx, int timeout_ms) {
- if (timeout_ms == -1 /* INFINITE*/)
- return cnd_wait(cnd, mtx);
-#if defined(_TTHREAD_WIN32_)
- return _cnd_timedwait_win32(cnd, mtx, (DWORD)timeout_ms);
-#else
- int ret;
- struct timeval tv;
- struct timespec ts;
-
- gettimeofday(&tv, NULL);
- ts.tv_sec = tv.tv_sec;
- ts.tv_nsec = tv.tv_usec * 1000;
-
- ts.tv_sec += timeout_ms / 1000;
- ts.tv_nsec += (timeout_ms % 1000) * 1000000;
-
- if (ts.tv_nsec >= 1000000000) {
- ts.tv_sec++;
- ts.tv_nsec -= 1000000000;
- }
-
- ret = pthread_cond_timedwait(cnd, mtx, &ts);
- if (ret == ETIMEDOUT)
- {
- return thrd_timedout;
- }
- return ret == 0 ? thrd_success : thrd_error;
-#endif
-}
-
-int cnd_timedwait_msp (cnd_t *cnd, mtx_t *mtx, int *timeout_msp) {
- rd_ts_t pre = rd_clock();
- int r;
- r = cnd_timedwait_ms(cnd, mtx, *timeout_msp);
- if (r != thrd_timedout) {
- /* Subtract spent time */
- (*timeout_msp) -= (int)(rd_clock()-pre) / 1000;
- }
- return r;
-}
#if defined(_TTHREAD_WIN32_)
struct TinyCThreadTSSData {
@@ -680,15 +639,6 @@
#endif
}
-int thrd_is_current(thrd_t thr) {
-#if defined(_TTHREAD_WIN32_)
- return GetThreadId(thr) == GetCurrentThreadId();
-#else
- return (pthread_self() == thr);
-#endif
-}
-
-
int thrd_detach(thrd_t thr)
{
thrd_is_detached = 1;
@@ -981,51 +931,9 @@
#endif /* defined(_TTHREAD_WIN32_) */
-#if !defined(_TTHREAD_WIN32_)
-int rwlock_init (rwlock_t *rwl) {
- int r = pthread_rwlock_init(rwl, NULL);
- if (r) {
- errno = r;
- return thrd_error;
- }
- return thrd_success;
-}
-
-int rwlock_destroy (rwlock_t *rwl) {
- int r = pthread_rwlock_destroy(rwl);
- if (r) {
- errno = r;
- return thrd_error;
- }
- return thrd_success;
-}
-
-int rwlock_rdlock (rwlock_t *rwl) {
- int r = pthread_rwlock_rdlock(rwl);
- assert(r == 0);
- return thrd_success;
-}
-
-int rwlock_wrlock (rwlock_t *rwl) {
- int r = pthread_rwlock_wrlock(rwl);
- assert(r == 0);
- return thrd_success;
-}
-
-int rwlock_rdunlock (rwlock_t *rwl) {
- int r = pthread_rwlock_unlock(rwl);
- assert(r == 0);
- return thrd_success;
-}
-
-int rwlock_wrunlock (rwlock_t *rwl) {
- int r = pthread_rwlock_unlock(rwl);
- assert(r == 0);
- return thrd_success;
-}
-
-#endif /* !defined(_TTHREAD_WIN32_) */
#ifdef __cplusplus
}
#endif
+
+#endif /* !WITH_C11THREADS */
diff -Nru librdkafka-0.11.3/src/tinycthread_extra.c librdkafka-0.11.6/src/tinycthread_extra.c
--- librdkafka-0.11.3/src/tinycthread_extra.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/tinycthread_extra.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,153 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+/**
+ * @brief Extra methods added to tinychtread/c11threads
+ */
+
+#include "rd.h"
+#include "rdtime.h"
+#include "tinycthread.h"
+
+
+int thrd_setname (const char *name) {
+#if HAVE_PTHREAD_SETNAME_GNU
+ if (!pthread_setname_np(pthread_self(), name))
+ return thrd_success;
+#endif
+ return thrd_error;
+}
+
+int thrd_is_current(thrd_t thr) {
+#if defined(_TTHREAD_WIN32_)
+ return GetThreadId(thr) == GetCurrentThreadId();
+#else
+ return (pthread_self() == thr);
+#endif
+}
+
+
+
+
+int cnd_timedwait_ms(cnd_t *cnd, mtx_t *mtx, int timeout_ms) {
+ if (timeout_ms == -1 /* INFINITE*/)
+ return cnd_wait(cnd, mtx);
+#if defined(_TTHREAD_WIN32_)
+ return _cnd_timedwait_win32(cnd, mtx, (DWORD)timeout_ms);
+#else
+ struct timeval tv;
+ struct timespec ts;
+
+ gettimeofday(&tv, NULL);
+ ts.tv_sec = tv.tv_sec;
+ ts.tv_nsec = tv.tv_usec * 1000;
+
+ ts.tv_sec += timeout_ms / 1000;
+ ts.tv_nsec += (timeout_ms % 1000) * 1000000;
+
+ if (ts.tv_nsec >= 1000000000) {
+ ts.tv_sec++;
+ ts.tv_nsec -= 1000000000;
+ }
+
+ return cnd_timedwait(cnd, mtx, &ts);
+#endif
+}
+
+int cnd_timedwait_msp (cnd_t *cnd, mtx_t *mtx, int *timeout_msp) {
+ rd_ts_t pre = rd_clock();
+ int r;
+ r = cnd_timedwait_ms(cnd, mtx, *timeout_msp);
+ if (r != thrd_timedout) {
+ /* Subtract spent time */
+ (*timeout_msp) -= (int)(rd_clock()-pre) / 1000;
+ }
+ return r;
+}
+
+int cnd_timedwait_abs (cnd_t *cnd, mtx_t *mtx, const struct timespec *tspec) {
+ if (tspec->tv_sec == RD_POLL_INFINITE)
+ return cnd_wait(cnd, mtx);
+ else if (tspec->tv_sec == RD_POLL_NOWAIT)
+ return thrd_timedout;
+
+ return cnd_timedwait(cnd, mtx, tspec);
+}
+
+
+/**
+ * @name Read-write locks
+ * @{
+ */
+#ifndef _MSC_VER
+int rwlock_init (rwlock_t *rwl) {
+ int r = pthread_rwlock_init(rwl, NULL);
+ if (r) {
+ errno = r;
+ return thrd_error;
+ }
+ return thrd_success;
+}
+
+int rwlock_destroy (rwlock_t *rwl) {
+ int r = pthread_rwlock_destroy(rwl);
+ if (r) {
+ errno = r;
+ return thrd_error;
+ }
+ return thrd_success;
+}
+
+int rwlock_rdlock (rwlock_t *rwl) {
+ int r = pthread_rwlock_rdlock(rwl);
+ assert(r == 0);
+ return thrd_success;
+}
+
+int rwlock_wrlock (rwlock_t *rwl) {
+ int r = pthread_rwlock_wrlock(rwl);
+ assert(r == 0);
+ return thrd_success;
+}
+
+int rwlock_rdunlock (rwlock_t *rwl) {
+ int r = pthread_rwlock_unlock(rwl);
+ assert(r == 0);
+ return thrd_success;
+}
+
+int rwlock_wrunlock (rwlock_t *rwl) {
+ int r = pthread_rwlock_unlock(rwl);
+ assert(r == 0);
+ return thrd_success;
+}
+/**@}*/
+
+
+#endif /* !_MSC_VER */
diff -Nru librdkafka-0.11.3/src/tinycthread_extra.h librdkafka-0.11.6/src/tinycthread_extra.h
--- librdkafka-0.11.3/src/tinycthread_extra.h 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/src/tinycthread_extra.h 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,117 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018 Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+/**
+ * @brief Extra methods added to tinychtread/c11threads
+ */
+
+
+#ifndef _TINYCTHREAD_EXTRA_H_
+#define _TINYCTHREAD_EXTRA_H_
+
+
+#ifndef _MSC_VER
+#include /* needed for rwlock_t */
+#endif
+
+
+/**
+ * @brief Set thread system name if platform supports it (pthreads)
+ * @return thrd_success or thrd_error
+ */
+int thrd_setname (const char *name);
+
+/**
+ * @brief Checks if passed thread is the current thread.
+ * @return non-zero if same thread, else 0.
+ */
+int thrd_is_current(thrd_t thr);
+
+
+
+
+/**
+ * @brief Same as cnd_timedwait() but takes a relative timeout in milliseconds.
+ */
+int cnd_timedwait_ms(cnd_t *cnd, mtx_t *mtx, int timeout_ms);
+
+/**
+ * @brief Same as cnd_timedwait_ms() but updates the remaining time.
+*/
+int cnd_timedwait_msp (cnd_t *cnd, mtx_t *mtx, int *timeout_msp);
+
+/**
+ * @brief Same as cnd_timedwait() but honours
+ * RD_POLL_INFINITE (uses cnd_wait()),
+ * and RD_POLL_NOWAIT (return thrd_timedout immediately).
+ *
+ * @remark Set up \p tspec with rd_timeout_init_timespec().
+ */
+int cnd_timedwait_abs (cnd_t *cnd, mtx_t *mtx, const struct timespec *tspec);
+
+
+
+
+/**
+ * @brief Read-write locks
+ */
+
+#if defined(_TTHREAD_WIN32_)
+typedef struct rwlock_t {
+ SRWLOCK lock;
+ int rcnt;
+ int wcnt;
+} rwlock_t;
+#define rwlock_init(rwl) do { (rwl)->rcnt = (rwl)->wcnt = 0; InitializeSRWLock(&(rwl)->lock); } while (0)
+#define rwlock_destroy(rwl)
+#define rwlock_rdlock(rwl) do { if (0) printf("Thr %i: at %i: RDLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt >= 0); AcquireSRWLockShared(&(rwl)->lock); InterlockedIncrement(&(rwl)->rcnt); } while (0)
+#define rwlock_wrlock(rwl) do { if (0) printf("Thr %i: at %i: WRLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt >= 0); AcquireSRWLockExclusive(&(rwl)->lock); InterlockedIncrement(&(rwl)->wcnt); } while (0)
+#define rwlock_rdunlock(rwl) do { if (0) printf("Thr %i: at %i: RDUNLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt > 0 && (rwl)->wcnt >= 0); ReleaseSRWLockShared(&(rwl)->lock); InterlockedDecrement(&(rwl)->rcnt); } while (0)
+#define rwlock_wrunlock(rwl) do { if (0) printf("Thr %i: at %i: RWUNLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt > 0); ReleaseSRWLockExclusive(&(rwl)->lock); InterlockedDecrement(&(rwl)->wcnt); } while (0)
+
+#define rwlock_rdlock_d(rwl) do { if (1) printf("Thr %i: at %i: RDLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt >= 0); AcquireSRWLockShared(&(rwl)->lock); InterlockedIncrement(&(rwl)->rcnt); } while (0)
+#define rwlock_wrlock_d(rwl) do { if (1) printf("Thr %i: at %i: WRLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt >= 0); AcquireSRWLockExclusive(&(rwl)->lock); InterlockedIncrement(&(rwl)->wcnt); } while (0)
+#define rwlock_rdunlock_d(rwl) do { if (1) printf("Thr %i: at %i: RDUNLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt > 0 && (rwl)->wcnt >= 0); ReleaseSRWLockShared(&(rwl)->lock); InterlockedDecrement(&(rwl)->rcnt); } while (0)
+#define rwlock_wrunlock_d(rwl) do { if (1) printf("Thr %i: at %i: RWUNLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt > 0); ReleaseSRWLockExclusive(&(rwl)->lock); InterlockedDecrement(&(rwl)->wcnt); } while (0)
+
+
+#else
+typedef pthread_rwlock_t rwlock_t;
+
+int rwlock_init (rwlock_t *rwl);
+int rwlock_destroy (rwlock_t *rwl);
+int rwlock_rdlock (rwlock_t *rwl);
+int rwlock_wrlock (rwlock_t *rwl);
+int rwlock_rdunlock (rwlock_t *rwl);
+int rwlock_wrunlock (rwlock_t *rwl);
+
+#endif
+
+
+#endif /* _TINYCTHREAD_EXTRA_H_ */
diff -Nru librdkafka-0.11.3/src/tinycthread.h librdkafka-0.11.6/src/tinycthread.h
--- librdkafka-0.11.3/src/tinycthread.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/tinycthread.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,6 +25,17 @@
#ifndef _TINYCTHREAD_H_
#define _TINYCTHREAD_H_
+/* Include config to know if C11 threads are available */
+#ifdef _MSC_VER
+#include "win32_config.h"
+#else
+#include "../config.h"
+#endif
+
+#if WITH_C11THREADS
+#include
+#else
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -83,6 +94,9 @@
/* Platform specific includes */
#if defined(_TTHREAD_POSIX_)
+ #ifndef _GNU_SOURCE
+ #define _GNU_SOURCE /* for pthread_setname_np() */
+ #endif
#include
#elif defined(_TTHREAD_WIN32_)
#ifndef WIN32_LEAN_AND_MEAN
@@ -310,12 +324,9 @@
*/
int cnd_timedwait(cnd_t *cond, mtx_t *mtx, const struct timespec *ts);
-/** Same as cnd_timedwait() but takes a relative timeout in milliseconds.
- */
-int cnd_timedwait_ms(cnd_t *cnd, mtx_t *mtx, int timeout_ms);
-
-/** Same as cnd_timedwait_ms() but updates the remaining time. */
-int cnd_timedwait_msp (cnd_t *cnd, mtx_t *mtx, int *timeout_msp);
+#if defined(_TTHREAD_WIN32_)
+int _cnd_timedwait_win32(cnd_t *cond, mtx_t *mtx, DWORD timeout);
+#endif
/* Thread */
#if defined(_TTHREAD_WIN32_)
@@ -354,12 +365,6 @@
thrd_t thrd_current(void);
-/** Checks if passed thread is the current thread.
- * @return non-zero if same thread, else 0.
- */
-int thrd_is_current(thrd_t thr);
-
-
/** Dispose of any resources allocated to the thread when that thread exits.
* @return thrd_success, or thrd_error on error
*/
@@ -480,41 +485,15 @@
-
-/**
-* FIXME: description */
-#if defined(_TTHREAD_WIN32_)
-typedef struct rwlock_t {
- SRWLOCK lock;
- int rcnt;
- int wcnt;
-} rwlock_t;
-#define rwlock_init(rwl) do { (rwl)->rcnt = (rwl)->wcnt = 0; InitializeSRWLock(&(rwl)->lock); } while (0)
-#define rwlock_destroy(rwl)
-#define rwlock_rdlock(rwl) do { if (0) printf("Thr %i: at %i: RDLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt >= 0); AcquireSRWLockShared(&(rwl)->lock); InterlockedIncrement(&(rwl)->rcnt); } while (0)
-#define rwlock_wrlock(rwl) do { if (0) printf("Thr %i: at %i: WRLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt >= 0); AcquireSRWLockExclusive(&(rwl)->lock); InterlockedIncrement(&(rwl)->wcnt); } while (0)
-#define rwlock_rdunlock(rwl) do { if (0) printf("Thr %i: at %i: RDUNLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt > 0 && (rwl)->wcnt >= 0); ReleaseSRWLockShared(&(rwl)->lock); InterlockedDecrement(&(rwl)->rcnt); } while (0)
-#define rwlock_wrunlock(rwl) do { if (0) printf("Thr %i: at %i: RWUNLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt > 0); ReleaseSRWLockExclusive(&(rwl)->lock); InterlockedDecrement(&(rwl)->wcnt); } while (0)
-
-#define rwlock_rdlock_d(rwl) do { if (1) printf("Thr %i: at %i: RDLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt >= 0); AcquireSRWLockShared(&(rwl)->lock); InterlockedIncrement(&(rwl)->rcnt); } while (0)
-#define rwlock_wrlock_d(rwl) do { if (1) printf("Thr %i: at %i: WRLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt >= 0); AcquireSRWLockExclusive(&(rwl)->lock); InterlockedIncrement(&(rwl)->wcnt); } while (0)
-#define rwlock_rdunlock_d(rwl) do { if (1) printf("Thr %i: at %i: RDUNLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt > 0 && (rwl)->wcnt >= 0); ReleaseSRWLockShared(&(rwl)->lock); InterlockedDecrement(&(rwl)->rcnt); } while (0)
-#define rwlock_wrunlock_d(rwl) do { if (1) printf("Thr %i: at %i: RWUNLOCK %p %s (%i, %i)\n", GetCurrentThreadId(), __LINE__, rwl, __FUNCTION__, (rwl)->rcnt, (rwl)->wcnt); assert((rwl)->rcnt >= 0 && (rwl)->wcnt > 0); ReleaseSRWLockExclusive(&(rwl)->lock); InterlockedDecrement(&(rwl)->wcnt); } while (0)
-
-
-#else
-typedef pthread_rwlock_t rwlock_t;
-
-int rwlock_init (rwlock_t *rwl);
-int rwlock_destroy (rwlock_t *rwl);
-int rwlock_rdlock (rwlock_t *rwl);
-int rwlock_wrlock (rwlock_t *rwl);
-int rwlock_rdunlock (rwlock_t *rwl);
-int rwlock_wrunlock (rwlock_t *rwl);
-
-#endif
#ifdef __cplusplus
}
#endif
+#endif /* !WITH_C11THREADS */
+
+/**
+ * @brief librdkafka extensions to c11threads
+ */
+#include "tinycthread_extra.h"
+
#endif /* _TINYTHREAD_H_ */
diff -Nru librdkafka-0.11.3/src/win32_config.h librdkafka-0.11.6/src/win32_config.h
--- librdkafka-0.11.3/src/win32_config.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src/win32_config.h 2018-10-10 06:54:38.000000000 +0000
@@ -29,12 +29,18 @@
/**
* Hand-crafted config header file for Win32 builds.
*/
-#pragma once
+#ifndef _RD_WIN32_CONFIG_H_
+#define _RD_WIN32_CONFIG_H_
+#ifndef WITHOUT_WIN32_CONFIG
#define WITH_SSL 1
#define WITH_ZLIB 1
#define WITH_SNAPPY 1
#define WITH_SASL_SCRAM 1
#define ENABLE_DEVEL 0
#define WITH_PLUGINS 1
+#define WITH_HDRHISTOGRAM 1
+#endif
#define SOLIB_EXT ".dll"
+
+#endif /* _RD_WIN32_CONFIG_H_ */
diff -Nru librdkafka-0.11.3/src-cpp/CMakeLists.txt librdkafka-0.11.6/src-cpp/CMakeLists.txt
--- librdkafka-0.11.3/src-cpp/CMakeLists.txt 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src-cpp/CMakeLists.txt 2018-10-10 06:54:38.000000000 +0000
@@ -17,7 +17,12 @@
# Support '#include '
target_include_directories(rdkafka++ PUBLIC "$")
-
+if(NOT RDKAFKA_BUILD_STATIC)
+ target_compile_definitions(rdkafka++ PRIVATE LIBRDKAFKACPP_EXPORTS)
+ if(WIN32)
+ target_compile_definitions(rdkafka++ INTERFACE LIBRDKAFKACPP_EXPORTS=0)
+ endif()
+endif()
install(
TARGETS rdkafka++
EXPORT "${targets_export_name}"
diff -Nru librdkafka-0.11.3/src-cpp/Makefile librdkafka-0.11.6/src-cpp/Makefile
--- librdkafka-0.11.3/src-cpp/Makefile 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src-cpp/Makefile 2018-10-10 06:54:38.000000000 +0000
@@ -43,16 +43,8 @@
check: file-check
install: lib-install
+uninstall: lib-uninstall
clean: lib-clean
-ifeq ($(WITH_LDS),y)
-# Enable linker script if supported by platform
-LIB_LDFLAGS+= $(LDFLAG_LINKERSCRIPT)$(LIBNAME).lds
-endif
-
-$(LIBNAME).lds: $(HDRS)
- @(printf "$(MKL_YELLOW)Generating linker script $@ from $(HDRS)$(MKL_CLR_RESET)\n" ; \
- cat ../src/rdkafka.h | ../lds-gen.py > $@)
-
-include $(DEPS)
diff -Nru librdkafka-0.11.3/src-cpp/ProducerImpl.cpp librdkafka-0.11.6/src-cpp/ProducerImpl.cpp
--- librdkafka-0.11.3/src-cpp/ProducerImpl.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src-cpp/ProducerImpl.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -43,7 +43,7 @@
rkmessage,
void *opaque) {
RdKafka::HandleImpl *handle = static_cast(opaque);
- RdKafka::MessageImpl message(NULL, rkmessage);
+ RdKafka::MessageImpl message(NULL, (rd_kafka_message_t *)rkmessage, false);
handle->dr_cb_->dr_cb(message);
}
diff -Nru librdkafka-0.11.3/src-cpp/rdkafkacpp.h librdkafka-0.11.6/src-cpp/rdkafkacpp.h
--- librdkafka-0.11.3/src-cpp/rdkafkacpp.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src-cpp/rdkafkacpp.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKACPP_H_
+#define _RDKAFKACPP_H_
/**
* @file rdkafkacpp.h
@@ -69,6 +70,13 @@
/**@endcond*/
+extern "C" {
+ /* Forward declarations */
+ struct rd_kafka_s;
+ struct rd_kafka_topic_s;
+ struct rd_kafka_message_s;
+}
+
namespace RdKafka {
@@ -91,7 +99,7 @@
* @remark This value should only be used during compile time,
* for runtime checks of version use RdKafka::version()
*/
-#define RD_KAFKA_VERSION 0x000b03ff
+#define RD_KAFKA_VERSION 0x000b06ff
/**
* @brief Returns the librdkafka version as integer.
@@ -238,6 +246,15 @@
ERR__VALUE_DESERIALIZATION = -159,
/** Partial response */
ERR__PARTIAL = -158,
+ /** Modification attempted on read-only object */
+ ERR__READ_ONLY = -157,
+ /** No such entry / item not found */
+ ERR__NOENT = -156,
+ /** Read underflow */
+ ERR__UNDERFLOW = -155,
+ /** Invalid type */
+ ERR__INVALID_TYPE = -154,
+
/** End internal error codes */
ERR__END = -100,
@@ -1143,6 +1160,41 @@
* retrieved in the allotted timespan.
*/
virtual const std::string clusterid (int timeout_ms) = 0;
+
+ /**
+ * @brief Returns the underlying librdkafka C rd_kafka_t handle.
+ *
+ * @warning Calling the C API on this handle is not recommended and there
+ * is no official support for it, but for cases where the C++
+ * does not provide the proper functionality this C handle can be
+ * used to interact directly with the core librdkafka API.
+ *
+ * @remark The lifetime of the returned pointer is the same as the Topic
+ * object this method is called on.
+ *
+ * @remark Include prior to including
+ *
+ *
+ * @returns \c rd_kafka_t*
+ */
+ virtual struct rd_kafka_s *c_ptr () = 0;
+
+ /**
+ * @brief Returns the current ControllerId (controller broker id)
+ * as reported in broker metadata.
+ *
+ * @param timeout_ms If there is no cached value from metadata retrieval
+ * then this specifies the maximum amount of time
+ * (in milliseconds) the call will block waiting
+ * for metadata to be retrieved.
+ * Use 0 for non-blocking calls.
+ *
+ * @remark Requires broker version >=0.10.0 and api.version.request=true.
+ *
+ * @returns Last cached ControllerId, or -1 if no ControllerId could be
+ * retrieved in the allotted timespan.
+ */
+ virtual int32_t controllerid (int timeout_ms) = 0;
};
@@ -1259,6 +1311,24 @@
* offsets could be stored.
*/
virtual ErrorCode offset_store (int32_t partition, int64_t offset) = 0;
+
+ /**
+ * @brief Returns the underlying librdkafka C rd_kafka_topic_t handle.
+ *
+ * @warning Calling the C API on this handle is not recommended and there
+ * is no official support for it, but for cases where the C++ API
+ * does not provide the underlying functionality this C handle can be
+ * used to interact directly with the core librdkafka API.
+ *
+ * @remark The lifetime of the returned pointer is the same as the Topic
+ * object this method is called on.
+ *
+ * @remark Include prior to including
+ *
+ *
+ * @returns \c rd_kafka_topic_t*
+ */
+ virtual struct rd_kafka_topic_s *c_ptr () = 0;
};
@@ -1363,6 +1433,24 @@
/** @returns the latency in microseconds for a produced message measured
* from the produce() call, or -1 if latency is not available. */
virtual int64_t latency () const = 0;
+
+ /**
+ * @brief Returns the underlying librdkafka C rd_kafka_message_t handle.
+ *
+ * @warning Calling the C API on this handle is not recommended and there
+ * is no official support for it, but for cases where the C++ API
+ * does not provide the underlying functionality this C handle can be
+ * used to interact directly with the core librdkafka API.
+ *
+ * @remark The lifetime of the returned pointer is the same as the Message
+ * object this method is called on.
+ *
+ * @remark Include prior to including
+ *
+ *
+ * @returns \c rd_kafka_message_t*
+ */
+ virtual struct rd_kafka_message_s *c_ptr () = 0;
};
/**@}*/
@@ -1506,7 +1594,7 @@
* topics's partitions to the consumers, depending on their subscription.
*
* The result of such an assignment is a rebalancing which is either
- * handled automatically in librdkafka or can be overriden by the application
+ * handled automatically in librdkafka or can be overridden by the application
* by providing a RdKafka::RebalanceCb.
*
* The rebalancing passes the assigned partition set to
@@ -2212,3 +2300,4 @@
}
+#endif /* _RDKAFKACPP_H_ */
diff -Nru librdkafka-0.11.3/src-cpp/rdkafkacpp_int.h librdkafka-0.11.6/src-cpp/rdkafkacpp_int.h
--- librdkafka-0.11.3/src-cpp/rdkafkacpp_int.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src-cpp/rdkafkacpp_int.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RDKAFKACPP_INT_H_
+#define _RDKAFKACPP_INT_H_
#include
#include
@@ -133,9 +134,6 @@
bool dofree):
topic_(topic), rkmessage_(rkmessage), free_rkmessage_(dofree), key_(NULL) { }
- MessageImpl (RdKafka::Topic *topic, const rd_kafka_message_t *rkmessage):
- topic_(topic), rkmessage_(rkmessage), free_rkmessage_(false), key_(NULL) { }
-
MessageImpl (rd_kafka_message_t *rkmessage):
topic_(NULL), rkmessage_(rkmessage), free_rkmessage_(true), key_(NULL) {
if (rkmessage->rkt) {
@@ -204,8 +202,12 @@
return rd_kafka_message_latency(rkmessage_);
}
+ struct rd_kafka_message_s *c_ptr () {
+ return rkmessage_;
+ }
+
RdKafka::Topic *topic_;
- const rd_kafka_message_t *rkmessage_;
+ rd_kafka_message_t *rkmessage_;
bool free_rkmessage_;
/* For error signalling by the C++ layer the .._err_ message is
* used as a place holder and rkmessage_ is set to point to it. */
@@ -601,6 +603,15 @@
return clusterid;
}
+ struct rd_kafka_s *c_ptr () {
+ return rk_;
+ }
+
+ int32_t controllerid (int timeout_ms) {
+ return rd_kafka_controllerid(rk_, timeout_ms);
+ }
+
+
rd_kafka_t *rk_;
/* All Producer and Consumer callbacks must reside in HandleImpl and
* the opaque provided to rdkafka must be a pointer to HandleImpl, since
@@ -640,6 +651,10 @@
static Topic *create (Handle &base, const std::string &topic,
Conf *conf);
+ struct rd_kafka_topic_s *c_ptr () {
+ return rkt_;
+ }
+
rd_kafka_topic_t *rkt_;
PartitionerCb *partitioner_cb_;
PartitionerKeyPointerCb *partitioner_kp_cb_;
@@ -896,3 +911,5 @@
}
+
+#endif /* _RDKAFKACPP_INT_H_ */
diff -Nru librdkafka-0.11.3/src-cpp/TopicImpl.cpp librdkafka-0.11.6/src-cpp/TopicImpl.cpp
--- librdkafka-0.11.3/src-cpp/TopicImpl.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/src-cpp/TopicImpl.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -81,13 +81,18 @@
RdKafka::ConfImpl *confimpl = static_cast(conf);
rd_kafka_topic_t *rkt;
rd_kafka_topic_conf_t *rkt_conf;
+ rd_kafka_t *rk = dynamic_cast(base)->rk_;
RdKafka::TopicImpl *topic = new RdKafka::TopicImpl();
- if (!confimpl)
- rkt_conf = rd_kafka_topic_conf_new();
- else /* Make a copy of conf struct to allow Conf reuse. */
+ if (!confimpl) {
+ /* Reuse default topic config, but we need our own copy to
+ * set the topic opaque. */
+ rkt_conf = rd_kafka_default_topic_conf_dup(rk);
+ } else {
+ /* Make a copy of conf struct to allow Conf reuse. */
rkt_conf = rd_kafka_topic_conf_dup(confimpl->rkt_conf_);
+ }
/* Set topic opaque to the topic so that we can reach our topic object
* from whatever callbacks get registered.
@@ -108,8 +113,7 @@
}
- if (!(rkt = rd_kafka_topic_new(dynamic_cast(base)->rk_,
- topic_str.c_str(), rkt_conf))) {
+ if (!(rkt = rd_kafka_topic_new(rk, topic_str.c_str(), rkt_conf))) {
errstr = rd_kafka_err2str(rd_kafka_last_error());
delete topic;
rd_kafka_topic_conf_destroy(rkt_conf);
diff -Nru librdkafka-0.11.3/STATISTICS.md librdkafka-0.11.6/STATISTICS.md
--- librdkafka-0.11.3/STATISTICS.md 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/STATISTICS.md 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,576 @@
+# Statistics
+
+librdkafka may be configured to emit internal metrics at a fixed interval
+by setting the `statistics.interval.ms` configuration property to a value > 0
+and registering a `stats_cb` (or similar, depending on language).
+
+The stats are provided as a JSON object string.
+
+**Note**: The metrics returned may not be completely consistent between
+ brokers, toppars and totals, due to the internal asynchronous
+ nature of librdkafka.
+ E.g., the top level `tx` total may be less than the sum of
+ the broker `tx` values which it represents.
+
+
+## General structure
+
+All fields that contain sizes are are in bytes unless otherwise noted.
+
+```
+{
+
+ "brokers": {
+ ,
+ "toppars": { }
+ },
+ "topics": {
+ ,
+ "partitions": {
+
+ }
+ }
+[, "cgrp": { } ]
+}
+```
+
+## Field type
+
+Fields are represented as follows:
+ * string - UTF8 string.
+ * int - Integer counter (64 bits wide). Ever increasing.
+ * int gauge - Integer gauge (64 bits wide). Will be reset to 0 on each stats emit.
+ * object - Nested JSON object.
+ * bool - `true` or `false`.
+
+
+## Top-level
+
+Field | Type | Example | Description
+----- | ---- | ------- | -----------
+name | string | `"rdkafka#producer-1"` | Handle instance name
+client_id | string | `"rdkafka"` | The configured (or default) `client.id`
+type | string | `"producer"` | Instance type (producer or consumer)
+ts | int | 12345678912345 | librdkafka's internal monotonic clock (micro seconds)
+time | int | | Wall clock time in seconds since the epoch
+replyq | int gauge | | Number of ops (callbacks, events, etc) waiting in queue for application to serve with rd_kafka_poll()
+msg_cnt | int gauge | | Current number of messages in producer queues
+msg_size | int gauge | | Current total size of messages in producer queues
+msg_max | int | | Threshold: maximum number of messages allowed allowed on the producer queues
+msg_size_max | int | | Threshold: maximum total size of messages allowed on the producer queues
+tx | int | | Total number of requests sent to Kafka brokers
+txbytes | int | | Total number of bytes transmitted to Kafka brokers
+rx | int | | Total number of responses received from Kafka brokers
+rxbytes | int | | Total number of bytes received from Kafka brokers
+txmsgs | int | | Total number of messages transmitted (produced) to Kafka brokers
+txmsg_bytes | int | | Total number of message bytes (including framing, such as per-Message framing and MessageSet/batch framing) transmitted to Kafka brokers
+rxmsgs | int | | Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.
+rxmsg_bytes | int | | Total number of message bytes (including framing) received from Kafka brokers
+simple_cnt | int gauge | | Internal tracking of legacy vs new consumer API state
+metadata_cache_cnt | int gauge | | Number of topics in the metadata cache.
+brokers | object | | Dict of brokers, key is broker name, value is object. See **brokers** below
+topics | object | | Dict of topics, key is topic name, value is object. See **topics** below
+cgrp | object | | Consumer group metrics. See **cgrp** below
+
+## brokers
+
+Per broker statistics.
+
+Field | Type | Example | Description
+----- | ---- | ------- | -----------
+name | string | `"example.com:9092/13"` | Broker hostname, port and broker id
+nodeid | int | 13 | Broker id (-1 for bootstraps)
+state | string | `"UP"` | Broker state (INIT, DOWN, CONNECT, AUTH, APIVERSION_QUERY, AUTH_HANDSHAKE, UP, UPDATE)
+stateage | int gauge | | Time since last broker state change (microseconds)
+outbuf_cnt | int gauge | | Number of requests awaiting transmission to broker
+outbuf_msg_cnt | int gauge | | Number of messages awaiting transmission to broker
+waitresp_cnt | int gauge | | Number of requests in-flight to broker awaiting response
+waitresp_msg_cnt | int gauge | | Number of messages in-flight to broker awaitign response
+tx | int | | Total number of requests sent
+txbytes | int | | Total number of bytes sent
+txerrs | int | | Total number of transmission errors
+txretries | int | | Total number of request retries
+req_timeouts | int | | Total number of requests timed out
+rx | int | | Total number of responses received
+rxbytes | int | | Total number of bytes received
+rxerrs | int | | Total number of receive errors
+rxcorriderrs | int | | Total number of unmatched correlation ids in response (typically for timed out requests)
+rxpartial | int | | Total number of partial MessageSets received. The broker may return partial responses if the full MessageSet could not fit in remaining Fetch response size.
+zbuf_grow | int | | Total number of decompression buffer size increases
+buf_grow | int | | Total number of buffer size increases (deprecated, unused)
+wakeups | int | | Broker thread poll wakeups
+int_latency | object | | Internal producer queue latency in microseconds. See *Window stats* below
+outbuf_latency | object | | Internal request queue latency in microseconds. This is the time between a request is enqueued on the transmit (outbuf) queue and the time the request is written to the TCP socket. Additional buffering and latency may be incurred by the TCP stack and network. See *Window stats* below
+rtt | object | | Broker latency / round-trip time in microseconds. See *Window stats* below
+throttle | object | | Broker throttling time in milliseconds. See *Window stats* below
+toppars | object | | Partitions handled by this broker handle. Key is "topic-partition". See *brokers.toppars* below
+
+
+## Window stats
+
+Rolling window statistics. The values are in microseconds unless otherwise stated.
+
+Field | Type | Example | Description
+----- | ---- | ------- | -----------
+min | int gauge | | Smallest value
+max | int gauge | | Largest value
+avg | int gauge | | Average value
+sum | int gauge | | Sum of values
+cnt | int gauge | | Number of values sampled
+stddev | int gauge | | Standard deviation (based on histogram)
+hdrsize | int gauge | | Memory size of Hdr Histogram
+p50 | int gauge | | 50th percentile
+p75 | int gauge | | 75th percentile
+p90 | int gauge | | 90th percentile
+p95 | int gauge | | 95th percentile
+p99 | int gauge | | 99th percentile
+p99_99 | int gauge | | 99.99th percentile
+outofrange | int gauge | | Values skipped due to out of histogram range
+
+
+## brokers.toppars
+
+Topic partition assigned to broker.
+
+Field | Type | Example | Description
+----- | ---- | ------- | -----------
+topic | string | `"mytopic"` | Topic name
+partition | int | 3 | Partition id
+
+## topics
+
+Field | Type | Example | Description
+----- | ---- | ------- | -----------
+topic | string | `"myatopic"` | Topic name
+metadata_age | int gauge | | Age of metadata from broker for this topic (milliseconds)
+batchsize | object | | Batch sizes in bytes. See *Window stats*·
+batchcnt | object | | Batch message counts. See *Window stats*·
+partitions | object | | Partitions dict, key is partition id. See **partitions** below.
+
+
+## partitions
+
+Field | Type | Example | Description
+----- | ---- | ------- | -----------
+partition | int | 3 | Partition Id (-1 for internal UA/UnAssigned partition)
+leader | int | | Current leader broker id
+desired | bool | | Partition is explicitly desired by application
+unknown | bool | | Partition not seen in topic metadata from broker
+msgq_cnt | int gauge | | Number of messages waiting to be produced in first-level queue
+msgq_bytes | int gauge | | Number of bytes in msgq_cnt
+xmit_msgq_cnt | int gauge | | Number of messages ready to be produced in transmit queue
+xmit_msgq_bytes | int gauge | | Number of bytes in xmit_msgq
+fetchq_cnt | int gauge | | Number of pre-fetched messages in fetch queue
+fetchq_size | int gauge | | Bytes in fetchq
+fetch_state | string | `"active"` | Consumer fetch state for this partition (none, stopping, stopped, offset-query, offset-wait, active).
+query_offset | int gauge | | Current/Last logical offset query
+next_offset | int gauge | | Next offset to fetch
+app_offset | int gauge | | Offset of last message passed to application + 1
+stored_offset | int gauge | | Offset to be committed
+committed_offset | int gauge | | Last committed offset
+eof_offset | int gauge | | Last PARTITION_EOF signaled offset
+lo_offset | int gauge | | Partition's low watermark offset on broker
+hi_offset | int gauge | | Partition's high watermark offset on broker
+consumer_lag | int gauge | | Difference between hi_offset - max(app_offset, committed_offset)
+txmsgs | int | | Total number of messages transmitted (produced)
+txbytes | int | | Total number of bytes transmitted for txmsgs
+rxmsgs | int | | Total number of messages consumed, not including ignored messages (due to offset, etc).
+rxbytes | int | | Total number of bytes received for rxmsgs
+msgs | int | | Total number of messages received (consumer, same as rxmsgs), or total number of messages produced (possibly not yet transmitted) (producer).
+rx_ver_drops | int | | Dropped outdated messages
+
+
+## cgrp
+
+Field | Type | Example | Description
+----- | ---- | ------- | -----------
+rebalance_age | int gauge | | Time elapsed since last rebalance (assign or revoke) (milliseconds)
+rebalance_cnt | int | | Total number of rebalances (assign or revoke)
+assignment_size | int gauge | | Current assignment's partition count
+
+
+# Example output
+
+This (prettified) example output is from a short-lived producer using the following command:
+`rdkafka_performance -b localhost -P -t test -T 1000 -Y 'cat >> stats.json'`.
+
+Note: this output is prettified using `jq .`, the JSON object emitted by librdkafka does not contain line breaks.
+
+```{
+ "name": "rdkafka#producer-1",
+ "client_id": "rdkafka",
+ "type": "producer",
+ "ts": 5016483227792,
+ "time": 1527060869,
+ "replyq": 0,
+ "msg_cnt": 22710,
+ "msg_size": 704010,
+ "msg_max": 500000,
+ "msg_size_max": 1073741824,
+ "simple_cnt": 0,
+ "metadata_cache_cnt": 1,
+ "brokers": {
+ "localhost:9092/2": {
+ "name": "localhost:9092/2",
+ "nodeid": 2,
+ "state": "UP",
+ "stateage": 9057234,
+ "outbuf_cnt": 0,
+ "outbuf_msg_cnt": 0,
+ "waitresp_cnt": 0,
+ "waitresp_msg_cnt": 0,
+ "tx": 320,
+ "txbytes": 84283332,
+ "txerrs": 0,
+ "txretries": 0,
+ "req_timeouts": 0,
+ "rx": 320,
+ "rxbytes": 15708,
+ "rxerrs": 0,
+ "rxcorriderrs": 0,
+ "rxpartial": 0,
+ "zbuf_grow": 0,
+ "buf_grow": 0,
+ "wakeups": 591067,
+ "int_latency": {
+ "min": 86,
+ "max": 59375,
+ "avg": 23726,
+ "sum": 5694616664,
+ "stddev": 13982,
+ "p50": 28031,
+ "p75": 36095,
+ "p90": 39679,
+ "p95": 43263,
+ "p99": 48639,
+ "p99_99": 59391,
+ "outofrange": 0,
+ "hdrsize": 11376,
+ "cnt": 240012
+ },
+ "rtt": {
+ "min": 1580,
+ "max": 3389,
+ "avg": 2349,
+ "sum": 79868,
+ "stddev": 474,
+ "p50": 2319,
+ "p75": 2543,
+ "p90": 3183,
+ "p95": 3199,
+ "p99": 3391,
+ "p99_99": 3391,
+ "outofrange": 0,
+ "hdrsize": 13424,
+ "cnt": 34
+ },
+ "throttle": {
+ "min": 0,
+ "max": 0,
+ "avg": 0,
+ "sum": 0,
+ "stddev": 0,
+ "p50": 0,
+ "p75": 0,
+ "p90": 0,
+ "p95": 0,
+ "p99": 0,
+ "p99_99": 0,
+ "outofrange": 0,
+ "hdrsize": 17520,
+ "cnt": 34
+ },
+ "toppars": {
+ "test-1": {
+ "topic": "test",
+ "partition": 1
+ }
+ }
+ },
+ "localhost:9093/3": {
+ "name": "localhost:9093/3",
+ "nodeid": 3,
+ "state": "UP",
+ "stateage": 9057209,
+ "outbuf_cnt": 0,
+ "outbuf_msg_cnt": 0,
+ "waitresp_cnt": 0,
+ "waitresp_msg_cnt": 0,
+ "tx": 310,
+ "txbytes": 84301122,
+ "txerrs": 0,
+ "txretries": 0,
+ "req_timeouts": 0,
+ "rx": 310,
+ "rxbytes": 15104,
+ "rxerrs": 0,
+ "rxcorriderrs": 0,
+ "rxpartial": 0,
+ "zbuf_grow": 0,
+ "buf_grow": 0,
+ "wakeups": 607956,
+ "int_latency": {
+ "min": 82,
+ "max": 58069,
+ "avg": 23404,
+ "sum": 5617432101,
+ "stddev": 14021,
+ "p50": 27391,
+ "p75": 35839,
+ "p90": 39679,
+ "p95": 42751,
+ "p99": 48639,
+ "p99_99": 58111,
+ "outofrange": 0,
+ "hdrsize": 11376,
+ "cnt": 240016
+ },
+ "rtt": {
+ "min": 1704,
+ "max": 3572,
+ "avg": 2493,
+ "sum": 87289,
+ "stddev": 559,
+ "p50": 2447,
+ "p75": 2895,
+ "p90": 3375,
+ "p95": 3407,
+ "p99": 3583,
+ "p99_99": 3583,
+ "outofrange": 0,
+ "hdrsize": 13424,
+ "cnt": 35
+ },
+ "throttle": {
+ "min": 0,
+ "max": 0,
+ "avg": 0,
+ "sum": 0,
+ "stddev": 0,
+ "p50": 0,
+ "p75": 0,
+ "p90": 0,
+ "p95": 0,
+ "p99": 0,
+ "p99_99": 0,
+ "outofrange": 0,
+ "hdrsize": 17520,
+ "cnt": 35
+ },
+ "toppars": {
+ "test-0": {
+ "topic": "test",
+ "partition": 0
+ }
+ }
+ },
+ "localhost:9094/4": {
+ "name": "localhost:9094/4",
+ "nodeid": 4,
+ "state": "UP",
+ "stateage": 9057207,
+ "outbuf_cnt": 0,
+ "outbuf_msg_cnt": 0,
+ "waitresp_cnt": 0,
+ "waitresp_msg_cnt": 0,
+ "tx": 1,
+ "txbytes": 25,
+ "txerrs": 0,
+ "txretries": 0,
+ "req_timeouts": 0,
+ "rx": 1,
+ "rxbytes": 272,
+ "rxerrs": 0,
+ "rxcorriderrs": 0,
+ "rxpartial": 0,
+ "zbuf_grow": 0,
+ "buf_grow": 0,
+ "wakeups": 4,
+ "int_latency": {
+ "min": 0,
+ "max": 0,
+ "avg": 0,
+ "sum": 0,
+ "stddev": 0,
+ "p50": 0,
+ "p75": 0,
+ "p90": 0,
+ "p95": 0,
+ "p99": 0,
+ "p99_99": 0,
+ "outofrange": 0,
+ "hdrsize": 11376,
+ "cnt": 0
+ },
+ "rtt": {
+ "min": 0,
+ "max": 0,
+ "avg": 0,
+ "sum": 0,
+ "stddev": 0,
+ "p50": 0,
+ "p75": 0,
+ "p90": 0,
+ "p95": 0,
+ "p99": 0,
+ "p99_99": 0,
+ "outofrange": 0,
+ "hdrsize": 13424,
+ "cnt": 0
+ },
+ "throttle": {
+ "min": 0,
+ "max": 0,
+ "avg": 0,
+ "sum": 0,
+ "stddev": 0,
+ "p50": 0,
+ "p75": 0,
+ "p90": 0,
+ "p95": 0,
+ "p99": 0,
+ "p99_99": 0,
+ "outofrange": 0,
+ "hdrsize": 17520,
+ "cnt": 0
+ },
+ "toppars": {}
+ }
+ },
+ "topics": {
+ "test": {
+ "topic": "test",
+ "metadata_age": 9060,
+ "batchsize": {
+ "min": 99,
+ "max": 391805,
+ "avg": 272593,
+ "sum": 18808985,
+ "stddev": 180408,
+ "p50": 393215,
+ "p75": 393215,
+ "p90": 393215,
+ "p95": 393215,
+ "p99": 393215,
+ "p99_99": 393215,
+ "outofrange": 0,
+ "hdrsize": 14448,
+ "cnt": 69
+ },
+ "batchcnt": {
+ "min": 1,
+ "max": 10000,
+ "avg": 6956,
+ "sum": 480028,
+ "stddev": 4608,
+ "p50": 10047,
+ "p75": 10047,
+ "p90": 10047,
+ "p95": 10047,
+ "p99": 10047,
+ "p99_99": 10047,
+ "outofrange": 0,
+ "hdrsize": 8304,
+ "cnt": 69
+ },
+ "partitions": {
+ "0": {
+ "partition": 0,
+ "leader": 3,
+ "desired": false,
+ "unknown": false,
+ "msgq_cnt": 1,
+ "msgq_bytes": 31,
+ "xmit_msgq_cnt": 0,
+ "xmit_msgq_bytes": 0,
+ "fetchq_cnt": 0,
+ "fetchq_size": 0,
+ "fetch_state": "none",
+ "query_offset": 0,
+ "next_offset": 0,
+ "app_offset": -1001,
+ "stored_offset": -1001,
+ "commited_offset": -1001,
+ "committed_offset": -1001,
+ "eof_offset": -1001,
+ "lo_offset": -1001,
+ "hi_offset": -1001,
+ "consumer_lag": -1,
+ "txmsgs": 2150617,
+ "txbytes": 66669127,
+ "rxmsgs": 0,
+ "rxbytes": 0,
+ "msgs": 2160510,
+ "rx_ver_drops": 0
+ },
+ "1": {
+ "partition": 1,
+ "leader": 2,
+ "desired": false,
+ "unknown": false,
+ "msgq_cnt": 0,
+ "msgq_bytes": 0,
+ "xmit_msgq_cnt": 0,
+ "xmit_msgq_bytes": 0,
+ "fetchq_cnt": 0,
+ "fetchq_size": 0,
+ "fetch_state": "none",
+ "query_offset": 0,
+ "next_offset": 0,
+ "app_offset": -1001,
+ "stored_offset": -1001,
+ "commited_offset": -1001,
+ "committed_offset": -1001,
+ "eof_offset": -1001,
+ "lo_offset": -1001,
+ "hi_offset": -1001,
+ "consumer_lag": -1,
+ "txmsgs": 2150136,
+ "txbytes": 66654216,
+ "rxmsgs": 0,
+ "rxbytes": 0,
+ "msgs": 2159735,
+ "rx_ver_drops": 0
+ },
+ "-1": {
+ "partition": -1,
+ "leader": -1,
+ "desired": false,
+ "unknown": false,
+ "msgq_cnt": 0,
+ "msgq_bytes": 0,
+ "xmit_msgq_cnt": 0,
+ "xmit_msgq_bytes": 0,
+ "fetchq_cnt": 0,
+ "fetchq_size": 0,
+ "fetch_state": "none",
+ "query_offset": 0,
+ "next_offset": 0,
+ "app_offset": -1001,
+ "stored_offset": -1001,
+ "commited_offset": -1001,
+ "committed_offset": -1001,
+ "eof_offset": -1001,
+ "lo_offset": -1001,
+ "hi_offset": -1001,
+ "consumer_lag": -1,
+ "txmsgs": 0,
+ "txbytes": 0,
+ "rxmsgs": 0,
+ "rxbytes": 0,
+ "msgs": 1177,
+ "rx_ver_drops": 0
+ }
+ }
+ }
+ },
+ "tx": 631,
+ "tx_bytes": 168584479,
+ "rx": 631,
+ "rx_bytes": 31084,
+ "txmsgs": 4300753,
+ "txmsg_bytes": 133323343,
+ "rxmsgs": 0,
+ "rxmsg_bytes": 0
+}
+```
diff -Nru librdkafka-0.11.3/tests/0004-conf.c librdkafka-0.11.6/tests/0004-conf.c
--- librdkafka-0.11.3/tests/0004-conf.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0004-conf.c 2018-10-10 06:54:38.000000000 +0000
@@ -176,21 +176,6 @@
conf = rd_kafka_conf_new();
- res = rd_kafka_conf_set(conf, "ssl.keystore.location", "abc",
- errstr, sizeof(errstr));
- /* Existing apps might not print the error string when conf_set
- * returns UNKNOWN, only on INVALID, so make sure that is
- * what is being returned. */
- TEST_ASSERT(res == RD_KAFKA_CONF_INVALID,
- "expected ssl.keystore.location to fail with INVALID, "
- "not %d", res);
- /* Make sure there is a link to documentation */
- TEST_ASSERT(strstr(errstr, "http"),
- "expected ssl.keystore.location to provide link to "
- "documentation, not \"%s\"", errstr);
- TEST_SAY(_C_GRN "Ok: %s\n" _C_CLR, errstr);
-
-
res = rd_kafka_conf_set(conf, "ssl.truststore.location", "abc",
errstr, sizeof(errstr));
/* Existing apps might not print the error string when conf_set
@@ -396,6 +381,30 @@
rd_kafka_conf_destroy(conf);
}
+ {
+ rd_kafka_conf_res_t res;
+
+ TEST_SAY("Error reporting for S2F properties\n");
+ conf = rd_kafka_conf_new();
+
+ res = rd_kafka_conf_set(conf, "debug",
+ "cgrp,invalid-value,topic", errstr, sizeof(errstr));
+
+ TEST_ASSERT(res == RD_KAFKA_CONF_INVALID,
+ "expected 'debug=invalid-value' to fail with INVALID, "
+ "not %d", res);
+ TEST_ASSERT(strstr(errstr, "invalid-value"),
+ "expected invalid value to be mentioned in error, "
+ "not \"%s\"", errstr);
+ TEST_ASSERT(
+ !strstr(errstr, "cgrp") && !strstr(errstr, "topic"),
+ "expected only invalid value to be mentioned, "
+ "not \"%s\"", errstr);
+ TEST_SAY(_C_GRN "Ok: %s\n" _C_CLR, errstr);
+
+ rd_kafka_conf_destroy(conf);
+ }
+
/* Canonical int values, aliases, s2i-verified strings */
{
static const struct {
diff -Nru librdkafka-0.11.3/tests/0005-order.c librdkafka-0.11.6/tests/0005-order.c
--- librdkafka-0.11.3/tests/0005-order.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0005-order.c 2018-10-10 06:54:38.000000000 +0000
@@ -74,7 +74,7 @@
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char msg[128];
- int msgcnt = 50000;
+ int msgcnt = test_on_ci ? 5000 : 50000;
int i;
test_timing_t t_produce, t_delivery;
diff -Nru librdkafka-0.11.3/tests/0011-produce_batch.c librdkafka-0.11.6/tests/0011-produce_batch.c
--- librdkafka-0.11.3/tests/0011-produce_batch.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0011-produce_batch.c 2018-10-10 06:54:38.000000000 +0000
@@ -3,24 +3,24 @@
*
* Copyright (c) 2012-2013, Magnus Edenhill
* All rights reserved.
- *
+ *
* Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
+ * modification, are permitted provided that the following conditions are met:
+ *
* 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
+ * this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
+ * and/or other materials provided with the distribution.
+ *
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
@@ -41,76 +41,81 @@
static int msgid_next = 0;
static int fails = 0;
static int msgcounter = 0;
+static int *dr_partition_count = NULL;
+static const int topic_num_partitions = 4;
+static int msg_partition_wo_flag = 2;
+static int msg_partition_wo_flag_success = 0;
/**
* Delivery reported callback.
* Called for each message once to signal its delivery status.
*/
static void dr_single_partition_cb (rd_kafka_t *rk, void *payload, size_t len,
- rd_kafka_resp_err_t err, void *opaque, void *msg_opaque) {
- int msgid = *(int *)msg_opaque;
+ rd_kafka_resp_err_t err, void *opaque, void *msg_opaque) {
+ int msgid = *(int *)msg_opaque;
- free(msg_opaque);
+ free(msg_opaque);
- if (err != RD_KAFKA_RESP_ERR_NO_ERROR)
- TEST_FAIL("Message delivery failed: %s\n",
- rd_kafka_err2str(err));
-
- if (msgid != msgid_next) {
- fails++;
- TEST_FAIL("Delivered msg %i, expected %i\n",
- msgid, msgid_next);
- return;
- }
+ if (err != RD_KAFKA_RESP_ERR_NO_ERROR)
+ TEST_FAIL("Message delivery failed: %s\n",
+ rd_kafka_err2str(err));
+
+ if (msgid != msgid_next) {
+ fails++;
+ TEST_FAIL("Delivered msg %i, expected %i\n",
+ msgid, msgid_next);
+ return;
+ }
- msgid_next = msgid+1;
+ msgid_next = msgid+1;
msgcounter--;
}
/* Produce a batch of messages to a single partition. */
static void test_single_partition (void) {
- int partition = 0;
- int r;
- rd_kafka_t *rk;
- rd_kafka_topic_t *rkt;
- rd_kafka_conf_t *conf;
- rd_kafka_topic_conf_t *topic_conf;
- char msg[128];
- int msgcnt = 100000;
- int failcnt = 0;
- int i;
+ int partition = 0;
+ int r;
+ rd_kafka_t *rk;
+ rd_kafka_topic_t *rkt;
+ rd_kafka_conf_t *conf;
+ rd_kafka_topic_conf_t *topic_conf;
+ char msg[128];
+ int msgcnt = test_on_ci ? 1000 : 100000;
+ int failcnt = 0;
+ int i;
rd_kafka_message_t *rkmessages;
msgid_next = 0;
- test_conf_init(&conf, &topic_conf, 20);
+ test_conf_init(&conf, &topic_conf, 20);
- /* Set delivery report callback */
- rd_kafka_conf_set_dr_cb(conf, dr_single_partition_cb);
+ /* Set delivery report callback */
+ rd_kafka_conf_set_dr_cb(conf, dr_single_partition_cb);
- /* Create kafka instance */
- rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+ /* Create kafka instance */
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
- TEST_SAY("test_single_partition: Created kafka instance %s\n",
- rd_kafka_name(rk));
+ TEST_SAY("test_single_partition: Created kafka instance %s\n",
+ rd_kafka_name(rk));
- rkt = rd_kafka_topic_new(rk, test_mk_topic_name("0011", 0),
+ rkt = rd_kafka_topic_new(rk, test_mk_topic_name("0011", 0),
topic_conf);
- if (!rkt)
- TEST_FAIL("Failed to create topic: %s\n",
- rd_strerror(errno));
+ if (!rkt)
+ TEST_FAIL("Failed to create topic: %s\n",
+ rd_strerror(errno));
/* Create messages */
rkmessages = calloc(sizeof(*rkmessages), msgcnt);
- for (i = 0 ; i < msgcnt ; i++) {
- int *msgidp = malloc(sizeof(*msgidp));
- *msgidp = i;
- rd_snprintf(msg, sizeof(msg), "%s:%s test message #%i",
+ for (i = 0 ; i < msgcnt ; i++) {
+ int *msgidp = malloc(sizeof(*msgidp));
+ *msgidp = i;
+ rd_snprintf(msg, sizeof(msg), "%s:%s test message #%i",
__FILE__, __FUNCTION__, i);
rkmessages[i].payload = rd_strdup(msg);
rkmessages[i].len = strlen(msg);
rkmessages[i]._private = msgidp;
+ rkmessages[i].partition = 2;
}
r = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_FREE,
@@ -139,29 +144,29 @@
}
free(rkmessages);
- TEST_SAY("Single partition: "
+ TEST_SAY("Single partition: "
"Produced %i messages, waiting for deliveries\n", r);
msgcounter = msgcnt;
- /* Wait for messages to be delivered */
+ /* Wait for messages to be delivered */
test_wait_delivery(rk, &msgcounter);
- if (fails)
- TEST_FAIL("%i failures, see previous errors", fails);
+ if (fails)
+ TEST_FAIL("%i failures, see previous errors", fails);
- if (msgid_next != msgcnt)
- TEST_FAIL("Still waiting for messages: next %i != end %i\n",
- msgid_next, msgcnt);
+ if (msgid_next != msgcnt)
+ TEST_FAIL("Still waiting for messages: next %i != end %i\n",
+ msgid_next, msgcnt);
- /* Destroy topic */
- rd_kafka_topic_destroy(rkt);
+ /* Destroy topic */
+ rd_kafka_topic_destroy(rkt);
- /* Destroy rdkafka instance */
- TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
- rd_kafka_destroy(rk);
+ /* Destroy rdkafka instance */
+ TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
+ rd_kafka_destroy(rk);
- return;
+ return;
}
@@ -171,14 +176,14 @@
* Called for each message once to signal its delivery status.
*/
static void dr_partitioner_cb (rd_kafka_t *rk, void *payload, size_t len,
- rd_kafka_resp_err_t err, void *opaque, void *msg_opaque) {
- int msgid = *(int *)msg_opaque;
+ rd_kafka_resp_err_t err, void *opaque, void *msg_opaque) {
+ int msgid = *(int *)msg_opaque;
- free(msg_opaque);
+ free(msg_opaque);
- if (err != RD_KAFKA_RESP_ERR_NO_ERROR)
- TEST_FAIL("Message delivery failed: %s\n",
- rd_kafka_err2str(err));
+ if (err != RD_KAFKA_RESP_ERR_NO_ERROR)
+ TEST_FAIL("Message delivery failed: %s\n",
+ rd_kafka_err2str(err));
if (msgcounter <= 0)
TEST_FAIL("Too many message dr_cb callback calls "
@@ -188,41 +193,41 @@
/* Produce a batch of messages using random (default) partitioner */
static void test_partitioner (void) {
- int partition = RD_KAFKA_PARTITION_UA;
- int r;
- rd_kafka_t *rk;
- rd_kafka_topic_t *rkt;
- rd_kafka_conf_t *conf;
- rd_kafka_topic_conf_t *topic_conf;
- char msg[128];
- int msgcnt = 100000;
+ int partition = RD_KAFKA_PARTITION_UA;
+ int r;
+ rd_kafka_t *rk;
+ rd_kafka_topic_t *rkt;
+ rd_kafka_conf_t *conf;
+ rd_kafka_topic_conf_t *topic_conf;
+ char msg[128];
+ int msgcnt = test_on_ci ? 1000 : 100000;
int failcnt = 0;
- int i;
+ int i;
rd_kafka_message_t *rkmessages;
- test_conf_init(&conf, &topic_conf, 30);
+ test_conf_init(&conf, &topic_conf, 30);
- /* Set delivery report callback */
- rd_kafka_conf_set_dr_cb(conf, dr_partitioner_cb);
+ /* Set delivery report callback */
+ rd_kafka_conf_set_dr_cb(conf, dr_partitioner_cb);
- /* Create kafka instance */
- rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+ /* Create kafka instance */
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
- TEST_SAY("test_partitioner: Created kafka instance %s\n",
- rd_kafka_name(rk));
+ TEST_SAY("test_partitioner: Created kafka instance %s\n",
+ rd_kafka_name(rk));
- rkt = rd_kafka_topic_new(rk, test_mk_topic_name("0011", 0),
+ rkt = rd_kafka_topic_new(rk, test_mk_topic_name("0011", 0),
topic_conf);
- if (!rkt)
- TEST_FAIL("Failed to create topic: %s\n",
- rd_strerror(errno));
+ if (!rkt)
+ TEST_FAIL("Failed to create topic: %s\n",
+ rd_strerror(errno));
/* Create messages */
rkmessages = calloc(sizeof(*rkmessages), msgcnt);
- for (i = 0 ; i < msgcnt ; i++) {
- int *msgidp = malloc(sizeof(*msgidp));
- *msgidp = i;
- rd_snprintf(msg, sizeof(msg), "%s:%s test message #%i",
+ for (i = 0 ; i < msgcnt ; i++) {
+ int *msgidp = malloc(sizeof(*msgidp));
+ *msgidp = i;
+ rd_snprintf(msg, sizeof(msg), "%s:%s test message #%i",
__FILE__, __FUNCTION__, i);
rkmessages[i].payload = rd_strdup(msg);
@@ -256,32 +261,291 @@
}
free(rkmessages);
- TEST_SAY("Partitioner: "
+ TEST_SAY("Partitioner: "
"Produced %i messages, waiting for deliveries\n", r);
msgcounter = msgcnt;
- /* Wait for messages to be delivered */
+ /* Wait for messages to be delivered */
test_wait_delivery(rk, &msgcounter);
- if (fails)
- TEST_FAIL("%i failures, see previous errors", fails);
+ if (fails)
+ TEST_FAIL("%i failures, see previous errors", fails);
if (msgcounter != 0)
- TEST_FAIL("Still waiting for %i/%i messages\n",
+ TEST_FAIL("Still waiting for %i/%i messages\n",
msgcounter, msgcnt);
- /* Destroy topic */
- rd_kafka_topic_destroy(rkt);
+ /* Destroy topic */
+ rd_kafka_topic_destroy(rkt);
+
+ /* Destroy rdkafka instance */
+ TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
+ rd_kafka_destroy(rk);
+
+ return;
+}
+
+static void
+dr_per_message_partition_cb (rd_kafka_t *rk,
+ const rd_kafka_message_t *rkmessage,
+ void *opaque) {
+
+ free(rkmessage->_private);
+
+ if (rkmessage->err != RD_KAFKA_RESP_ERR_NO_ERROR)
+ TEST_FAIL("Message delivery failed: %s\n",
+ rd_kafka_err2str(rkmessage->err));
- /* Destroy rdkafka instance */
- TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
- rd_kafka_destroy(rk);
+ if (msgcounter <= 0)
+ TEST_FAIL("Too many message dr_cb callback calls "
+ "(at msg offset #%"PRId64")\n", rkmessage->offset);
+
+ TEST_ASSERT(rkmessage->partition < topic_num_partitions);
+ msgcounter--;
- return;
+ dr_partition_count[rkmessage->partition]++;
}
+/* Produce a batch of messages using with per message partition flag */
+static void test_per_message_partition_flag (void) {
+ int partition = 0;
+ int r;
+ rd_kafka_t *rk;
+ rd_kafka_topic_t *rkt;
+ rd_kafka_conf_t *conf;
+ rd_kafka_topic_conf_t *topic_conf;
+ char msg[128];
+ int msgcnt = 1000;
+ int failcnt = 0;
+ int i;
+ int *rkpartition_counts;
+ rd_kafka_message_t *rkmessages;
+ const char *topic_name;
+
+ test_conf_init(&conf, &topic_conf, 30);
+
+ /* Set delivery report callback */
+ rd_kafka_conf_set_dr_msg_cb(conf, dr_per_message_partition_cb);
+
+ /* Create kafka instance */
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+
+ TEST_SAY("test_per_message_partition_flag: Created kafka instance %s\n",
+ rd_kafka_name(rk));
+ topic_name = test_mk_topic_name("0011_per_message_flag", 1);
+ test_create_topic(topic_name, topic_num_partitions, 1);
+
+ rkt = rd_kafka_topic_new(rk, topic_name,
+ topic_conf);
+ if (!rkt)
+ TEST_FAIL("Failed to create topic: %s\n",
+ rd_strerror(errno));
+
+ /* Create messages */
+ rkpartition_counts = calloc(sizeof(int), topic_num_partitions);
+ dr_partition_count = calloc(sizeof(int), topic_num_partitions);
+ rkmessages = calloc(sizeof(*rkmessages), msgcnt);
+ for (i = 0 ; i < msgcnt ; i++) {
+ int *msgidp = malloc(sizeof(*msgidp));
+ *msgidp = i;
+ rd_snprintf(msg, sizeof(msg), "%s:%s test message #%i",
+ __FILE__, __FUNCTION__, i);
+
+ rkmessages[i].payload = rd_strdup(msg);
+ rkmessages[i].len = strlen(msg);
+ rkmessages[i]._private = msgidp;
+ rkmessages[i].partition = jitter(0, topic_num_partitions - 1);
+ rkpartition_counts[rkmessages[i].partition]++;
+ }
+
+ r = rd_kafka_produce_batch(rkt, partition,
+ RD_KAFKA_MSG_F_PARTITION|RD_KAFKA_MSG_F_FREE,
+ rkmessages, msgcnt);
+
+ /* Scan through messages to check for errors. */
+ for (i = 0 ; i < msgcnt ; i++) {
+ if (rkmessages[i].err) {
+ failcnt++;
+ if (failcnt < 100)
+ TEST_SAY("Message #%i failed: %s\n",
+ i,
+ rd_kafka_err2str(rkmessages[i].err));
+ }
+ }
+
+ /* All messages should've been produced. */
+ if (r < msgcnt) {
+ TEST_SAY("Not all messages were accepted "
+ "by produce_batch(): %i < %i\n", r, msgcnt);
+ if (msgcnt - r != failcnt)
+ TEST_SAY("Discrepency between failed messages (%i) "
+ "and return value %i (%i - %i)\n",
+ failcnt, msgcnt - r, msgcnt, r);
+ TEST_FAIL("%i/%i messages failed\n", msgcnt - r, msgcnt);
+ }
+
+ free(rkmessages);
+ TEST_SAY("Per-message partition: "
+ "Produced %i messages, waiting for deliveries\n", r);
+
+ msgcounter = msgcnt;
+ /* Wait for messages to be delivered */
+ test_wait_delivery(rk, &msgcounter);
+
+ if (msgcounter != 0)
+ TEST_FAIL("Still waiting for %i/%i messages\n",
+ msgcounter, msgcnt);
+
+ for (i = 0; i < topic_num_partitions; i++) {
+ if (dr_partition_count[i] != rkpartition_counts[i]) {
+ TEST_FAIL("messages were not sent to designated "
+ "partitions expected messages %i in "
+ "partition %i, but only "
+ "%i messages were sent",
+ rkpartition_counts[i],
+ i, dr_partition_count[i]);
+ }
+ }
+
+ free(rkpartition_counts);
+ free(dr_partition_count);
+
+ /* Destroy topic */
+ rd_kafka_topic_destroy(rkt);
+
+ /* Destroy rdkafka instance */
+ TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
+ rd_kafka_destroy(rk);
+
+ return;
+}
+
+static void
+dr_partitioner_wo_per_message_flag_cb (rd_kafka_t *rk,
+ const rd_kafka_message_t *rkmessage,
+ void *opaque) {
+ free(rkmessage->_private);
+
+ if (rkmessage->err != RD_KAFKA_RESP_ERR_NO_ERROR)
+ TEST_FAIL("Message delivery failed: %s\n",
+ rd_kafka_err2str(rkmessage->err));
+ if (msgcounter <= 0)
+ TEST_FAIL("Too many message dr_cb callback calls "
+ "(at msg offset #%"PRId64")\n", rkmessage->offset);
+ if (rkmessage->partition != msg_partition_wo_flag)
+ msg_partition_wo_flag_success = 1;
+ msgcounter--;
+}
+
+/**
+ * @brief Produce a batch of messages using partitioner
+ * without per message partition flag
+ */
+static void test_message_partitioner_wo_per_message_flag (void) {
+ int partition = RD_KAFKA_PARTITION_UA;
+ int r;
+ rd_kafka_t *rk;
+ rd_kafka_topic_t *rkt;
+ rd_kafka_conf_t *conf;
+ rd_kafka_topic_conf_t *topic_conf;
+ char msg[128];
+ int msgcnt = 1000;
+ int failcnt = 0;
+ int i;
+ rd_kafka_message_t *rkmessages;
+
+ test_conf_init(&conf, &topic_conf, 30);
+
+ /* Set delivery report callback */
+ rd_kafka_conf_set_dr_msg_cb(conf,
+ dr_partitioner_wo_per_message_flag_cb);
+
+ /* Create kafka instance */
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+
+ TEST_SAY("test_partitioner: Created kafka instance %s\n",
+ rd_kafka_name(rk));
+
+ rkt = rd_kafka_topic_new(rk, test_mk_topic_name("0011", 0),
+ topic_conf);
+ if (!rkt)
+ TEST_FAIL("Failed to create topic: %s\n",
+ rd_strerror(errno));
+
+ /* Create messages */
+ rkmessages = calloc(sizeof(*rkmessages), msgcnt);
+ for (i = 0 ; i < msgcnt ; i++) {
+ int *msgidp = malloc(sizeof(*msgidp));
+ *msgidp = i;
+ rd_snprintf(msg, sizeof(msg), "%s:%s test message #%i",
+ __FILE__, __FUNCTION__, i);
+
+ rkmessages[i].payload = rd_strdup(msg);
+ rkmessages[i].len = strlen(msg);
+ rkmessages[i]._private = msgidp;
+ rkmessages[i].partition = msg_partition_wo_flag;
+ }
+
+ r = rd_kafka_produce_batch(rkt, partition, RD_KAFKA_MSG_F_FREE,
+ rkmessages, msgcnt);
+
+ /* Scan through messages to check for errors. */
+ for (i = 0 ; i < msgcnt ; i++) {
+ if (rkmessages[i].err) {
+ failcnt++;
+ if (failcnt < 100)
+ TEST_SAY("Message #%i failed: %s\n",
+ i,
+ rd_kafka_err2str(rkmessages[i].err));
+ }
+ }
+
+ /* All messages should've been produced. */
+ if (r < msgcnt) {
+ TEST_SAY("Not all messages were accepted "
+ "by produce_batch(): %i < %i\n", r, msgcnt);
+ if (msgcnt - r != failcnt)
+ TEST_SAY("Discrepency between failed messages (%i) "
+ "and return value %i (%i - %i)\n",
+ failcnt, msgcnt - r, msgcnt, r);
+ TEST_FAIL("%i/%i messages failed\n", msgcnt - r, msgcnt);
+ }
+
+ free(rkmessages);
+ TEST_SAY("Partitioner: "
+ "Produced %i messages, waiting for deliveries\n", r);
+
+ msgcounter = msgcnt;
+ /* Wait for messages to be delivered */
+ test_wait_delivery(rk, &msgcounter);
+
+ if (fails)
+ TEST_FAIL("%i failures, see previous errors", fails);
+
+ if (msgcounter != 0)
+ TEST_FAIL("Still waiting for %i/%i messages\n",
+ msgcounter, msgcnt);
+ if (msg_partition_wo_flag_success == 0) {
+ TEST_FAIL("partitioner was not used, all messages were sent to"
+ "message specified partition %i", i);
+ }
+
+ /* Destroy topic */
+ rd_kafka_topic_destroy(rkt);
+
+ /* Destroy rdkafka instance */
+ TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
+ rd_kafka_destroy(rk);
+
+ return;
+}
+
+
int main_0011_produce_batch (int argc, char **argv) {
+ test_message_partitioner_wo_per_message_flag();
test_single_partition();
test_partitioner();
- return 0;
+ if (test_can_create_topics(1))
+ test_per_message_partition_flag();
+ return 0;
}
diff -Nru librdkafka-0.11.3/tests/0022-consume_batch.c librdkafka-0.11.6/tests/0022-consume_batch.c
--- librdkafka-0.11.3/tests/0022-consume_batch.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0022-consume_batch.c 2018-10-10 06:54:38.000000000 +0000
@@ -47,7 +47,7 @@
rd_kafka_queue_t *rkq;
rd_kafka_topic_t *rkts[topic_cnt];
rd_kafka_resp_err_t err;
- const int msgcnt = 10000;
+ const int msgcnt = test_on_ci ? 5000 : 10000;
uint64_t testid;
int i, p;
int batch_cnt = 0;
diff -Nru librdkafka-0.11.3/tests/0025-timers.c librdkafka-0.11.6/tests/0025-timers.c
--- librdkafka-0.11.3/tests/0025-timers.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0025-timers.c 2018-10-10 06:54:38.000000000 +0000
@@ -123,9 +123,17 @@
TEST_SAY("Got more calls than expected: %d > %d\n",
state.calls, exp_calls);
- if (state.fails)
- TEST_FAIL("%d/%d intervals failed\n", state.fails, state.calls);
- else
+ if (state.fails) {
+ /* We can't rely on CIs giving our test job enough CPU to finish
+ * in time, so don't error out even if the time is outside
+ * the window */
+ if (test_on_ci)
+ TEST_WARN("%d/%d intervals failed\n",
+ state.fails, state.calls);
+ else
+ TEST_FAIL("%d/%d intervals failed\n",
+ state.fails, state.calls);
+ } else
TEST_SAY("All %d intervals okay\n", state.calls);
}
diff -Nru librdkafka-0.11.3/tests/0026-consume_pause.c librdkafka-0.11.6/tests/0026-consume_pause.c
--- librdkafka-0.11.3/tests/0026-consume_pause.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0026-consume_pause.c 2018-10-10 06:54:38.000000000 +0000
@@ -42,6 +42,7 @@
static int consume_pause (void) {
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
+ const int partition_cnt = 3;
rd_kafka_t *rk;
rd_kafka_topic_conf_t *tconf;
rd_kafka_topic_partition_list_t *topics;
@@ -51,10 +52,13 @@
int it, iterations = 3;
int msg_base = 0;
int fails = 0;
+ char group_id[32];
test_conf_init(NULL, &tconf, 60 + (test_session_timeout_ms * 3 / 1000));
test_topic_conf_set(tconf, "auto.offset.reset", "smallest");
-
+
+ test_create_topic(topic, partition_cnt, 1);
+
/* Produce messages */
testid = test_produce_msgs_easy(topic, 0,
RD_KAFKA_PARTITION_UA, msgcnt);
@@ -63,10 +67,10 @@
rd_kafka_topic_partition_list_add(topics, topic, -1);
for (it = 0 ; it < iterations ; it++) {
- char group_id[32];
const int pause_cnt = 5;
- const int per_pause_msg_cnt = msgcnt / pause_cnt;
- const int pause_time = 1200 /* 1.2s */;
+ int per_pause_msg_cnt = msgcnt / pause_cnt;
+ const int pause_time = 1200 /* 1.2s */;
+ int eof_cnt = -1;
int pause;
rd_kafka_topic_partition_list_t *parts;
test_msgver_t mv_all;
@@ -74,10 +78,21 @@
test_msgver_init(&mv_all, testid); /* All messages */
- test_str_id_generate(group_id, sizeof(group_id));
-
- TEST_SAY("Iteration %d/%d, using group.id %s\n", it, iterations,
- group_id);
+ /* On the last iteration reuse the previous group.id
+ * to make consumer start at committed offsets which should
+ * also be EOF. This to trigger #1307. */
+ if (it < iterations-1)
+ test_str_id_generate(group_id, sizeof(group_id));
+ else {
+ TEST_SAY("Reusing previous group.id %s\n", group_id);
+ per_pause_msg_cnt = 0;
+ eof_cnt = partition_cnt;
+ }
+
+ TEST_SAY("Iteration %d/%d, using group.id %s, "
+ "expecting %d messages/pause and %d EOFs\n",
+ it, iterations-1, group_id,
+ per_pause_msg_cnt, eof_cnt);
rk = test_create_consumer(group_id, NULL, NULL,
rd_kafka_topic_conf_dup(tconf));
@@ -103,8 +118,10 @@
"msg_base %d\n", pause, per_pause_msg_cnt,
msg_base);
rcnt = test_consumer_poll("consume.part", rk, testid,
- -1,
- msg_base, per_pause_msg_cnt,
+ eof_cnt,
+ msg_base,
+ per_pause_msg_cnt == 0 ?
+ -1 : per_pause_msg_cnt,
&mv);
TEST_ASSERT(rcnt == per_pause_msg_cnt,
@@ -175,9 +192,13 @@
rd_kafka_topic_partition_list_destroy(parts);
}
- test_msgver_verify("all.msgs", &mv_all, TEST_MSGVER_ALL_PART,
- 0, msgcnt);
- test_msgver_clear(&mv_all);
+ if (per_pause_msg_cnt > 0)
+ test_msgver_verify("all.msgs", &mv_all,
+ TEST_MSGVER_ALL_PART, 0, msgcnt);
+ else
+ test_msgver_verify("all.msgs", &mv_all,
+ TEST_MSGVER_ALL_PART, 0, 0);
+ test_msgver_clear(&mv_all);
/* Should now not see any more messages. */
test_consumer_poll_no_msgs("end.exp.no.msgs", rk, testid, 3000);
@@ -196,11 +217,145 @@
+/**
+ * @brief Verify that the paused partition state is not used after
+ * the partition has been re-assigned.
+ *
+ * 1. Produce N messages
+ * 2. Consume N/4 messages
+ * 3. Pause partitions
+ * 4. Manually commit offset N/2
+ * 5. Unassign partitions
+ * 6. Assign partitions again
+ * 7. Verify that consumption starts at N/2 and not N/4
+ */
+static int consume_pause_resume_after_reassign (void) {
+ const char *topic = test_mk_topic_name(__FUNCTION__, 1);
+ const int32_t partition = 0;
+ const int msgcnt = 4000;
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ rd_kafka_topic_partition_list_t *partitions, *pos;
+ rd_kafka_resp_err_t err;
+ int exp_msg_cnt;
+ uint64_t testid;
+ int r;
+ int msg_base = 0;
+ test_msgver_t mv;
+ rd_kafka_topic_partition_t *toppar;
+
+ test_conf_init(&conf, NULL, 60);
+
+ test_create_topic(topic, (int)partition+1, 1);
+
+ /* Produce messages */
+ testid = test_produce_msgs_easy(topic, 0, partition, msgcnt);
+
+ /* Set start offset to beginning */
+ partitions = rd_kafka_topic_partition_list_new(1);
+ toppar = rd_kafka_topic_partition_list_add(partitions, topic,
+ partition);
+ toppar->offset = RD_KAFKA_OFFSET_BEGINNING;
+
+
+ /**
+ * Create consumer.
+ */
+ rk = test_create_consumer(topic, NULL, conf, NULL);
+
+ test_consumer_assign("assign", rk, partitions);
+
+
+ exp_msg_cnt = msgcnt/4;
+ TEST_SAY("Consuming first quarter (%d) of messages\n", exp_msg_cnt);
+ test_msgver_init(&mv, testid);
+ r = test_consumer_poll("consume.first.quarter", rk, testid, 0,
+ msg_base, exp_msg_cnt, &mv);
+ TEST_ASSERT(r == exp_msg_cnt,
+ "expected %d messages, got %d", exp_msg_cnt, r);
+
+
+ TEST_SAY("Pausing partitions\n");
+ if ((err = rd_kafka_pause_partitions(rk, partitions)))
+ TEST_FAIL("Failed to pause: %s", rd_kafka_err2str(err));
+
+ TEST_SAY("Verifying pause, should see no new messages...\n");
+ test_consumer_poll_no_msgs("silence.while.paused", rk, testid, 3000);
+
+ test_msgver_verify("first.quarter", &mv, TEST_MSGVER_ALL_PART,
+ msg_base, exp_msg_cnt);
+ test_msgver_clear(&mv);
+
+
+ /* Check position */
+ pos = rd_kafka_topic_partition_list_copy(partitions);
+ if ((err = rd_kafka_position(rk, pos)))
+ TEST_FAIL("position() failed: %s", rd_kafka_err2str(err));
+
+ TEST_ASSERT(!pos->elems[0].err,
+ "position() returned error for our partition: %s",
+ rd_kafka_err2str(pos->elems[0].err));
+ TEST_SAY("Current application consume position is %"PRId64"\n",
+ pos->elems[0].offset);
+ TEST_ASSERT(pos->elems[0].offset == (int64_t)exp_msg_cnt,
+ "expected position %"PRId64", not %"PRId64,
+ (int64_t)exp_msg_cnt, pos->elems[0].offset);
+ rd_kafka_topic_partition_list_destroy(pos);
+
+
+ toppar->offset = (int64_t)(msgcnt/2);
+ TEST_SAY("Committing (yet unread) offset %"PRId64"\n", toppar->offset);
+ if ((err = rd_kafka_commit(rk, partitions, 0/*sync*/)))
+ TEST_FAIL("Commit failed: %s", rd_kafka_err2str(err));
+
+
+ TEST_SAY("Unassigning\n");
+ test_consumer_unassign("Unassign", rk);
+
+ /* Set start offset to INVALID so that the standard start offset
+ * logic kicks in. */
+ toppar->offset = RD_KAFKA_OFFSET_INVALID;
+
+ TEST_SAY("Reassigning\n");
+ test_consumer_assign("Reassign", rk, partitions);
+
+
+ TEST_SAY("Resuming partitions\n");
+ if ((err = rd_kafka_resume_partitions(rk, partitions)))
+ TEST_FAIL("Failed to resume: %s", rd_kafka_err2str(err));
+
+ msg_base = msgcnt / 2;
+ exp_msg_cnt = msgcnt / 2;
+ TEST_SAY("Consuming second half (%d) of messages at msg_base %d\n",
+ exp_msg_cnt, msg_base);
+ test_msgver_init(&mv, testid);
+ r = test_consumer_poll("consume.second.half", rk, testid, 1/*exp eof*/,
+ msg_base, exp_msg_cnt, &mv);
+ TEST_ASSERT(r == exp_msg_cnt,
+ "expected %d messages, got %d", exp_msg_cnt, r);
+
+ test_msgver_verify("second.half", &mv, TEST_MSGVER_ALL_PART,
+ msg_base, exp_msg_cnt);
+ test_msgver_clear(&mv);
+
+
+ rd_kafka_topic_partition_list_destroy(partitions);
+
+ test_consumer_close(rk);
+
+ rd_kafka_destroy(rk);
+
+ return 0;
+}
+
int main_0026_consume_pause (int argc, char **argv) {
int fails = 0;
- fails += consume_pause();
+ if (test_can_create_topics(1)) {
+ fails += consume_pause();
+ fails += consume_pause_resume_after_reassign();
+ }
if (fails > 0)
TEST_FAIL("See %d previous error(s)\n", fails);
diff -Nru librdkafka-0.11.3/tests/0030-offset_commit.c librdkafka-0.11.6/tests/0030-offset_commit.c
--- librdkafka-0.11.3/tests/0030-offset_commit.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0030-offset_commit.c 2018-10-10 06:54:38.000000000 +0000
@@ -81,7 +81,7 @@
"should be above committed_offset %"PRId64,
rktpar->offset, committed_offset);
else if (rktpar->offset == committed_offset)
- TEST_SAYL(1, "Current offset re-commited: %"PRId64"\n",
+ TEST_SAYL(1, "Current offset re-committed: %"PRId64"\n",
rktpar->offset);
else
committed_offset = rktpar->offset;
diff -Nru librdkafka-0.11.3/tests/0033-regex_subscribe.c librdkafka-0.11.6/tests/0033-regex_subscribe.c
--- librdkafka-0.11.3/tests/0033-regex_subscribe.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0033-regex_subscribe.c 2018-10-10 06:54:38.000000000 +0000
@@ -427,7 +427,7 @@
rd_kafka_topic_partition_list_add(invalids, "not_a_regex", 0);
rd_kafka_topic_partition_list_add(invalids, "^My[vV]alid..regex+", 0);
- rd_kafka_topic_partition_list_add(invalids, "^??++", 99);
+ rd_kafka_topic_partition_list_add(invalids, "^a[b", 99);
rd_kafka_topic_partition_list_add(empty, "not_a_regex", 0);
rd_kafka_topic_partition_list_add(empty, "", 0);
diff -Nru librdkafka-0.11.3/tests/0039-event.c librdkafka-0.11.6/tests/0039-event.c
--- librdkafka-0.11.3/tests/0039-event.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0039-event.c 2018-10-10 06:54:38.000000000 +0000
@@ -27,7 +27,7 @@
*/
/**
- * Tests messages are produced in order.
+ * Tests event API.
*/
@@ -74,7 +74,10 @@
}
-int main_0039_event (int argc, char **argv) {
+/**
+ * @brief Test delivery report events
+ */
+int main_0039_event_dr (int argc, char **argv) {
int partition = 0;
int r;
rd_kafka_t *rk;
@@ -82,7 +85,7 @@
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *topic_conf;
char msg[128];
- int msgcnt = 50000;
+ int msgcnt = test_on_ci ? 5000 : 50000;
int i;
test_timing_t t_produce, t_delivery;
rd_kafka_queue_t *eventq;
@@ -124,7 +127,7 @@
TIMING_START(&t_delivery, "DELIVERY");
while (rd_kafka_outq_len(rk) > 0) {
rd_kafka_event_t *rkev;
- rkev = rd_kafka_queue_poll(eventq, 100);
+ rkev = rd_kafka_queue_poll(eventq, 1000);
switch (rd_kafka_event_type(rkev))
{
case RD_KAFKA_EVENT_DR:
@@ -160,3 +163,57 @@
return 0;
}
+
+
+
+/**
+ * @brief Local test: test event generation
+ */
+int main_0039_event (int argc, char **argv) {
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ rd_kafka_queue_t *eventq;
+ int waitevent = 1;
+
+ /* Set up a config with ERROR events enabled and
+ * configure an invalid broker so that _TRANSPORT or ALL_BROKERS_DOWN
+ * is promptly generated. */
+
+ conf = rd_kafka_conf_new();
+
+ rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_ERROR);
+ rd_kafka_conf_set(conf, "bootstrap.servers", "0:65534", NULL, 0);
+
+ /* Create kafka instance */
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+
+ eventq = rd_kafka_queue_get_main(rk);
+
+ while (waitevent) {
+ rd_kafka_event_t *rkev;
+ rkev = rd_kafka_queue_poll(eventq, 1000);
+ switch (rd_kafka_event_type(rkev))
+ {
+ case RD_KAFKA_EVENT_ERROR:
+ TEST_SAY("Got %s event: %s: %s\n",
+ rd_kafka_event_name(rkev),
+ rd_kafka_err2name(rd_kafka_event_error(rkev)),
+ rd_kafka_event_error_string(rkev));
+ waitevent = 0;
+ break;
+ default:
+ TEST_SAY("Unhandled event: %s\n",
+ rd_kafka_event_name(rkev));
+ break;
+ }
+ rd_kafka_event_destroy(rkev);
+ }
+
+ rd_kafka_queue_destroy(eventq);
+
+ /* Destroy rdkafka instance */
+ TEST_SAY("Destroying kafka instance %s\n", rd_kafka_name(rk));
+ rd_kafka_destroy(rk);
+
+ return 0;
+}
diff -Nru librdkafka-0.11.3/tests/0040-io_event.c librdkafka-0.11.6/tests/0040-io_event.c
--- librdkafka-0.11.3/tests/0040-io_event.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0040-io_event.c 2018-10-10 06:54:38.000000000 +0000
@@ -78,7 +78,7 @@
rk_p = test_create_producer();
rkt_p = test_create_producer_topic(rk_p, topic, NULL);
- err = test_auto_create_topic_rkt(rk_p, rkt_p);
+ err = test_auto_create_topic_rkt(rk_p, rkt_p, tmout_multip(5000));
TEST_ASSERT(!err, "Topic auto creation failed: %s",
rd_kafka_err2str(err));
diff -Nru librdkafka-0.11.3/tests/0044-partition_cnt.c librdkafka-0.11.6/tests/0044-partition_cnt.c
--- librdkafka-0.11.3/tests/0044-partition_cnt.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0044-partition_cnt.c 2018-10-10 06:54:38.000000000 +0000
@@ -52,7 +52,7 @@
rd_kafka_topic_t *rkt;
const char *topic = test_mk_topic_name(__FUNCTION__, 1);
const int partition_cnt = 4;
- int msgcnt = 100000;
+ int msgcnt = test_on_ci ? 5000 : 100000;
test_timing_t t_destroy;
int produced = 0;
@@ -61,7 +61,7 @@
topic, partition_cnt/2);
test_conf_init(&conf, NULL, 20);
-
+ rd_kafka_conf_set_dr_cb(conf, test_dr_cb);
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
rkt = test_create_topic_object(rk, __FUNCTION__,
"message.timeout.ms",
diff -Nru librdkafka-0.11.3/tests/0047-partial_buf_tmout.c librdkafka-0.11.6/tests/0047-partial_buf_tmout.c
--- librdkafka-0.11.3/tests/0047-partial_buf_tmout.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0047-partial_buf_tmout.c 2018-10-10 06:54:38.000000000 +0000
@@ -53,9 +53,9 @@
static void my_error_cb (rd_kafka_t *rk, int err,
const char *reason, void *opaque) {
- got_timeout_err += (err == RD_KAFKA_RESP_ERR__MSG_TIMED_OUT);
+ got_timeout_err += (err == RD_KAFKA_RESP_ERR__TIMED_OUT);
- if (err == RD_KAFKA_RESP_ERR__MSG_TIMED_OUT ||
+ if (err == RD_KAFKA_RESP_ERR__TIMED_OUT ||
err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN)
TEST_SAY("Expected error: %s: %s\n",
rd_kafka_err2str(err), reason);
diff -Nru librdkafka-0.11.3/tests/0048-partitioner.c librdkafka-0.11.6/tests/0048-partitioner.c
--- librdkafka-0.11.3/tests/0048-partitioner.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0048-partitioner.c 2018-10-10 06:54:38.000000000 +0000
@@ -35,6 +35,7 @@
* Various partitioner tests
*
* - Issue #797 - deadlock on failed partitioning
+ * - Verify that partitioning works across partitioners.
*/
int32_t my_invalid_partitioner (const rd_kafka_topic_t *rkt,
@@ -87,7 +88,201 @@
rd_kafka_destroy(rk);
}
+
+static void part_dr_msg_cb (rd_kafka_t *rk,
+ const rd_kafka_message_t *rkmessage, void *opaque) {
+ int32_t *partp = rkmessage->_private;
+ int *remainsp = opaque;
+
+ if (rkmessage->err) {
+ /* Will fail later */
+ TEST_WARN("Delivery failed: %s\n",
+ rd_kafka_err2str(rkmessage->err));
+ *partp = -1;
+ } else {
+ *partp = rkmessage->partition;
+ }
+
+ (*remainsp)--;
+}
+
+/**
+ * @brief Test single \p partitioner
+ */
+static void do_test_partitioner (const char *topic, const char *partitioner,
+ int msgcnt, const char **keys,
+ const int32_t *exp_part) {
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ int i;
+ int32_t *parts;
+ int remains = msgcnt;
+ int randcnt = 0;
+ int fails = 0;
+
+ TEST_SAY(_C_MAG "Test partitioner \"%s\"\n", partitioner);
+
+ test_conf_init(&conf, NULL, 30);
+ rd_kafka_conf_set_opaque(conf, &remains);
+ rd_kafka_conf_set_dr_msg_cb(conf, part_dr_msg_cb);
+ test_conf_set(conf, "partitioner", partitioner);
+
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+
+ parts = malloc(msgcnt * sizeof(*parts));
+ for (i = 0 ; i < msgcnt ; i++)
+ parts[i] = -1;
+
+ /*
+ * Produce messages
+ */
+ for (i = 0 ; i < msgcnt ; i++) {
+ rd_kafka_resp_err_t err;
+
+ err = rd_kafka_producev(rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_KEY(keys[i],
+ keys[i] ?
+ strlen(keys[i]) : 0),
+ RD_KAFKA_V_OPAQUE(&parts[i]),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(!err,
+ "producev() failed: %s", rd_kafka_err2str(err));
+
+ randcnt += exp_part[i] == -1;
+ }
+
+ rd_kafka_flush(rk, tmout_multip(10000));
+
+ TEST_ASSERT(remains == 0,
+ "Expected remains=%d, not %d for %d messages",
+ 0, remains, msgcnt);
+
+ /*
+ * Verify produced partitions to expected partitions.
+ */
+
+ /* First look for produce failures */
+ for (i = 0 ; i < msgcnt ; i++) {
+ if (parts[i] == -1) {
+ TEST_WARN("Message #%d (exp part %"PRId32") "
+ "was not successfully produced\n",
+ i, exp_part[i]);
+ fails++;
+ }
+ }
+
+ TEST_ASSERT(!fails, "See %d previous failure(s)", fails);
+
+
+ if (randcnt == msgcnt) {
+ /* If all expected partitions are random make sure
+ * the produced partitions have some form of
+ * random distribution */
+ int32_t last_part = parts[0];
+ int samecnt = 0;
+
+ for (i = 0 ; i < msgcnt ; i++) {
+ samecnt += parts[i] == last_part;
+ last_part = parts[i];
+ }
+
+ TEST_ASSERT(samecnt < msgcnt,
+ "No random distribution, all on partition %"PRId32,
+ last_part);
+ } else {
+ for (i = 0 ; i < msgcnt ; i++) {
+ if (exp_part[i] != -1 &&
+ parts[i] != exp_part[i]) {
+ TEST_WARN("Message #%d expected partition "
+ "%"PRId32" but got %"PRId32": %s\n",
+ i, exp_part[i], parts[i],
+ keys[i]);
+ fails++;
+ }
+ }
+
+
+ TEST_ASSERT(!fails, "See %d previous failure(s)", fails);
+ }
+
+ free(parts);
+
+ rd_kafka_destroy(rk);
+
+ TEST_SAY(_C_GRN "Test partitioner \"%s\": PASS\n", partitioner);
+}
+
+extern uint32_t rd_crc32 (const char *, size_t);
+
+/**
+ * @brief Test all builtin partitioners
+ */
+static void do_test_partitioners (void) {
+#define _PART_CNT 17
+#define _MSG_CNT 5
+ const char *unaligned = "123456";
+ /* Message keys */
+ const char *keys[_MSG_CNT] = {
+ NULL,
+ "", // empty
+ unaligned+1,
+ "this is another string with more length to it perhaps",
+ "hejsan"
+ };
+ struct {
+ const char *partitioner;
+ /* Expected partition per message (see keys above) */
+ int32_t exp_part[_MSG_CNT];
+ } ptest[] = {
+ { "random", { -1, -1, -1, -1, -1 } },
+ { "consistent", {
+ /* These constants were acquired using
+ * the 'crc32' command on OSX */
+ 0x0 % _PART_CNT,
+ 0x0 % _PART_CNT,
+ 0xb1b451d7 % _PART_CNT,
+ 0xb0150df7 % _PART_CNT,
+ 0xd077037e % _PART_CNT
+ } },
+ { "consistent_random", {
+ -1,
+ -1,
+ 0xb1b451d7 % _PART_CNT,
+ 0xb0150df7 % _PART_CNT,
+ 0xd077037e % _PART_CNT
+ } },
+ { "murmur2", {
+ /* .. using tests/java/Murmur2Cli */
+ 0x106e08d9 % _PART_CNT,
+ 0x106e08d9 % _PART_CNT,
+ 0x058d780f % _PART_CNT,
+ 0x4f7703da % _PART_CNT,
+ 0x5ec19395 % _PART_CNT
+ } },
+ { "murmur2_random", {
+ -1,
+ 0x106e08d9 % _PART_CNT,
+ 0x058d780f % _PART_CNT,
+ 0x4f7703da % _PART_CNT,
+ 0x5ec19395 % _PART_CNT
+ } },
+ { NULL }
+ };
+ int pi;
+ const char *topic = test_mk_topic_name(__FUNCTION__, 1);
+
+ test_create_topic(topic, _PART_CNT, 1);
+
+ for (pi = 0 ; ptest[pi].partitioner ; pi++) {
+ do_test_partitioner(topic, ptest[pi].partitioner,
+ _MSG_CNT, keys, ptest[pi].exp_part);
+ }
+}
+
int main_0048_partitioner (int argc, char **argv) {
+ if (test_can_create_topics(0))
+ do_test_partitioners();
do_test_failed_partitioning();
return 0;
}
diff -Nru librdkafka-0.11.3/tests/0049-consume_conn_close.c librdkafka-0.11.6/tests/0049-consume_conn_close.c
--- librdkafka-0.11.3/tests/0049-consume_conn_close.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0049-consume_conn_close.c 2018-10-10 06:54:38.000000000 +0000
@@ -79,7 +79,7 @@
rd_kafka_t *rk;
const char *topic = test_mk_topic_name("0049_consume_conn_close", 1);
uint64_t testid;
- int msgcnt = 100000;
+ int msgcnt = test_on_ci ? 1000 : 10000;
test_msgver_t mv;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
diff -Nru librdkafka-0.11.3/tests/0050-subscribe_adds.c librdkafka-0.11.6/tests/0050-subscribe_adds.c
--- librdkafka-0.11.3/tests/0050-subscribe_adds.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0050-subscribe_adds.c 2018-10-10 06:54:38.000000000 +0000
@@ -52,7 +52,7 @@
rd_strdup(test_mk_topic_name("0050_subscribe_adds_3", 1)),
};
uint64_t testid;
- int msgcnt = 10000;
+ int msgcnt = test_on_ci ? 1000 : 10000;
test_msgver_t mv;
rd_kafka_conf_t *conf;
rd_kafka_topic_conf_t *tconf;
diff -Nru librdkafka-0.11.3/tests/0053-stats_cb.cpp librdkafka-0.11.6/tests/0053-stats_cb.cpp
--- librdkafka-0.11.3/tests/0053-stats_cb.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0053-stats_cb.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -1,7 +1,7 @@
/*
* librdkafka - Apache Kafka C library
*
- * Copyright (c) 2012-2015, Magnus Edenhill
+ * Copyright (c) 2012-2018, Magnus Edenhill
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -27,44 +27,167 @@
*/
#include
+#include
+#include
+#include
#include "testcpp.h"
+#if WITH_RAPIDJSON
+#include
+#include
+#include
+#include
+#include
+#include
+#endif
+
+static const char *stats_schema_path = "../src/statistics_schema.json";
+
+#if WITH_RAPIDJSON
+/**
+ * @brief Statistics schema validator
+ */
+class TestSchemaValidator {
+ public:
+ TestSchemaValidator () {
+
+ }
+ TestSchemaValidator (const std::string schema_path) {
+ /* Read schema from file */
+ schema_path_ = schema_path;
+
+ std::ifstream f(schema_path);
+ if (!f.is_open())
+ Test::Fail(tostr() << "Failed to open schema " << schema_path <<
+ ": " << strerror(errno));
+ std::string schema_str((std::istreambuf_iterator(f)),
+ (std::istreambuf_iterator()));
+
+ /* Parse schema */
+ sd_ = new rapidjson::Document();
+ if (sd_->Parse(schema_str.c_str()).HasParseError())
+ Test::Fail(tostr() << "Failed to parse statistics schema: " <<
+ rapidjson::GetParseError_En(sd_->GetParseError()) <<
+ " at " << sd_->GetErrorOffset());
+
+ schema_ = new rapidjson::SchemaDocument(*sd_);
+ validator_ = new rapidjson::SchemaValidator(*schema_);
+ }
+
+ ~TestSchemaValidator () {
+ if (sd_)
+ delete sd_;
+ if (schema_)
+ delete schema_;
+ if (validator_)
+ delete validator_;
+ }
+
+ void validate (const std::string &json_doc) {
+ /* Parse JSON to validate */
+ rapidjson::Document d;
+ if (d.Parse(json_doc.c_str()).HasParseError())
+ Test::Fail(tostr() << "Failed to parse stats JSON: " <<
+ rapidjson::GetParseError_En(d.GetParseError()) <<
+ " at " << d.GetErrorOffset());
+
+ /* Validate using schema */
+ if (!d.Accept(*validator_)) {
+
+ rapidjson::StringBuffer sb;
+
+ validator_->GetInvalidSchemaPointer().StringifyUriFragment(sb);
+ Test::Say(tostr() << "Schema: " << sb.GetString() << "\n");
+ Test::Say(tostr() << "Invalid keyword: " << validator_->GetInvalidSchemaKeyword() << "\n");
+ sb.Clear();
+
+ validator_->GetInvalidDocumentPointer().StringifyUriFragment(sb);
+ Test::Say(tostr() << "Invalid document: " << sb.GetString() << "\n");
+ sb.Clear();
+
+ Test::Fail(tostr() << "JSON validation using schema " << schema_path_ << " failed");
+ }
+
+ Test::Say(3, "JSON document validated using schema " + schema_path_ + "\n");
+ }
+
+ private:
+ std::string schema_path_;
+ rapidjson::Document *sd_;
+ rapidjson::SchemaDocument *schema_;
+ rapidjson::SchemaValidator *validator_;
+};
+
+
+#else
+
+/* Dummy validator doing nothing when RapidJSON is unavailable */
+class TestSchemaValidator {
+ public:
+ TestSchemaValidator () {
+
+ }
+ TestSchemaValidator (const std::string schema_path) {
+ }
+
+ ~TestSchemaValidator () {
+ }
+
+ void validate (const std::string &json_doc) {
+ }
+};
+
+#endif
class myEventCb : public RdKafka::EventCb {
public:
- myEventCb() {
- stats_cnt = 0;
- }
+ myEventCb(const std::string schema_path):
+ validator_(TestSchemaValidator(schema_path)) {
+ stats_cnt = 0;
+ }
+
int stats_cnt;
+ std::string last; /**< Last stats document */
+
void event_cb (RdKafka::Event &event) {
switch (event.type())
{
case RdKafka::Event::EVENT_STATS:
- Test::Say(tostr() << "Stats (#" << stats_cnt << "): " <<
- event.str() << "\n");
- if (event.str().length() > 20)
- stats_cnt += 1;
+ if (!(stats_cnt % 10))
+ Test::Say(tostr() << "Stats (#" << stats_cnt << "): " <<
+ event.str() << "\n");
+ if (event.str().length() > 20)
+ stats_cnt += 1;
+ validator_.validate(event.str());
+ last = event.str();
break;
default:
break;
}
}
+
+ private:
+ TestSchemaValidator validator_;
};
-void test_stats_cb () {
+
+/**
+ * @brief Verify that stats are emitted according to statistics.interval.ms
+ */
+void test_stats_timing () {
RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
- myEventCb my_event = myEventCb();
+ myEventCb my_event = myEventCb(stats_schema_path);
std::string errstr;
if (conf->set("statistics.interval.ms", "100", errstr) != RdKafka::Conf::CONF_OK)
- Test::Fail(errstr);
+ Test::Fail(errstr);
if (conf->set("event_cb", &my_event, errstr) != RdKafka::Conf::CONF_OK)
- Test::Fail(errstr);
+ Test::Fail(errstr);
RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
if (!p)
- Test::Fail("Failed to create Producer: " + errstr);
+ Test::Fail("Failed to create Producer: " + errstr);
delete conf;
int64_t t_start = test_clock();
@@ -76,18 +199,326 @@
const int expected_time = 1200;
Test::Say(tostr() << my_event.stats_cnt << " (expected 12) stats callbacks received in " <<
- elapsed << "ms (expected " << expected_time << "ms +-25%)\n");
+ elapsed << "ms (expected " << expected_time << "ms +-25%)\n");
if (elapsed < expected_time * 0.75 ||
- elapsed > expected_time * 1.25)
- Test::Fail(tostr() << "Elapsed time " << elapsed << "ms outside +-25% window (" <<
- expected_time << "ms), cnt " << my_event.stats_cnt);
+ elapsed > expected_time * 1.25) {
+ /* We can't rely on CIs giving our test job enough CPU to finish
+ * in time, so don't error out even if the time is outside the window */
+ if (test_on_ci)
+ Test::Say(tostr() << "WARNING: Elapsed time " << elapsed << "ms outside +-25% window (" <<
+ expected_time << "ms), cnt " << my_event.stats_cnt);
+ else
+ Test::Fail(tostr() << "Elapsed time " << elapsed << "ms outside +-25% window (" <<
+ expected_time << "ms), cnt " << my_event.stats_cnt);
+ }
+ delete p;
+}
+
+
+
+#if WITH_RAPIDJSON
+
+/**
+ * @brief Expected partition stats
+ */
+struct exp_part_stats {
+ std::string topic; /**< Topic */
+ int32_t part; /**< Partition id */
+ int msgcnt; /**< Expected message count */
+ int msgsize; /**< Expected per message size.
+ * This includes both key and value lengths */
+
+ /* Calculated */
+ int64_t totsize; /**< Message size sum */
+};
+
+/**
+ * @brief Verify end-to-end producer and consumer stats.
+ */
+static void verify_e2e_stats (const std::string &prod_stats,
+ const std::string &cons_stats,
+ struct exp_part_stats *exp_parts, int partcnt) {
+ /**
+ * Parse JSON stats
+ * These documents are already validated in the Event callback.
+ */
+ rapidjson::Document p;
+ if (p.Parse(prod_stats.c_str()).HasParseError())
+ Test::Fail(tostr() << "Failed to parse producer stats JSON: " <<
+ rapidjson::GetParseError_En(p.GetParseError()) <<
+ " at " << p.GetErrorOffset());
+
+ rapidjson::Document c;
+ if (c.Parse(cons_stats.c_str()).HasParseError())
+ Test::Fail(tostr() << "Failed to parse consumer stats JSON: " <<
+ rapidjson::GetParseError_En(c.GetParseError()) <<
+ " at " << c.GetErrorOffset());
+
+ assert(p.HasMember("name"));
+ assert(c.HasMember("name"));
+ assert(p.HasMember("type"));
+ assert(c.HasMember("type"));
+
+ Test::Say(tostr() << "Verifying stats from Producer " << p["name"].GetString() <<
+ " and Consumer " << c["name"].GetString() << "\n");
+
+ assert(!strcmp(p["type"].GetString(), "producer"));
+ assert(!strcmp(c["type"].GetString(), "consumer"));
+
+ int64_t exp_tot_txmsgs = 0;
+ int64_t exp_tot_txmsg_bytes = 0;
+ int64_t exp_tot_rxmsgs = 0;
+ int64_t exp_tot_rxmsg_bytes = 0;
+
+ for (int part = 0 ; part < partcnt ; part++) {
+
+ /*
+ * Find partition stats.
+ */
+
+ /* Construct the partition path. */
+ char path[256];
+ rd_snprintf(path, sizeof(path),
+ "/topics/%s/partitions/%d",
+ exp_parts[part].topic.c_str(), exp_parts[part].part);
+ Test::Say(tostr() << "Looking up partition " << exp_parts[part].part <<
+ " with path " << path << "\n");
+
+ /* Even though GetValueByPointer() takes a "char[]" it can only be used
+ * with perfectly sized char buffers or string literals since it
+ * does not respect NUL terminators.
+ * So instead convert the path to a Pointer.*/
+ rapidjson::Pointer jpath((const char *)path);
+
+ rapidjson::Value *pp = rapidjson::GetValueByPointer(p, jpath);
+ if (!pp)
+ Test::Fail(tostr() << "Producer: could not find " << path <<
+ " in " << prod_stats << "\n");
+
+ rapidjson::Value *cp = rapidjson::GetValueByPointer(c, jpath);
+ if (!pp)
+ Test::Fail(tostr() << "Consumer: could not find " << path <<
+ " in " << cons_stats << "\n");
+
+ assert(pp->HasMember("partition"));
+ assert(pp->HasMember("txmsgs"));
+ assert(pp->HasMember("txbytes"));
+
+ assert(cp->HasMember("partition"));
+ assert(cp->HasMember("rxmsgs"));
+ assert(cp->HasMember("rxbytes"));
+
+ Test::Say(tostr() << "partition: " << (*pp)["partition"].GetInt() << "\n");
+
+ int64_t txmsgs = (*pp)["txmsgs"].GetInt();
+ int64_t txbytes = (*pp)["txbytes"].GetInt();
+ int64_t rxmsgs = (*cp)["rxmsgs"].GetInt();
+ int64_t rxbytes = (*cp)["rxbytes"].GetInt();
+
+ exp_tot_txmsgs += txmsgs;
+ exp_tot_txmsg_bytes += txbytes;
+ exp_tot_rxmsgs += rxmsgs;
+ exp_tot_rxmsg_bytes += rxbytes;
+
+ Test::Say(tostr() << "Producer partition: " << (*pp)["partition"].GetInt() << ": " <<
+ "txmsgs: " << txmsgs << " vs " << exp_parts[part].msgcnt << ", " <<
+ "txbytes: " << txbytes << " vs " << exp_parts[part].totsize << "\n");
+ Test::Say(tostr() << "Consumer partition: " << (*cp)["partition"].GetInt() << ": " <<
+ "rxmsgs: " << rxmsgs << " vs " << exp_parts[part].msgcnt << ", " <<
+ "rxbytes: " << rxbytes << " vs " << exp_parts[part].totsize << "\n");
+ }
+
+ /* Check top-level total stats */
+
+ assert(p.HasMember("txmsgs"));
+ assert(p.HasMember("txmsg_bytes"));
+ assert(p.HasMember("rxmsgs"));
+ assert(p.HasMember("rxmsg_bytes"));
+
+ int64_t tot_txmsgs = p["txmsgs"].GetInt();
+ int64_t tot_txmsg_bytes = p["txmsg_bytes"].GetInt();
+ int64_t tot_rxmsgs = c["rxmsgs"].GetInt();
+ int64_t tot_rxmsg_bytes = c["rxmsg_bytes"].GetInt();
+
+ Test::Say(tostr() << "Producer total: " <<
+ "txmsgs: " << tot_txmsgs << " vs " << exp_tot_txmsgs << ", " <<
+ "txbytes: " << tot_txmsg_bytes << " vs " << exp_tot_txmsg_bytes << "\n");
+ Test::Say(tostr() << "Consumer total: " <<
+ "rxmsgs: " << tot_rxmsgs << " vs " << exp_tot_rxmsgs << ", " <<
+ "rxbytes: " << tot_rxmsg_bytes << " vs " << exp_tot_rxmsg_bytes << "\n");
+
+}
+
+/**
+ * @brief Verify stats JSON structure and individual metric fields.
+ *
+ * To capture as much verifiable data as possible we run a full
+ * producer - consumer end to end test and verify that counters
+ * and states are emitted accordingly.
+ *
+ * Requires RapidJSON (for parsing the stats).
+ */
+static void test_stats () {
+ std::string errstr;
+ RdKafka::Conf *conf;
+ myEventCb producer_event(stats_schema_path);
+ myEventCb consumer_event(stats_schema_path);
+
+ std::string topic = Test::mk_topic_name("0053_stats", 1);
+
+ const int partcnt = 2;
+ int msgcnt = 100 * partcnt;
+ const int msgsize = 6*1024;
+
+ /*
+ * Common config for producer and consumer
+ */
+ Test::conf_init(&conf, NULL, 60);
+ if (conf->set("statistics.interval.ms", "1000", errstr) != RdKafka::Conf::CONF_OK)
+ Test::Fail(errstr);
+
+
+ /*
+ * Create Producer
+ */
+ if (conf->set("event_cb", &producer_event, errstr) != RdKafka::Conf::CONF_OK)
+ Test::Fail(errstr);
+
+ RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
+ if (!p)
+ Test::Fail("Failed to create Producer: " + errstr);
+
+
+ /*
+ * Create Consumer
+ */
+ conf->set("group.id", topic, errstr);
+ conf->set("auto.offset.reset", "earliest", errstr);
+ conf->set("enable.partition.eof", "false", errstr);
+ if (conf->set("event_cb", &consumer_event, errstr) != RdKafka::Conf::CONF_OK)
+ Test::Fail(errstr);
+
+ RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
+ if (!c)
+ Test::Fail("Failed to create KafkaConsumer: " + errstr);
+ delete conf;
+
+ /*
+ * Set up consumer assignment (but assign after producing
+ * since there will be no topics now) and expected partitions
+ * for later verification.
+ */
+ std::vector toppars;
+ struct exp_part_stats exp_parts[partcnt] = {};
+
+ for (int32_t part = 0 ; part < (int32_t)partcnt ; part++) {
+ toppars.push_back(RdKafka::TopicPartition::create(topic, part,
+ RdKafka::Topic::OFFSET_BEGINNING));
+ exp_parts[part].topic = topic;
+ exp_parts[part].part = part;
+ exp_parts[part].msgcnt = msgcnt / partcnt;
+ exp_parts[part].msgsize = msgsize;
+ exp_parts[part].totsize = 0;
+ }
+
+ /*
+ * Produce messages
+ */
+ uint64_t testid = test_id_generate();
+
+ char key[256];
+ char *buf = (char *)malloc(msgsize);
+
+ for (int32_t part = 0 ; part < (int32_t)partcnt ; part++) {
+ for (int i = 0 ; i < msgcnt / partcnt ; i++) {
+ test_prepare_msg(testid, part, i, buf, msgsize, key, sizeof(key));
+ RdKafka::ErrorCode err = p->produce(topic, part,
+ RdKafka::Producer::RK_MSG_COPY,
+ buf, msgsize, key, sizeof(key),
+ -1, NULL);
+ if (err)
+ Test::Fail("Produce failed: " + RdKafka::err2str(err));
+ exp_parts[part].totsize += msgsize + sizeof(key);
+ p->poll(0);
+ }
+ }
+
+ free(buf);
+
+ Test::Say("Waiting for final message delivery\n");
+ /* Wait for delivery */
+ p->flush(15*1000);
+
+ /*
+ * Start consuming partitions
+ */
+ c->assign(toppars);
+ RdKafka::TopicPartition::destroy(toppars);
+
+ /*
+ * Consume the messages
+ */
+ int recvcnt = 0;
+ Test::Say(tostr() << "Consuming " << msgcnt << " messages\n");
+ while (recvcnt < msgcnt) {
+ RdKafka::Message *msg = c->consume(-1);
+ if (msg->err())
+ Test::Fail("Consume failed: " + msg->errstr());
+
+ int msgid;
+ TestMessageVerify(testid, -1, &msgid, msg);
+ recvcnt++;
+ delete msg;
+ }
+
+ /*
+ * Producer:
+ * Wait for one last stats emit when all messages have been delivered.
+ */
+ int prev_cnt = producer_event.stats_cnt;
+ while (prev_cnt == producer_event.stats_cnt) {
+ Test::Say("Waiting for final producer stats event\n");
+ p->poll(100);
+ }
+
+ /*
+ * Consumer:
+ * Wait for a one last stats emit when all messages have been received,
+ * since previous stats may have been enqueued but not served we
+ * skip the first 2.
+ */
+ prev_cnt = consumer_event.stats_cnt;
+ while (prev_cnt + 2 >= consumer_event.stats_cnt) {
+ Test::Say(tostr() << "Waiting for final consumer stats event: " <<
+ consumer_event.stats_cnt << "\n");
+ c->poll(100);
+ }
+
+
+ verify_e2e_stats(producer_event.last, consumer_event.last,
+ exp_parts, partcnt);
+
+
+ c->close();
+
delete p;
+ delete c;
}
+#endif
extern "C" {
- int main_0053_stats_cb (int argc, char **argv) {
- test_stats_cb();
+ int main_0053_stats_timing (int argc, char **argv) {
+ test_stats_timing();
+ return 0;
+ }
+
+ int main_0053_stats (int argc, char **argv) {
+#if WITH_RAPIDJSON
+ test_stats();
+#else
+ Test::Skip("RapidJSON >=1.1.0 not available\n");
+#endif
return 0;
}
}
diff -Nru librdkafka-0.11.3/tests/0059-bsearch.cpp librdkafka-0.11.6/tests/0059-bsearch.cpp
--- librdkafka-0.11.3/tests/0059-bsearch.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0059-bsearch.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -117,6 +117,7 @@
topic = Test::mk_topic_name("0059-bsearch", 1);
Test::conf_init(&conf, &tconf, 0);
Test::conf_set(tconf, "produce.offset.report", "true");
+ Test::conf_set(tconf, "acks", "all");
Test::conf_set(conf, "api.version.request", "true");
conf->set("dr_cb", &my_dr, errstr);
conf->set("default_topic_conf", tconf, errstr);
diff -Nru librdkafka-0.11.3/tests/0061-consumer_lag.cpp librdkafka-0.11.6/tests/0061-consumer_lag.cpp
--- librdkafka-0.11.3/tests/0061-consumer_lag.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0061-consumer_lag.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -163,7 +163,7 @@
break;
case RdKafka::ERR_NO_ERROR:
- /* Proper message. Updated calculated lag for later
+ /* Proper message. Update calculated lag for later
* checking in stats callback */
stats.calc_lag = msgcnt - (msg->offset()+1);
cnt++;
diff -Nru librdkafka-0.11.3/tests/0062-stats_event.c librdkafka-0.11.6/tests/0062-stats_event.c
--- librdkafka-0.11.3/tests/0062-stats_event.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0062-stats_event.c 2018-10-10 06:54:38.000000000 +0000
@@ -98,14 +98,21 @@
}
TIMING_STOP(&t_delivery);
- if (!strcmp(test_mode, "bare")) {
- /* valgrind is too slow to make this meaningful. */
- if (TIMING_DURATION(&t_delivery) < 1000 * 100 * 0.5 ||
- TIMING_DURATION(&t_delivery) > 1000 * 100 * 1.5)
+ if (TIMING_DURATION(&t_delivery) < 1000 * 100 * 0.5 ||
+ TIMING_DURATION(&t_delivery) > 1000 * 100 * 1.5) {
+ /* CIs and valgrind are too flaky/slow to
+ * make this failure meaningful. */
+ if (!test_on_ci && !strcmp(test_mode, "bare")) {
TEST_FAIL("Stats duration %.3fms is >= 50%% "
"outside statistics.interval.ms 100",
(float)TIMING_DURATION(&t_delivery)/
1000.0f);
+ } else {
+ TEST_WARN("Stats duration %.3fms is >= 50%% "
+ "outside statistics.interval.ms 100\n",
+ (float)TIMING_DURATION(&t_delivery)/
+ 1000.0f);
+ }
}
}
diff -Nru librdkafka-0.11.3/tests/0063-clusterid.cpp librdkafka-0.11.6/tests/0063-clusterid.cpp
--- librdkafka-0.11.3/tests/0063-clusterid.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0063-clusterid.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -32,10 +32,13 @@
#include "testcpp.h"
/**
- * Test Handle::clusterid()
+ * Test Handle::clusterid() and Handle::controllerid()
*/
static void do_test_clusterid (void) {
+
+ Test::Say("[ do_test_clusterid ]\n");
+
/*
* Create client with appropriate protocol support for
* retrieving clusterid
@@ -98,9 +101,80 @@
delete p_bad;
}
+
+/**
+ * @brief controllerid() testing.
+ * This instantiates its own client to avoid having the value cached
+ * from do_test_clusterid(), but they are basically the same tests.
+ */
+static void do_test_controllerid (void) {
+
+ Test::Say("[ do_test_controllerid ]\n");
+
+ /*
+ * Create client with appropriate protocol support for
+ * retrieving controllerid
+ */
+ RdKafka::Conf *conf;
+ Test::conf_init(&conf, NULL, 10);
+ Test::conf_set(conf, "api.version.request", "true");
+ std::string errstr;
+ RdKafka::Producer *p_good = RdKafka::Producer::create(conf, errstr);
+ if (!p_good)
+ Test::Fail("Failed to create client: " + errstr);
+ delete conf;
+
+ /*
+ * Create client with lacking protocol support.
+ */
+ Test::conf_init(&conf, NULL, 10);
+ Test::conf_set(conf, "api.version.request", "false");
+ Test::conf_set(conf, "broker.version.fallback", "0.9.0");
+ RdKafka::Producer *p_bad = RdKafka::Producer::create(conf, errstr);
+ if (!p_bad)
+ Test::Fail("Failed to create client: " + errstr);
+ delete conf;
+
+ /*
+ * good producer, give the first call a timeout to allow time
+ * for background metadata requests to finish.
+ */
+ int32_t controllerid_good_1 = p_good->controllerid(tmout_multip(2000));
+ if (controllerid_good_1 == -1)
+ Test::Fail("good producer(w timeout): Controllerid is -1");
+ Test::Say(tostr() << "good producer(w timeout): Controllerid " << controllerid_good_1 << "\n");
+
+ /* Then retrieve a cached copy. */
+ int32_t controllerid_good_2 = p_good->controllerid(0);
+ if (controllerid_good_2 == -1)
+ Test::Fail("good producer(0): Controllerid is -1");
+ Test::Say(tostr() << "good producer(0): Controllerid " << controllerid_good_2 << "\n");
+
+ if (controllerid_good_1 != controllerid_good_2)
+ Test::Fail(tostr() << "Good Controllerid mismatch: " <<
+ controllerid_good_1 << " != " << controllerid_good_2);
+
+ /*
+ * Try bad producer, should return -1
+ */
+ int32_t controllerid_bad_1 = p_bad->controllerid(tmout_multip(2000));
+ if (controllerid_bad_1 != -1)
+ Test::Fail(tostr() <<
+ "bad producer(w timeout): Controllerid should be -1, not " <<
+ controllerid_bad_1);
+ int32_t controllerid_bad_2 = p_bad->controllerid(0);
+ if (controllerid_bad_2 != -1)
+ Test::Fail(tostr() << "bad producer(0): Controllerid should be -1, not " <<
+ controllerid_bad_2);
+
+ delete p_good;
+ delete p_bad;
+}
+
extern "C" {
int main_0063_clusterid (int argc, char **argv) {
do_test_clusterid();
+ do_test_controllerid();
return 0;
}
}
diff -Nru librdkafka-0.11.3/tests/0065-yield.cpp librdkafka-0.11.6/tests/0065-yield.cpp
--- librdkafka-0.11.3/tests/0065-yield.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0065-yield.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -77,6 +77,9 @@
Test::conf_init(&conf, NULL, 10);
DrCb0065 dr(do_yield);
conf->set("dr_cb", &dr, errstr);
+ /* Make sure messages are produced in batches of 100 */
+ conf->set("batch.num.messages", "100", errstr);
+ conf->set("linger.ms", "10000", errstr);
RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
if (!p)
diff -Nru librdkafka-0.11.3/tests/0068-produce_timeout.c librdkafka-0.11.6/tests/0068-produce_timeout.c
--- librdkafka-0.11.3/tests/0068-produce_timeout.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0068-produce_timeout.c 2018-10-10 06:54:38.000000000 +0000
@@ -58,7 +58,7 @@
if (err == RD_KAFKA_RESP_ERR__TRANSPORT ||
err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN ||
err == RD_KAFKA_RESP_ERR__AUTHENTICATION ||
- err == RD_KAFKA_RESP_ERR__MSG_TIMED_OUT)
+ err == RD_KAFKA_RESP_ERR__TIMED_OUT)
return 0;
return 1;
}
@@ -103,7 +103,7 @@
"message.timeout.ms", "100", NULL);
TEST_SAY("Auto-creating topic %s\n", topic);
- test_auto_create_topic_rkt(rk, rkt);
+ test_auto_create_topic_rkt(rk, rkt, tmout_multip(5000));
TEST_SAY("Producing %d messages that should timeout\n", msgcnt);
test_produce_msgs_nowait(rk, rkt, testid, 0, 0, msgcnt,
diff -Nru librdkafka-0.11.3/tests/0070-null_empty.cpp librdkafka-0.11.6/tests/0070-null_empty.cpp
--- librdkafka-0.11.3/tests/0070-null_empty.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/0070-null_empty.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -80,6 +80,7 @@
Test::conf_init(&conf, NULL, 0);
Test::conf_set(conf, "api.version.request",
api_version_request ? "true" : "false");
+ Test::conf_set(conf, "acks", "all");
std::string errstr;
diff -Nru librdkafka-0.11.3/tests/0072-headers_ut.c librdkafka-0.11.6/tests/0072-headers_ut.c
--- librdkafka-0.11.3/tests/0072-headers_ut.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0072-headers_ut.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,468 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+/**
+ * Local (no broker) unit-like tests of Message Headers
+ */
+
+
+
+static int exp_msgid = 0;
+
+struct expect {
+ const char *name;
+ const char *value;
+};
+
+/**
+ * @brief returns the message id
+ */
+static int expect_check (const char *what, const struct expect *expected,
+ const rd_kafka_message_t *rkmessage) {
+ const struct expect *exp;
+ rd_kafka_resp_err_t err;
+ size_t idx = 0;
+ const char *name;
+ const char *value;
+ size_t size;
+ rd_kafka_headers_t *hdrs;
+ int msgid;
+
+ if (rkmessage->len != sizeof(msgid))
+ TEST_FAIL("%s: expected message len %"PRIusz" == sizeof(int)",
+ what, rkmessage->len);
+
+ memcpy(&msgid, rkmessage->payload, rkmessage->len);
+
+ if ((err = rd_kafka_message_headers(rkmessage, &hdrs))) {
+ if (msgid == 0)
+ return 0; /* No headers expected for first message */
+
+ TEST_FAIL("%s: Expected headers in message %d: %s", what, msgid,
+ rd_kafka_err2str(err));
+ } else {
+ TEST_ASSERT(msgid != 0,
+ "%s: first message should have no headers", what);
+ }
+
+ /* msgid should always be first and has a variable value so hard to
+ * match with the expect struct. */
+ for (idx = 0, exp = expected ;
+ !rd_kafka_header_get_all(hdrs, idx, &name,
+ (const void **)&value, &size) ;
+ idx++, exp++) {
+
+ TEST_SAYL(3, "%s: Msg #%d: "
+ "Header #%"PRIusz": %s='%s' (expecting %s='%s')\n",
+ what, msgid, idx, name, value ? value : "(NULL)",
+ exp->name, exp->value ? exp->value : "(NULL)");
+
+ if (strcmp(name, exp->name))
+ TEST_FAIL("%s: Expected header %s at idx #%"PRIusz
+ ", not %s",
+ what, exp->name, idx-1, name);
+
+ if (!strcmp(name, "msgid")) {
+ int vid;
+
+ /* Special handling: compare msgid header value
+ * to message body, should be identical */
+ if (size != rkmessage->len || size != sizeof(int))
+ TEST_FAIL("%s: "
+ "Expected msgid/int-sized payload "
+ "%"PRIusz", got %"PRIusz,
+ what, size, rkmessage->len);
+
+ /* Copy to avoid unaligned access (by cast) */
+ memcpy(&vid, value, size);
+
+ if (vid != msgid)
+ TEST_FAIL("%s: Header msgid %d != payload %d",
+ what, vid, msgid);
+
+ if (exp_msgid != vid)
+ TEST_FAIL("%s: Expected msgid %d, not %d",
+ what, exp_msgid, vid);
+ continue;
+ }
+
+ if (!exp->value) {
+ /* Expected NULL value */
+ TEST_ASSERT(!value,
+ "%s: Expected NULL value for %s, got %s",
+ what, exp->name, value);
+
+ } else {
+ TEST_ASSERT(value,
+ "%s: "
+ "Expected non-NULL value for %s, got NULL",
+ what, exp->name);
+
+ TEST_ASSERT(size == strlen(exp->value),
+ "%s: Expected size %"PRIusz" for %s, "
+ "not %"PRIusz,
+ what, strlen(exp->value), exp->name, size);
+
+ TEST_ASSERT(value[size] == '\0',
+ "%s: "
+ "Expected implicit null-terminator for %s",
+ what, exp->name);
+
+ TEST_ASSERT(!strcmp(exp->value, value),
+ "%s: "
+ "Expected value %s for %s, not %s",
+ what, exp->value, exp->name, value);
+ }
+ }
+
+ TEST_ASSERT(exp->name == NULL,
+ "%s: Expected the expected, but stuck at %s which was "
+ "unexpected",
+ what, exp->name);
+
+ return msgid;
+}
+
+
+/**
+ * @brief Delivery report callback
+ */
+static void dr_msg_cb (rd_kafka_t *rk,
+ const rd_kafka_message_t *rkmessage, void *opaque) {
+ const struct expect expected[] = {
+ { "msgid", NULL }, /* special handling */
+ { "static", "hey" },
+ { "null", NULL },
+ { "empty", "" },
+ { "send1", "1" },
+ { "multi", "multi5" },
+ { NULL }
+ };
+ const struct expect replace_expected[] = {
+ { "msgid", NULL },
+ { "new", "one" },
+ { "this is the", NULL },
+ { "replaced headers\"", "" },
+ { "new", "right?" },
+ { NULL }
+ };
+ const struct expect *exp;
+ rd_kafka_headers_t *new_hdrs;
+ int msgid;
+
+ TEST_ASSERT(rkmessage->err == RD_KAFKA_RESP_ERR__MSG_TIMED_OUT,
+ "Expected message to fail with MSG_TIMED_OUT, not %s",
+ rd_kafka_err2str(rkmessage->err));
+
+ msgid = expect_check(__FUNCTION__, expected, rkmessage);
+
+ /* Replace entire headers list */
+ if (msgid > 0) {
+ new_hdrs = rd_kafka_headers_new(1);
+ rd_kafka_header_add(new_hdrs, "msgid", -1,
+ &msgid, sizeof(msgid));
+ for (exp = &replace_expected[1] ; exp->name ; exp++)
+ rd_kafka_header_add(new_hdrs,
+ exp->name, -1, exp->value, -1);
+
+ rd_kafka_message_set_headers((rd_kafka_message_t *)rkmessage,
+ new_hdrs);
+
+ expect_check(__FUNCTION__, replace_expected, rkmessage);
+ }
+
+ exp_msgid++;
+
+}
+
+static void expect_iter (const char *what,
+ const rd_kafka_headers_t *hdrs, const char *name,
+ const char **expected, size_t cnt) {
+ size_t idx;
+ rd_kafka_resp_err_t err;
+ const void *value;
+ size_t size;
+
+ for (idx = 0 ;
+ !(err = rd_kafka_header_get(hdrs, idx, name, &value, &size)) ;\
+ idx++) {
+ TEST_ASSERT(idx < cnt,
+ "%s: too many headers matching '%s', "
+ "expected %"PRIusz,
+ what, name, cnt);
+ TEST_SAYL(3, "%s: get(%"PRIusz", '%s') "
+ "expecting '%s' =? '%s'\n",
+ what, idx, name, expected[idx], (const char *)value);
+
+
+ TEST_ASSERT(!strcmp((const char *)value, expected[idx]),
+ "%s: get(%"PRIusz", '%s') expected '%s', not '%s'",
+ what, idx, name, expected[idx],
+ (const char *)value);
+ }
+
+ TEST_ASSERT(idx == cnt,
+ "%s: expected %"PRIusz" headers matching '%s', not %"PRIusz,
+ what, cnt, name, idx);
+}
+
+
+
+/**
+ * @brief First on_send() interceptor
+ */
+static rd_kafka_resp_err_t on_send1 (rd_kafka_t *rk,
+ rd_kafka_message_t *rkmessage,
+ void *ic_opaque) {
+ const struct expect expected[] = {
+ { "msgid", NULL }, /* special handling */
+ { "static", "hey" },
+ { "multi", "multi1" },
+ { "multi", "multi2" },
+ { "multi", "multi3" },
+ { "null", NULL },
+ { "empty", "" },
+ { NULL }
+ };
+ const char *expect_iter_multi[4] = {
+ "multi1",
+ "multi2",
+ "multi3",
+ "multi4" /* added below */
+ };
+ const char *expect_iter_static[1] = {
+ "hey"
+ };
+ rd_kafka_headers_t *hdrs;
+ size_t header_cnt;
+ rd_kafka_resp_err_t err;
+ const void *value;
+ size_t size;
+
+ expect_check(__FUNCTION__, expected, rkmessage);
+
+ err = rd_kafka_message_headers(rkmessage, &hdrs);
+ if (err) /* First message has no headers. */
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ header_cnt = rd_kafka_header_cnt(hdrs);
+ TEST_ASSERT(header_cnt == 7,
+ "Expected 7 length got %zd", header_cnt);
+
+ rd_kafka_header_add(hdrs, "multi", -1, "multi4", -1);
+
+ header_cnt = rd_kafka_header_cnt(hdrs);
+ TEST_ASSERT(header_cnt == 8,
+ "Expected 8 length got %zd", header_cnt);
+
+ /* test iter() */
+ expect_iter(__FUNCTION__, hdrs, "multi", expect_iter_multi, 4);
+ expect_iter(__FUNCTION__, hdrs, "static", expect_iter_static, 1);
+ expect_iter(__FUNCTION__, hdrs, "notexists", NULL, 0);
+
+ rd_kafka_header_add(hdrs, "send1", -1, "1", -1);
+
+ header_cnt = rd_kafka_header_cnt(hdrs);
+ TEST_ASSERT(header_cnt == 9,
+ "Expected 9 length got %zd", header_cnt);
+
+ rd_kafka_header_remove(hdrs, "multi");
+
+ header_cnt = rd_kafka_header_cnt(hdrs);
+ TEST_ASSERT(header_cnt == 5,
+ "Expected 5 length got %zd", header_cnt);
+
+ rd_kafka_header_add(hdrs, "multi", -1, "multi5", -1);
+
+ header_cnt = rd_kafka_header_cnt(hdrs);
+ TEST_ASSERT(header_cnt == 6,
+ "Expected 6 length got %zd", header_cnt);
+
+ /* test get_last() */
+ err = rd_kafka_header_get_last(hdrs, "multi", &value, &size);
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+ TEST_ASSERT(size == strlen("multi5") &&
+ !strcmp((const char *)value, "multi5"),
+ "expected 'multi5', not '%s'",
+ (const char *)value);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+/**
+ * @brief Second on_send() interceptor
+ */
+static rd_kafka_resp_err_t on_send2 (rd_kafka_t *rk,
+ rd_kafka_message_t *rkmessage,
+ void *ic_opaque) {
+ const struct expect expected[] = {
+ { "msgid", NULL }, /* special handling */
+ { "static", "hey" },
+ { "null", NULL },
+ { "empty", "" },
+ { "send1", "1" },
+ { "multi", "multi5" },
+ { NULL }
+ };
+
+ expect_check(__FUNCTION__, expected, rkmessage);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+/**
+ * @brief on_new() interceptor to set up message interceptors
+ * from rd_kafka_new().
+ */
+static rd_kafka_resp_err_t on_new (rd_kafka_t *rk, const rd_kafka_conf_t *conf,
+ void *ic_opaque,
+ char *errstr, size_t errstr_size) {
+ rd_kafka_interceptor_add_on_send(rk, __FILE__, on_send1, NULL);
+ rd_kafka_interceptor_add_on_send(rk, __FILE__, on_send2, NULL);
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+int main_0072_headers_ut (int argc, char **argv) {
+ const char *topic = test_mk_topic_name(__FUNCTION__ + 5, 0);
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ int i;
+ size_t header_cnt;
+ const int msgcnt = 10;
+ rd_kafka_resp_err_t err;
+
+ conf = rd_kafka_conf_new();
+ test_conf_set(conf, "message.timeout.ms", "1");
+ rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);
+
+ rd_kafka_conf_interceptor_add_on_new(conf, __FILE__, on_new, NULL);
+
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+
+ /* First message is without headers (negative testing) */
+ i = 0;
+ err = rd_kafka_producev(
+ rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_VALUE(&i, sizeof(i)),
+ RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(!err,
+ "producev() failed: %s", rd_kafka_err2str(err));
+ exp_msgid++;
+
+ for (i = 1 ; i < msgcnt ; i++, exp_msgid++) {
+ /* Use headers list on one message */
+ if (i == 3) {
+ rd_kafka_headers_t *hdrs = rd_kafka_headers_new(4);
+
+ header_cnt = rd_kafka_header_cnt(hdrs);
+ TEST_ASSERT(header_cnt == 0,
+ "Expected 0 length got %zd", header_cnt);
+
+ rd_kafka_headers_t *copied;
+
+ rd_kafka_header_add(hdrs, "msgid", -1, &i, sizeof(i));
+ rd_kafka_header_add(hdrs, "static", -1, "hey", -1);
+ rd_kafka_header_add(hdrs, "multi", -1, "multi1", -1);
+ rd_kafka_header_add(hdrs, "multi", -1, "multi2", 6);
+ rd_kafka_header_add(hdrs, "multi", -1, "multi3", strlen("multi3"));
+ rd_kafka_header_add(hdrs, "null", -1, NULL, 0);
+
+ /* Make a copy of the headers to verify copy() */
+ copied = rd_kafka_headers_copy(hdrs);
+
+ header_cnt = rd_kafka_header_cnt(hdrs);
+ TEST_ASSERT(header_cnt == 6,
+ "Expected 6 length got %zd", header_cnt);
+
+ rd_kafka_headers_destroy(hdrs);
+
+ /* Last header ("empty") is added below */
+
+ /* Try unsupported _V_HEADER() and _V_HEADERS() mix,
+ * must fail with CONFLICT */
+ err = rd_kafka_producev(
+ rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_VALUE(&i, sizeof(i)),
+ RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+ RD_KAFKA_V_HEADER("will_be_removed", "yep", -1),
+ RD_KAFKA_V_HEADERS(copied),
+ RD_KAFKA_V_HEADER("empty", "", 0),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(err == RD_KAFKA_RESP_ERR__CONFLICT,
+ "producev(): expected CONFLICT, got %s",
+ rd_kafka_err2str(err));
+
+ /* Proper call using only _V_HEADERS() */
+ rd_kafka_header_add(copied, "empty", -1, "", -1);
+ err = rd_kafka_producev(
+ rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_VALUE(&i, sizeof(i)),
+ RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+ RD_KAFKA_V_HEADERS(copied),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(!err, "producev() failed: %s",
+ rd_kafka_err2str(err));
+
+ } else {
+ err = rd_kafka_producev(
+ rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_VALUE(&i, sizeof(i)),
+ RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+ RD_KAFKA_V_HEADER("msgid", &i, sizeof(i)),
+ RD_KAFKA_V_HEADER("static", "hey", -1),
+ RD_KAFKA_V_HEADER("multi", "multi1", -1),
+ RD_KAFKA_V_HEADER("multi", "multi2", 6),
+ RD_KAFKA_V_HEADER("multi", "multi3", strlen("multi3")),
+ RD_KAFKA_V_HEADER("null", NULL, 0),
+ RD_KAFKA_V_HEADER("empty", "", 0),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(!err,
+ "producev() failed: %s", rd_kafka_err2str(err));
+ }
+ }
+
+ /* Reset expected message id for dr */
+ exp_msgid = 0;
+
+ /* Wait for timeouts and delivery reports */
+ rd_kafka_flush(rk, 5000);
+
+ rd_kafka_destroy(rk);
+
+ return 0;
+}
diff -Nru librdkafka-0.11.3/tests/0073-headers.c librdkafka-0.11.6/tests/0073-headers.c
--- librdkafka-0.11.3/tests/0073-headers.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0073-headers.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,398 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+/**
+ * Message Headers end-to-end tests
+ */
+
+
+
+static int exp_msgid = 0;
+
+struct expect {
+ const char *name;
+ const char *value;
+};
+
+
+
+static void expect_check (const char *what, const struct expect *expected,
+ rd_kafka_message_t *rkmessage, int is_const) {
+ const struct expect *exp;
+ rd_kafka_resp_err_t err;
+ size_t idx = 0;
+ const char *name;
+ const char *value;
+ size_t size;
+ rd_kafka_headers_t *hdrs;
+ int msgid;
+
+ if (rkmessage->len != sizeof(msgid))
+ TEST_FAIL("%s: expected message len %"PRIusz" == sizeof(int)",
+ what, rkmessage->len);
+
+ memcpy(&msgid, rkmessage->payload, rkmessage->len);
+
+ if ((err = rd_kafka_message_headers(rkmessage, &hdrs))) {
+ if (msgid == 0) {
+ rd_kafka_resp_err_t err2;
+ TEST_SAYL(3, "%s: Msg #%d: no headers, good\n",
+ what, msgid);
+
+ err2 = rd_kafka_message_detach_headers(rkmessage, &hdrs);
+ TEST_ASSERT(err == err2,
+ "expected detach_headers() error %s "
+ "to match headers() error %s",
+ rd_kafka_err2str(err2),
+ rd_kafka_err2str(err));
+
+ return; /* No headers expected for first message */
+ }
+
+ TEST_FAIL("%s: Expected headers in message %d: %s", what, msgid,
+ rd_kafka_err2str(err));
+ } else {
+ TEST_ASSERT(msgid != 0,
+ "%s: first message should have no headers", what);
+ }
+
+ test_headers_dump(what, 3, hdrs);
+
+ for (idx = 0, exp = expected ;
+ !rd_kafka_header_get_all(hdrs, idx, &name,
+ (const void **)&value, &size) ;
+ idx++, exp++) {
+
+ TEST_SAYL(3, "%s: Msg #%d: "
+ "Header #%"PRIusz": %s='%s' (expecting %s='%s')\n",
+ what, msgid, idx, name, value ? value : "(NULL)",
+ exp->name, exp->value ? exp->value : "(NULL)");
+
+ if (strcmp(name, exp->name))
+ TEST_FAIL("%s: Msg #%d: "
+ "Expected header %s at idx #%"PRIusz
+ ", not '%s' (%"PRIusz")",
+ what, msgid, exp->name, idx, name,
+ strlen(name));
+
+ if (!strcmp(name, "msgid")) {
+ int vid;
+
+ /* Special handling: compare msgid header value
+ * to message body, should be identical */
+ if (size != rkmessage->len || size != sizeof(int))
+ TEST_FAIL("%s: "
+ "Expected msgid/int-sized payload "
+ "%"PRIusz", got %"PRIusz,
+ what, size, rkmessage->len);
+
+ /* Copy to avoid unaligned access (by cast) */
+ memcpy(&vid, value, size);
+
+ if (vid != msgid)
+ TEST_FAIL("%s: Header msgid %d != payload %d",
+ what, vid, msgid);
+
+ if (exp_msgid != vid)
+ TEST_FAIL("%s: Expected msgid %d, not %d",
+ what, exp_msgid, vid);
+ continue;
+ }
+
+ if (!exp->value) {
+ /* Expected NULL value */
+ TEST_ASSERT(!value,
+ "%s: Expected NULL value for %s, got %s",
+ what, exp->name, value);
+
+ } else {
+ TEST_ASSERT(value,
+ "%s: "
+ "Expected non-NULL value for %s, got NULL",
+ what, exp->name);
+
+ TEST_ASSERT(size == strlen(exp->value),
+ "%s: Expected size %"PRIusz" for %s, "
+ "not %"PRIusz,
+ what, strlen(exp->value), exp->name, size);
+
+ TEST_ASSERT(value[size] == '\0',
+ "%s: "
+ "Expected implicit null-terminator for %s",
+ what, exp->name);
+
+ TEST_ASSERT(!strcmp(exp->value, value),
+ "%s: "
+ "Expected value %s for %s, not %s",
+ what, exp->value, exp->name, value);
+ }
+ }
+
+ TEST_ASSERT(exp->name == NULL,
+ "%s: Expected the expected, but stuck at %s which was "
+ "unexpected",
+ what, exp->name);
+
+ if (!strcmp(what, "handle_consumed_msg") && !is_const &&
+ (msgid % 3) == 0) {
+ rd_kafka_headers_t *dhdrs;
+
+ err = rd_kafka_message_detach_headers(rkmessage, &dhdrs);
+ TEST_ASSERT(!err,
+ "detach_headers() should not fail, got %s",
+ rd_kafka_err2str(err));
+ TEST_ASSERT(hdrs == dhdrs);
+
+ /* Verify that a new headers object can be obtained */
+ err = rd_kafka_message_headers(rkmessage, &hdrs);
+ TEST_ASSERT(err == RD_KAFKA_RESP_ERR_NO_ERROR);
+ TEST_ASSERT(hdrs != dhdrs);
+ rd_kafka_headers_destroy(dhdrs);
+
+ expect_check("post_detach_headers", expected,
+ rkmessage, is_const);
+ }
+}
+
+
+/**
+ * @brief Final (as in no more header modifications) message check.
+ */
+static void msg_final_check (const char *what,
+ rd_kafka_message_t *rkmessage, int is_const) {
+ const struct expect expected[] = {
+ { "msgid", NULL }, /* special handling */
+ { "static", "hey" },
+ { "null", NULL },
+ { "empty", "" },
+ { "send1", "1" },
+ { "multi", "multi5" },
+ { NULL }
+ };
+
+ expect_check(what, expected, rkmessage, is_const);
+
+ exp_msgid++;
+
+
+}
+
+/**
+ * @brief Handle consumed message, must be identical to dr_msg_cb
+ */
+static void handle_consumed_msg (rd_kafka_message_t *rkmessage) {
+ msg_final_check(__FUNCTION__, rkmessage, 0);
+}
+
+/**
+ * @brief Delivery report callback
+ */
+static void dr_msg_cb (rd_kafka_t *rk,
+ const rd_kafka_message_t *rkmessage, void *opaque) {
+ TEST_ASSERT(!rkmessage->err,
+ "Message delivery failed: %s",
+ rd_kafka_err2str(rkmessage->err));
+
+ msg_final_check(__FUNCTION__, (rd_kafka_message_t *)rkmessage, 1);
+}
+
+
+/**
+ * @brief First on_send() interceptor
+ */
+static rd_kafka_resp_err_t on_send1 (rd_kafka_t *rk,
+ rd_kafka_message_t *rkmessage,
+ void *ic_opaque) {
+ const struct expect expected[] = {
+ { "msgid", NULL }, /* special handling */
+ { "static", "hey" },
+ { "multi", "multi1" },
+ { "multi", "multi2" },
+ { "multi", "multi3" },
+ { "null", NULL },
+ { "empty", "" },
+ { NULL }
+ };
+ rd_kafka_headers_t *hdrs;
+ rd_kafka_resp_err_t err;
+
+ expect_check(__FUNCTION__, expected, rkmessage, 0);
+
+ err = rd_kafka_message_headers(rkmessage, &hdrs);
+ if (err) /* First message has no headers. */
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ rd_kafka_header_add(hdrs, "multi", -1, "multi4", -1);
+ rd_kafka_header_add(hdrs, "send1", -1, "1", -1);
+ rd_kafka_header_remove(hdrs, "multi");
+ rd_kafka_header_add(hdrs, "multi", -1, "multi5", -1);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+/**
+ * @brief Second on_send() interceptor
+ */
+static rd_kafka_resp_err_t on_send2 (rd_kafka_t *rk,
+ rd_kafka_message_t *rkmessage,
+ void *ic_opaque) {
+ const struct expect expected[] = {
+ { "msgid", NULL }, /* special handling */
+ { "static", "hey" },
+ { "null", NULL },
+ { "empty", "" },
+ { "send1", "1" },
+ { "multi", "multi5" },
+ { NULL }
+ };
+
+ expect_check(__FUNCTION__, expected, rkmessage, 0);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+/**
+ * @brief on_new() interceptor to set up message interceptors
+ * from rd_kafka_new().
+ */
+static rd_kafka_resp_err_t on_new (rd_kafka_t *rk, const rd_kafka_conf_t *conf,
+ void *ic_opaque,
+ char *errstr, size_t errstr_size) {
+ rd_kafka_interceptor_add_on_send(rk, __FILE__, on_send1, NULL);
+ rd_kafka_interceptor_add_on_send(rk, __FILE__, on_send2, NULL);
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+static void do_produce (const char *topic, int msgcnt) {
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ int i;
+ rd_kafka_resp_err_t err;
+
+ test_conf_init(&conf, NULL, 0);
+ test_conf_set(conf, "acks", "all");
+ rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);
+
+ rd_kafka_conf_interceptor_add_on_new(conf, __FILE__, on_new, NULL);
+
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+
+ /* First message is without headers (negative testing) */
+ i = 0;
+ err = rd_kafka_producev(
+ rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_PARTITION(0),
+ RD_KAFKA_V_VALUE(&i, sizeof(i)),
+ RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(!err,
+ "producev() failed: %s", rd_kafka_err2str(err));
+ exp_msgid++;
+
+ for (i = 1 ; i < msgcnt ; i++, exp_msgid++) {
+ err = rd_kafka_producev(
+ rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_PARTITION(0),
+ RD_KAFKA_V_VALUE(&i, sizeof(i)),
+ RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+ RD_KAFKA_V_HEADER("msgid", &i, sizeof(i)),
+ RD_KAFKA_V_HEADER("static", "hey", -1),
+ RD_KAFKA_V_HEADER("multi", "multi1", -1),
+ RD_KAFKA_V_HEADER("multi", "multi2", 6),
+ RD_KAFKA_V_HEADER("multi", "multi3", strlen("multi3")),
+ RD_KAFKA_V_HEADER("null", NULL, 0),
+ RD_KAFKA_V_HEADER("empty", "", 0),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(!err,
+ "producev() failed: %s", rd_kafka_err2str(err));
+ }
+
+ /* Reset expected message id for dr */
+ exp_msgid = 0;
+
+ /* Wait for timeouts and delivery reports */
+ rd_kafka_flush(rk, tmout_multip(5000));
+
+ rd_kafka_destroy(rk);
+}
+
+static void do_consume (const char *topic, int msgcnt) {
+ rd_kafka_t *rk;
+ rd_kafka_topic_partition_list_t *parts;
+
+ rk = test_create_consumer(topic, NULL, NULL, NULL);
+
+ parts = rd_kafka_topic_partition_list_new(1);
+ rd_kafka_topic_partition_list_add(parts, topic, 0)->offset =
+ RD_KAFKA_OFFSET_BEGINNING;
+
+ test_consumer_assign("assign", rk, parts);
+
+ rd_kafka_topic_partition_list_destroy(parts);
+
+ exp_msgid = 0;
+
+ while (exp_msgid < msgcnt) {
+ rd_kafka_message_t *rkm;
+
+ rkm = rd_kafka_consumer_poll(rk, 1000);
+ if (!rkm)
+ continue;
+
+ if (rkm->err)
+ TEST_FAIL("consume error while expecting msgid %d/%d: "
+ "%s",
+ exp_msgid, msgcnt,
+ rd_kafka_message_errstr(rkm));
+
+ handle_consumed_msg(rkm);
+
+ rd_kafka_message_destroy(rkm);
+ }
+
+ test_consumer_close(rk);
+ rd_kafka_destroy(rk);
+}
+
+
+int main_0073_headers (int argc, char **argv) {
+ const char *topic = test_mk_topic_name(__FUNCTION__ + 5, 1);
+ const int msgcnt = 10;
+
+ do_produce(topic, msgcnt);
+ do_consume(topic, msgcnt);
+
+ return 0;
+}
diff -Nru librdkafka-0.11.3/tests/0075-retry.c librdkafka-0.11.6/tests/0075-retry.c
--- librdkafka-0.11.3/tests/0075-retry.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0075-retry.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,246 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+
+#if WITH_SOCKEM
+#include "rdkafka.h"
+
+#include
+#include
+
+/**
+ * Request retry testing
+ */
+
+/* Hang on to the first broker socket we see in connect_cb,
+ * reject all the rest (connection refused) to make sure we're only
+ * playing with one single broker for this test. */
+static struct {
+ mtx_t lock;
+ cnd_t cnd;
+ sockem_t *skm;
+ thrd_t thrd;
+ struct {
+ int64_t ts_at; /* to ctrl thread: at this time, set delay */
+ int delay;
+ int ack; /* from ctrl thread: new delay acked */
+ } cmd;
+ struct {
+ int64_t ts_at; /* to ctrl thread: at this time, set delay */
+ int delay;
+
+ } next;
+ int term;
+} ctrl;
+
+static int ctrl_thrd_main (void *arg) {
+
+
+ mtx_lock(&ctrl.lock);
+ while (!ctrl.term) {
+ int64_t now;
+
+ cnd_timedwait_ms(&ctrl.cnd, &ctrl.lock, 10);
+
+ if (ctrl.cmd.ts_at) {
+ ctrl.next.ts_at = ctrl.cmd.ts_at;
+ ctrl.next.delay = ctrl.cmd.delay;
+ ctrl.cmd.ts_at = 0;
+ ctrl.cmd.ack = 1;
+ printf(_C_CYA "## %s: sockem: "
+ "receieved command to set delay "
+ "to %d in %dms\n" _C_CLR,
+ __FILE__,
+ ctrl.next.delay,
+ (int)(ctrl.next.ts_at - test_clock()) / 1000);
+
+ }
+
+ now = test_clock();
+ if (ctrl.next.ts_at && now > ctrl.next.ts_at) {
+ assert(ctrl.skm);
+ printf(_C_CYA "## %s: "
+ "sockem: setting socket delay to %d\n" _C_CLR,
+ __FILE__, ctrl.next.delay);
+ sockem_set(ctrl.skm, "delay", ctrl.next.delay, NULL);
+ ctrl.next.ts_at = 0;
+ cnd_signal(&ctrl.cnd); /* signal back to caller */
+ }
+ }
+ mtx_unlock(&ctrl.lock);
+
+ return 0;
+}
+
+
+/**
+ * @brief Sockem connect, called from **internal librdkafka thread** through
+ * librdkafka's connect_cb
+ */
+static int connect_cb (struct test *test, sockem_t *skm, const char *id) {
+
+ mtx_lock(&ctrl.lock);
+ if (ctrl.skm) {
+ /* Reject all but the first connect */
+ mtx_unlock(&ctrl.lock);
+ return ECONNREFUSED;
+ }
+
+ ctrl.skm = skm;
+
+ /* signal wakeup to main thread */
+ cnd_broadcast(&ctrl.cnd);
+ mtx_unlock(&ctrl.lock);
+
+ return 0;
+}
+
+static int is_fatal_cb (rd_kafka_t *rk, rd_kafka_resp_err_t err,
+ const char *reason) {
+ /* Ignore connectivity errors since we'll be bringing down
+ * .. connectivity.
+ * SASL auther will think a connection-down even in the auth
+ * state means the broker doesn't support SASL PLAIN. */
+ TEST_SAY("is_fatal?: %s: %s\n", rd_kafka_err2str(err), reason);
+ if (err == RD_KAFKA_RESP_ERR__TRANSPORT ||
+ err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN ||
+ err == RD_KAFKA_RESP_ERR__AUTHENTICATION ||
+ err == RD_KAFKA_RESP_ERR__TIMED_OUT)
+ return 0;
+ return 1;
+}
+
+/**
+ * @brief Set socket delay to kick in after \p after ms
+ */
+static void set_delay (int after, int delay) {
+ TEST_SAY("Set delay to %dms (after %dms)\n", delay, after);
+
+ mtx_lock(&ctrl.lock);
+ ctrl.cmd.ts_at = test_clock() + (after*1000);
+ ctrl.cmd.delay = delay;
+ ctrl.cmd.ack = 0;
+ cnd_broadcast(&ctrl.cnd);
+
+ /* Wait for ack from sockem thread */
+ while (!ctrl.cmd.ack) {
+ TEST_SAY("Waiting for sockem control ack\n");
+ cnd_timedwait_ms(&ctrl.cnd, &ctrl.lock, 1000);
+ }
+ mtx_unlock(&ctrl.lock);
+}
+
+/**
+ * @brief Test that Metadata requests are retried properly when
+ * timing out due to high broker rtt.
+ */
+static void do_test_low_socket_timeout (const char *topic) {
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ rd_kafka_topic_t *rkt;
+ rd_kafka_resp_err_t err;
+ const struct rd_kafka_metadata *md;
+
+ mtx_init(&ctrl.lock, mtx_plain);
+ cnd_init(&ctrl.cnd);
+
+ TEST_SAY("Test Metadata request retries on timeout\n");
+
+ test_conf_init(&conf, NULL, 60);
+ test_conf_set(conf, "socket.timeout.ms", "1000");
+ test_conf_set(conf, "socket.max.fails", "12345");
+ test_conf_set(conf, "retry.backoff.ms", "5000");
+ /* Avoid api version requests (with their own timeout) to get in
+ * the way of our test */
+ test_conf_set(conf, "api.version.request", "false");
+ test_socket_enable(conf);
+ test_curr->connect_cb = connect_cb;
+ test_curr->is_fatal_cb = is_fatal_cb;
+
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+ rkt = test_create_producer_topic(rk, topic, NULL);
+
+ TEST_SAY("Waiting for sockem connect..\n");
+ mtx_lock(&ctrl.lock);
+ while (!ctrl.skm)
+ cnd_wait(&ctrl.cnd, &ctrl.lock);
+ mtx_unlock(&ctrl.lock);
+
+ TEST_SAY("Connected, fire off a undelayed metadata() to "
+ "make sure connection is up\n");
+
+ err = rd_kafka_metadata(rk, 0, rkt, &md, tmout_multip(2000));
+ TEST_ASSERT(!err, "metadata(undelayed) failed: %s",
+ rd_kafka_err2str(err));
+ rd_kafka_metadata_destroy(md);
+
+ if (thrd_create(&ctrl.thrd, ctrl_thrd_main, NULL) != thrd_success)
+ TEST_FAIL("Failed to create sockem ctrl thread");
+
+ set_delay(0, 3000); /* Takes effect immediately */
+
+ /* After two retries, remove the delay, the third retry
+ * should kick in and work. */
+ set_delay(((1000 /*socket.timeout.ms*/ +
+ 5000 /*retry.backoff.ms*/) * 2) - 2000, 0);
+
+ TEST_SAY("Calling metadata() again which should succeed after "
+ "3 internal retries\n");
+ /* Metadata should be returned after the third retry */
+ err = rd_kafka_metadata(rk, 0, rkt, &md,
+ ((1000 /*socket.timeout.ms*/ +
+ 5000 /*retry.backoff.ms*/) * 2) + 5000);
+ TEST_SAY("metadata() returned %s\n", rd_kafka_err2str(err));
+ TEST_ASSERT(!err, "metadata(undelayed) failed: %s",
+ rd_kafka_err2str(err));
+ rd_kafka_metadata_destroy(md);
+
+ rd_kafka_topic_destroy(rkt);
+ rd_kafka_destroy(rk);
+
+ /* Join controller thread */
+ mtx_lock(&ctrl.lock);
+ ctrl.term = 1;
+ mtx_unlock(&ctrl.lock);
+ thrd_join(ctrl.thrd, NULL);
+
+ cnd_destroy(&ctrl.cnd);
+ mtx_destroy(&ctrl.lock);
+}
+
+int main_0075_retry (int argc, char **argv) {
+ const char *topic = test_mk_topic_name("0075_retry", 1);
+
+ do_test_low_socket_timeout(topic);
+
+ return 0;
+}
+
+
+#endif
diff -Nru librdkafka-0.11.3/tests/0076-produce_retry.c librdkafka-0.11.6/tests/0076-produce_retry.c
--- librdkafka-0.11.3/tests/0076-produce_retry.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0076-produce_retry.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,369 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+#include
+#include
+
+static int is_fatal_cb (rd_kafka_t *rk, rd_kafka_resp_err_t err,
+ const char *reason) {
+ /* Ignore connectivity errors since we'll be bringing down
+ * .. connectivity.
+ * SASL auther will think a connection-down even in the auth
+ * state means the broker doesn't support SASL PLAIN. */
+ TEST_SAY("is_fatal?: %s: %s\n", rd_kafka_err2str(err), reason);
+ if (err == RD_KAFKA_RESP_ERR__TRANSPORT ||
+ err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN ||
+ err == RD_KAFKA_RESP_ERR__AUTHENTICATION ||
+ err == RD_KAFKA_RESP_ERR__TIMED_OUT)
+ return 0;
+ return 1;
+}
+
+
+#if WITH_SOCKEM
+/**
+ * Producer message retry testing
+ */
+
+/* Hang on to the first broker socket we see in connect_cb,
+ * reject all the rest (connection refused) to make sure we're only
+ * playing with one single broker for this test. */
+static struct {
+ mtx_t lock;
+ cnd_t cnd;
+ thrd_t thrd;
+ struct {
+ int64_t ts_at; /* to ctrl thread: at this time, set delay */
+ int delay;
+ int ack; /* from ctrl thread: new delay acked */
+ } cmd;
+ struct {
+ int64_t ts_at; /* to ctrl thread: at this time, set delay */
+ int delay;
+
+ } next;
+ int term;
+} ctrl;
+
+static int ctrl_thrd_main (void *arg) {
+ test_curr = (struct test *)arg;
+
+ mtx_lock(&ctrl.lock);
+ while (!ctrl.term) {
+ int64_t now;
+
+ cnd_timedwait_ms(&ctrl.cnd, &ctrl.lock, 10);
+
+ if (ctrl.cmd.ts_at) {
+ ctrl.next.ts_at = ctrl.cmd.ts_at;
+ ctrl.next.delay = ctrl.cmd.delay;
+ ctrl.cmd.ts_at = 0;
+ ctrl.cmd.ack = 1;
+ printf(_C_CYA "## %s: sockem: "
+ "receieved command to set delay "
+ "to %d in %dms\n" _C_CLR,
+ __FILE__,
+ ctrl.next.delay,
+ (int)(ctrl.next.ts_at - test_clock()) / 1000);
+
+ }
+
+ now = test_clock();
+ if (ctrl.next.ts_at && now > ctrl.next.ts_at) {
+ printf(_C_CYA "## %s: "
+ "sockem: setting socket delay to %d\n" _C_CLR,
+ __FILE__, ctrl.next.delay);
+ test_socket_sockem_set_all("delay", ctrl.next.delay);
+ ctrl.next.ts_at = 0;
+ cnd_signal(&ctrl.cnd); /* signal back to caller */
+ }
+ }
+ mtx_unlock(&ctrl.lock);
+
+ return 0;
+}
+
+
+
+/**
+ * @brief Set socket delay to kick in after \p after ms
+ */
+static void set_delay (int after, int delay) {
+ TEST_SAY("Set delay to %dms (after %dms)\n", delay, after);
+
+ mtx_lock(&ctrl.lock);
+ ctrl.cmd.ts_at = test_clock() + (after*1000);
+ ctrl.cmd.delay = delay;
+ ctrl.cmd.ack = 0;
+ cnd_broadcast(&ctrl.cnd);
+
+ /* Wait for ack from sockem thread */
+ while (!ctrl.cmd.ack) {
+ TEST_SAY("Waiting for sockem control ack\n");
+ cnd_timedwait_ms(&ctrl.cnd, &ctrl.lock, 1000);
+ }
+ mtx_unlock(&ctrl.lock);
+}
+
+/**
+ * @brief Test produce retries.
+ *
+ * @param should_fail If true, do negative testing which should fail.
+ */
+static void do_test_produce_retries (const char *topic, int should_fail) {
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ rd_kafka_topic_t *rkt;
+ uint64_t testid;
+ rd_kafka_resp_err_t err;
+ int msgcnt = 1;
+
+ TEST_SAY(_C_BLU "Test produce retries (should_fail=%d)\n", should_fail);
+
+ memset(&ctrl, 0, sizeof(ctrl));
+ mtx_init(&ctrl.lock, mtx_plain);
+ cnd_init(&ctrl.cnd);
+
+ testid = test_id_generate();
+
+ test_conf_init(&conf, NULL, 60);
+ test_conf_set(conf, "socket.timeout.ms", "1000");
+ /* Avoid disconnects on request timeouts */
+ test_conf_set(conf, "socket.max.fails", "100");
+ if (!should_fail) {
+ test_conf_set(conf, "retries", "5");
+ } else {
+ test_conf_set(conf, "retries", "0");
+ test_curr->exp_dr_err = RD_KAFKA_RESP_ERR__MSG_TIMED_OUT;
+ }
+ test_conf_set(conf, "retry.backoff.ms", "5000");
+ rd_kafka_conf_set_dr_cb(conf, test_dr_cb);
+ test_socket_enable(conf);
+ test_curr->is_fatal_cb = is_fatal_cb;
+
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+ rkt = test_create_producer_topic(rk, topic, NULL);
+
+ if (thrd_create(&ctrl.thrd, ctrl_thrd_main, test_curr) != thrd_success)
+ TEST_FAIL("Failed to create sockem ctrl thread");
+
+ /* Create the topic to make sure connections are up and ready. */
+ err = test_auto_create_topic_rkt(rk, rkt, tmout_multip(5000));
+ TEST_ASSERT(!err, "topic creation failed: %s", rd_kafka_err2str(err));
+
+ /* Set initial delay to 3s */
+ set_delay(0, 3000); /* Takes effect immediately */
+
+ /* After two retries, remove the delay, the third retry
+ * should kick in and work. */
+ set_delay(((1000 /*socket.timeout.ms*/ +
+ 5000 /*retry.backoff.ms*/) * 2) - 2000, 0);
+
+ test_produce_msgs(rk, rkt, testid, RD_KAFKA_PARTITION_UA,
+ 0, msgcnt, NULL, 0);
+
+
+ rd_kafka_topic_destroy(rkt);
+ rd_kafka_destroy(rk);
+
+ if (!should_fail) {
+ TEST_SAY("Verifying messages with consumer\n");
+ test_consume_msgs_easy(NULL, topic, testid, -1, msgcnt, NULL);
+ }
+
+ /* Join controller thread */
+ mtx_lock(&ctrl.lock);
+ ctrl.term = 1;
+ mtx_unlock(&ctrl.lock);
+ thrd_join(ctrl.thrd, NULL);
+
+ cnd_destroy(&ctrl.cnd);
+ mtx_destroy(&ctrl.lock);
+
+ TEST_SAY(_C_GRN "Test produce retries (should_fail=%d): PASS\n",
+ should_fail);
+}
+#endif
+
+
+
+
+/**
+ * @brief Simple on_request_sent interceptor that simply disconnects
+ * the socket when first ProduceRequest is seen.
+ * Sub-sequent ProduceRequests will not trigger a disconnect, to allow
+ * for retries.
+ */
+static mtx_t produce_disconnect_lock;
+static int produce_disconnects = 0;
+static rd_kafka_resp_err_t on_request_sent (rd_kafka_t *rk,
+ int sockfd,
+ const char *brokername,
+ int32_t brokerid,
+ int16_t ApiKey,
+ int16_t ApiVersion,
+ int32_t CorrId,
+ size_t size,
+ void *ic_opaque) {
+
+ /* Ignore if not a ProduceRequest */
+ if (ApiKey != 0)
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ mtx_lock(&produce_disconnect_lock);
+ if (produce_disconnects == 0) {
+ char buf[512];
+ ssize_t r;
+ printf(_C_CYA "%s:%d: shutting down socket %d (%s)\n" _C_CLR,
+ __FILE__, __LINE__, sockfd, brokername);
+#ifdef _MSC_VER
+ closesocket(sockfd);
+#else
+ close(sockfd);
+#endif
+ /* There is a chance the broker responded in the
+ * time it took us to get here, so purge the
+ * socket recv buffer to make sure librdkafka does not see
+ * the response. */
+ while ((r = recv(sockfd, buf, sizeof(buf), 0)) > 0)
+ printf(_C_CYA "%s:%d: "
+ "purged %"PRIdsz" bytes from socket\n",
+ __FILE__, __LINE__, r);
+ produce_disconnects = 1;
+ }
+ mtx_unlock(&produce_disconnect_lock);
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+
+static rd_kafka_resp_err_t on_new_producer (rd_kafka_t *rk,
+ const rd_kafka_conf_t *conf,
+ void *ic_opaque,
+ char *errstr, size_t errstr_size) {
+ return rd_kafka_interceptor_add_on_request_sent(
+ rk, "disconnect_on_send",
+ on_request_sent, NULL);
+}
+
+/**
+ * @brief Test produce retries by disconnecting right after ProduceRequest
+ * has been sent.
+ *
+ * @param should_fail If true, do negative testing which should fail.
+ */
+static void do_test_produce_retries_disconnect (const char *topic,
+ int should_fail) {
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ rd_kafka_topic_t *rkt;
+ uint64_t testid;
+ rd_kafka_resp_err_t err;
+ int msgcnt = 1;
+ int partition_cnt;
+
+ TEST_SAY(_C_BLU "Test produce retries by disconnect (should_fail=%d)\n",
+ should_fail);
+
+ test_curr->is_fatal_cb = is_fatal_cb;
+
+ testid = test_id_generate();
+
+ test_conf_init(&conf, NULL, 60);
+ rd_kafka_conf_set_dr_cb(conf, test_dr_cb);
+ test_conf_set(conf, "socket.timeout.ms", "10000");
+ test_conf_set(conf, "message.timeout.ms", "30000");
+ if (!should_fail)
+ test_conf_set(conf, "retries", "1");
+ else
+ test_conf_set(conf, "retries", "0");
+
+ mtx_init(&produce_disconnect_lock, mtx_plain);
+ produce_disconnects = 0;
+
+ rd_kafka_conf_interceptor_add_on_new(conf, "on_new_producer",
+ on_new_producer, NULL);
+
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+ rkt = test_create_producer_topic(rk, topic, NULL);
+
+ err = test_produce_sync(rk, rkt, testid, 0);
+
+ if (should_fail) {
+ if (!err)
+ TEST_FAIL("Expected produce to fail\n");
+ else
+ TEST_SAY("Produced message failed as expected: %s\n",
+ rd_kafka_err2str(err));
+ } else {
+ if (err)
+ TEST_FAIL("Produced message failed: %s\n",
+ rd_kafka_err2str(err));
+ else
+ TEST_SAY("Produced message delivered\n");
+ }
+
+ mtx_lock(&produce_disconnect_lock);
+ TEST_ASSERT(produce_disconnects == 1,
+ "expected %d disconnects, not %d", 1, produce_disconnects);
+ mtx_unlock(&produce_disconnect_lock);
+
+
+ partition_cnt = test_get_partition_count(rk, topic, tmout_multip(5000));
+
+ rd_kafka_topic_destroy(rkt);
+ rd_kafka_destroy(rk);
+
+ TEST_SAY("Verifying messages with consumer\n");
+ test_consume_msgs_easy(NULL, topic, testid,
+ partition_cnt, should_fail ? 0 : msgcnt, NULL);
+
+ TEST_SAY(_C_GRN "Test produce retries by disconnect "
+ "(should_fail=%d): PASS\n",
+ should_fail);
+}
+
+
+int main_0076_produce_retry (int argc, char **argv) {
+ const char *topic = test_mk_topic_name("0076_produce_retry", 1);
+
+#if WITH_SOCKEM
+ do_test_produce_retries(topic, 0/*good test*/);
+ do_test_produce_retries(topic, 1/*fail test*/);
+#endif
+
+ do_test_produce_retries_disconnect(topic, 0/*good test*/);
+ do_test_produce_retries_disconnect(topic, 1/*fail test*/);
+
+ return 0;
+}
+
+
diff -Nru librdkafka-0.11.3/tests/0077-compaction.c librdkafka-0.11.6/tests/0077-compaction.c
--- librdkafka-0.11.3/tests/0077-compaction.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0077-compaction.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,334 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+/**
+ * @brief Verify handling of compacted topics.
+ *
+ * General idea:
+ * - create a compacted topic with a low cleanup interval to promote quick
+ * compaction.
+ * - produce messages for 3 keys and interleave with unkeyed messages.
+ * interleave tombstones for k1 and k2, but not k3.
+ * - consume before compaction - verify all messages in place
+ * - wait for compaction
+ * - consume after compaction - verify expected messages.
+ */
+
+
+
+/**
+ * @brief Get low watermark in partition, we use this see if compaction
+ * has kicked in.
+ */
+static int64_t get_low_wmark (rd_kafka_t *rk, const char *topic,
+ int32_t partition) {
+ rd_kafka_resp_err_t err;
+ int64_t low, high;
+
+ err = rd_kafka_query_watermark_offsets(rk, topic, partition,
+ &low, &high,
+ tmout_multip(10000));
+
+ TEST_ASSERT(!err, "query_warmark_offsets(%s, %d) failed: %s",
+ topic, (int)partition, rd_kafka_err2str(err));
+
+ return low;
+}
+
+
+/**
+ * @brief Wait for compaction by checking for
+ * partition low-watermark increasing */
+static void wait_compaction (rd_kafka_t *rk,
+ const char *topic, int32_t partition,
+ int64_t low_offset,
+ int timeout_ms) {
+ int64_t low = -1;
+ int64_t ts_start = test_clock();
+
+ TEST_SAY("Waiting for compaction to kick in and increase the "
+ "Low watermark offset from %"PRId64" on %s [%"PRId32"]\n",
+ low_offset, topic, partition);
+
+ while (1) {
+ low = get_low_wmark(rk, topic, partition);
+
+ TEST_SAY("Low watermark offset for %s [%"PRId32"] is "
+ "%"PRId64" (want > %"PRId64")\n",
+ topic, partition, low, low_offset);
+
+ if (low > low_offset)
+ break;
+
+ if (ts_start + (timeout_ms * 1000) < test_clock())
+ break;
+
+ rd_sleep(5);
+ }
+}
+
+static void produce_compactable_msgs (const char *topic, int32_t partition,
+ uint64_t testid,
+ int msgcnt, size_t msgsize) {
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ int i;
+ char *val;
+ char key[16];
+ rd_kafka_resp_err_t err;
+ int msgcounter = msgcnt;
+
+ if (!testid)
+ testid = test_id_generate();
+
+ test_str_id_generate(key, sizeof(key));
+
+ val = calloc(1, msgsize);
+
+ TEST_SAY("Producing %d messages (total of %"PRIusz" bytes) of "
+ "compactable messages\n", msgcnt, (size_t)msgcnt*msgsize);
+
+ test_conf_init(&conf, NULL, 0);
+ rd_kafka_conf_set_dr_cb(conf, test_dr_cb);
+ /* Make sure batch size does not exceed segment.bytes since that
+ * will make the ProduceRequest fail. */
+ test_conf_set(conf, "batch.num.messages", "1");
+
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+
+ for (i = 0 ; i < msgcnt-1 ; i++) {
+ err = rd_kafka_producev(rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_PARTITION(partition),
+ RD_KAFKA_V_KEY(key, sizeof(key)-1),
+ RD_KAFKA_V_VALUE(val, msgsize),
+ RD_KAFKA_V_OPAQUE(&msgcounter),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(!err, "producev(): %s", rd_kafka_err2str(err));
+ }
+
+ /* Final message is the tombstone */
+ err = rd_kafka_producev(rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_PARTITION(partition),
+ RD_KAFKA_V_KEY(key, sizeof(key)-1),
+ RD_KAFKA_V_OPAQUE(&msgcounter),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(!err, "producev(): %s", rd_kafka_err2str(err));
+
+ test_flush(rk, tmout_multip(10000));
+ TEST_ASSERT(msgcounter == 0, "%d messages unaccounted for", msgcounter);
+
+ rd_kafka_destroy(rk);
+
+ free(val);
+}
+
+
+
+static void do_test_compaction (int msgs_per_key, const char *compression) {
+ const char *topic = test_mk_topic_name(__FILE__, 1);
+#define _KEY_CNT 4
+ const char *keys[_KEY_CNT] = { "k1", "k2", "k3", NULL/*generate unique*/ };
+ int msgcnt = msgs_per_key * _KEY_CNT;
+ rd_kafka_conf_t *conf;
+ rd_kafka_t *rk;
+ rd_kafka_topic_t *rkt;
+ uint64_t testid;
+ int32_t partition = 0;
+ int cnt = 0;
+ test_msgver_t mv;
+ test_msgver_t mv_correct;
+ int msgcounter = 0;
+ const int fillcnt = 20;
+
+ testid = test_id_generate();
+
+ TEST_SAY(_C_MAG "Test compaction on topic %s with %s compression (%d messages)\n",
+ topic, compression ? compression : "no", msgcnt);
+
+ test_kafka_topics("--create --topic \"%s\" "
+ "--partitions %d "
+ "--replication-factor 1 "
+ "--config cleanup.policy=compact "
+ "--config segment.ms=10000 "
+ "--config segment.bytes=10000 "
+ "--config min.cleanable.dirty.ratio=0.01 "
+ "--config delete.retention.ms=86400 "
+ "--config file.delete.delay.ms=10000",
+ topic, partition+1);
+
+ test_conf_init(&conf, NULL, 120);
+ rd_kafka_conf_set_dr_cb(conf, test_dr_cb);
+ if (compression)
+ test_conf_set(conf, "compression.codec", compression);
+ /* Limit max batch size below segment.bytes to avoid messages
+ * to accumulate into a batch that will be rejected by the broker. */
+ test_conf_set(conf, "message.max.bytes", "6000");
+ test_conf_set(conf, "linger.ms", "10");
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+ rkt = rd_kafka_topic_new(rk, topic, NULL);
+
+ /* The low watermark is not updated on message deletion(compaction)
+ * but on segment deletion, so fill up the first segment with
+ * random messages eligible for hasty compaction. */
+ produce_compactable_msgs(topic, 0, partition, fillcnt, 1000);
+
+ /* Populate a correct msgver for later comparison after compact. */
+ test_msgver_init(&mv_correct, testid);
+
+ TEST_SAY("Producing %d messages for %d keys\n", msgcnt, _KEY_CNT);
+ for (cnt = 0 ; cnt < msgcnt ; ) {
+ int k;
+
+ for (k = 0 ; k < _KEY_CNT ; k++) {
+ rd_kafka_resp_err_t err;
+ int is_last = cnt + _KEY_CNT >= msgcnt;
+ /* Let keys[0] have some tombstones */
+ int is_tombstone = (k == 0 && (is_last || !(cnt % 7)));
+ char *valp;
+ size_t valsize;
+ char rdk_msgid[256];
+ char unique_key[16];
+ const void *key;
+ size_t keysize;
+ int64_t offset = fillcnt + cnt;
+
+ test_msg_fmt(rdk_msgid, sizeof(rdk_msgid),
+ testid, partition, cnt);
+
+ if (is_tombstone) {
+ valp = NULL;
+ valsize = 0;
+ } else {
+ valp = rdk_msgid;
+ valsize = strlen(valp);
+ }
+
+ if (!(key = keys[k])) {
+ rd_snprintf(unique_key, sizeof(unique_key),
+ "%d", cnt);
+ key = unique_key;
+ }
+ keysize = strlen(key);
+
+ /* All unique-key messages should remain intact
+ * after compaction. */
+ if (!keys[k] || is_last) {
+ TEST_SAYL(4,
+ "Add to correct msgvec: "
+ "msgid: %d: %s is_last=%d, "
+ "is_tomb=%d\n",
+ cnt, (const char *)key,
+ is_last, is_tombstone);
+ test_msgver_add_msg00(__FUNCTION__, __LINE__,
+ &mv_correct, testid,
+ topic, partition,
+ offset, -1, 0, cnt);
+ }
+
+
+ msgcounter++;
+ err = rd_kafka_producev(
+ rk,
+ RD_KAFKA_V_TOPIC(topic),
+ RD_KAFKA_V_PARTITION(0),
+ RD_KAFKA_V_KEY(key, keysize),
+ RD_KAFKA_V_VALUE(valp, valsize),
+ RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
+ RD_KAFKA_V_HEADER("rdk_msgid", rdk_msgid, -1),
+ /* msgcounter as msg_opaque is used
+ * by test delivery report callback to
+ * count number of messages. */
+ RD_KAFKA_V_OPAQUE(&msgcounter),
+ RD_KAFKA_V_END);
+ TEST_ASSERT(!err, "producev(#%d) failed: %s",
+ cnt, rd_kafka_err2str(err));
+
+ cnt++;
+ }
+ }
+
+ TEST_ASSERT(cnt == msgcnt, "cnt %d != msgcnt %d", cnt, msgcnt);
+
+ msgcounter = cnt;
+ test_wait_delivery(rk, &msgcounter);
+
+ /* Trigger compaction by filling up the segment with dummy messages,
+ * do it in chunks to avoid too good compression which then won't
+ * fill up the segments..
+ * We can't reuse the existing producer instance because it
+ * might be using compression which makes it hard to know how
+ * much data we need to produce to trigger compaction. */
+ produce_compactable_msgs(topic, 0, partition, 20, 1024);
+
+ /* Wait for compaction:
+ * this doesn't really work because the low watermark offset
+ * is not updated on compaction if the first segment is not deleted.
+ * But it serves as a pause to let compaction kick in
+ * which is triggered by the dummy produce above. */
+ wait_compaction(rk, topic, partition, 0, 20*1000);
+
+ TEST_SAY(_C_YEL "Verify messages after compaction\n");
+ /* After compaction we expect the following messages:
+ * last message for each of k1, k2, k3, all messages for unkeyed. */
+ test_msgver_init(&mv, testid);
+ mv.msgid_hdr = "rdk_msgid";
+ test_consume_msgs_easy_mv(NULL, topic, testid, 1, -1, NULL, &mv);
+ test_msgver_verify_compare("post-compaction", &mv, &mv_correct,
+ TEST_MSGVER_BY_MSGID|TEST_MSGVER_BY_OFFSET);
+ test_msgver_clear(&mv);
+
+ test_msgver_clear(&mv_correct);
+
+ rd_kafka_topic_destroy(rkt);
+ rd_kafka_destroy(rk);
+
+ TEST_SAY(_C_GRN "Compaction test with %s compression: PASS\n",
+ compression ? compression : "no");
+}
+
+int main_0077_compaction (int argc, char **argv) {
+
+ if (!test_can_create_topics(1))
+ return 0;
+
+ do_test_compaction(10, NULL);
+ do_test_compaction(1000, NULL);
+#if WITH_SNAPPY
+ do_test_compaction(10, "snappy");
+#endif
+#if WITH_ZLIB
+ do_test_compaction(10000, "gzip");
+#endif
+
+ return 0;
+}
diff -Nru librdkafka-0.11.3/tests/0078-c_from_cpp.cpp librdkafka-0.11.6/tests/0078-c_from_cpp.cpp
--- librdkafka-0.11.3/tests/0078-c_from_cpp.cpp 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0078-c_from_cpp.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,94 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2016, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "rdkafka.h" /* Include before rdkafkacpp.h (from testcpp.h) */
+#include "testcpp.h"
+#include
+
+/**
+ * @name Verify that the c_ptr()'s returned from C++ can be used
+ * to interact directly with the C API.
+ */
+
+
+extern "C" {
+ int main_0078_c_from_cpp (int argc, char **argv) {
+ RdKafka::Conf *conf = RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL);
+
+ std::string errstr;
+
+ if (conf->set("client.id", "myclient", errstr))
+ Test::Fail("conf->set() failed: " + errstr);
+
+ RdKafka::Producer *p = RdKafka::Producer::create(conf, errstr);
+ if (!p)
+ Test::Fail("Failed to create Producer: " + errstr);
+
+ delete conf;
+
+ /*
+ * Acquire rd_kafka_t and compare its name to the configured client.id
+ */
+ rd_kafka_t *rk = p->c_ptr();
+ if (!rk)
+ Test::Fail("Failed to acquire c_ptr");
+
+ std::string name = p->name();
+ std::string c_name = rd_kafka_name(rk);
+
+ Test::Say("Compare C name " + c_name + " to C++ name " + name + "\n");
+ if (c_name != name)
+ Test::Fail("Expected C client name " + c_name + " to match C++ " + name);
+
+ /*
+ * Create topic object, acquire rd_kafka_topic_t and compare
+ * its topic name.
+ */
+
+ RdKafka::Topic *topic = RdKafka::Topic::create(p, "mytopic", NULL, errstr);
+ if (!topic)
+ Test::Fail("Failed to create Topic: " + errstr);
+
+ rd_kafka_topic_t *rkt = topic->c_ptr();
+ if (!rkt)
+ Test::Fail("Failed to acquire topic c_ptr");
+
+ std::string topicname = topic->name();
+ std::string c_topicname = rd_kafka_topic_name(rkt);
+
+ Test::Say("Compare C topic " + c_topicname + " to C++ topic " + topicname + "\n");
+ if (c_topicname != topicname)
+ Test::Fail("Expected C topic " + c_topicname + " to match C++ topic " + topicname);
+
+ delete topic;
+ delete p;
+
+ return 0;
+ }
+}
diff -Nru librdkafka-0.11.3/tests/0079-fork.c librdkafka-0.11.6/tests/0079-fork.c
--- librdkafka-0.11.3/tests/0079-fork.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0079-fork.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,96 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+#ifndef _MSC_VER
+#include
+#include
+#endif
+
+/**
+ * @brief Forking a threaded process will not transfer threads (such as
+ * librdkafka's background threads) to the child process.
+ * There is no way such a forked client instance will work
+ * in the child process, but it should not crash on destruction: #1674
+ */
+
+int main_0079_fork (int argc, char **argv) {
+
+#if __SANITIZE_ADDRESS__
+ TEST_SKIP("AddressSanitizer is enabled: this test leaks memory (due to fork())\n");
+ return 0;
+#endif
+#ifdef _MSC_VER
+ TEST_SKIP("No fork() support on Windows");
+ return 0;
+#else
+ pid_t pid;
+ rd_kafka_t *rk;
+ int status;
+
+ rk = test_create_producer();
+
+ rd_kafka_producev(rk,
+ RD_KAFKA_V_TOPIC("atopic"),
+ RD_KAFKA_V_VALUE("hi", 2),
+ RD_KAFKA_V_END);
+
+ pid = fork();
+ TEST_ASSERT(pid != 1, "fork() failed: %s", strerror(errno));
+
+ if (pid == 0) {
+ /* Child process */
+
+ /* This call will enqueue the message on a queue
+ * which is not served by any thread, but it should not crash */
+ rd_kafka_producev(rk,
+ RD_KAFKA_V_TOPIC("atopic"),
+ RD_KAFKA_V_VALUE("hello", 5),
+ RD_KAFKA_V_END);
+
+ /* Don't crash on us */
+ rd_kafka_destroy(rk);
+
+ exit(0);
+ }
+
+ /* Parent process, wait for child to exit cleanly. */
+ if (waitpid(pid, &status, 0) == -1)
+ TEST_FAIL("waitpid(%d) failed: %s", (int)pid, strerror(errno));
+
+ if (!WIFEXITED(status) ||
+ WEXITSTATUS(status) != 0)
+ TEST_FAIL("child exited with status %d", WEXITSTATUS(status));
+
+ rd_kafka_destroy(rk);
+
+ return 0;
+#endif
+}
diff -Nru librdkafka-0.11.3/tests/0080-admin_ut.c librdkafka-0.11.6/tests/0080-admin_ut.c
--- librdkafka-0.11.3/tests/0080-admin_ut.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0080-admin_ut.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,751 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+/**
+ * @brief Admin API local dry-run unit-tests.
+ */
+
+#define MY_SOCKET_TIMEOUT_MS 1500
+#define MY_SOCKET_TIMEOUT_MS_STR "1500"
+
+
+
+static mtx_t last_event_lock;
+static cnd_t last_event_cnd;
+static rd_kafka_event_t *last_event = NULL;
+
+/**
+ * @brief The background event callback is called automatically
+ * by librdkafka from a background thread.
+ */
+static void background_event_cb (rd_kafka_t *rk, rd_kafka_event_t *rkev,
+ void *opaque) {
+ mtx_lock(&last_event_lock);
+ TEST_ASSERT(!last_event, "Multiple events seen in background_event_cb "
+ "(existing %s, new %s)",
+ rd_kafka_event_name(last_event), rd_kafka_event_name(rkev));
+ last_event = rkev;
+ mtx_unlock(&last_event_lock);
+ cnd_broadcast(&last_event_cnd);
+ rd_sleep(1);
+}
+
+static rd_kafka_event_t *wait_background_event_cb (void) {
+ rd_kafka_event_t *rkev;
+ mtx_lock(&last_event_lock);
+ while (!(rkev = last_event))
+ cnd_wait(&last_event_cnd, &last_event_lock);
+ last_event = NULL;
+ mtx_unlock(&last_event_lock);
+
+ return rkev;
+}
+
+
+/**
+ * @brief CreateTopics tests
+ *
+ *
+ *
+ */
+static void do_test_CreateTopics (const char *what,
+ rd_kafka_t *rk, rd_kafka_queue_t *useq,
+ int with_background_event_cb,
+ int with_options) {
+ rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
+#define MY_NEW_TOPICS_CNT 6
+ rd_kafka_NewTopic_t *new_topics[MY_NEW_TOPICS_CNT];
+ rd_kafka_AdminOptions_t *options = NULL;
+ int exp_timeout = MY_SOCKET_TIMEOUT_MS;
+ int i;
+ char errstr[512];
+ const char *errstr2;
+ rd_kafka_resp_err_t err;
+ test_timing_t timing;
+ rd_kafka_event_t *rkev;
+ const rd_kafka_CreateTopics_result_t *res;
+ const rd_kafka_topic_result_t **restopics;
+ size_t restopic_cnt;
+ void *my_opaque = NULL, *opaque;
+
+ TEST_SAY(_C_MAG "[ %s CreateTopics with %s, timeout %dms ]\n",
+ rd_kafka_name(rk), what, exp_timeout);
+
+ /**
+ * Construct NewTopic array with different properties for
+ * different partitions.
+ */
+ for (i = 0 ; i < MY_NEW_TOPICS_CNT ; i++) {
+ const char *topic = test_mk_topic_name(__FUNCTION__, 1);
+ int num_parts = i * 51 + 1;
+ int num_replicas = jitter(1, MY_NEW_TOPICS_CNT-1);
+ int set_config = (i & 2);
+ int set_replicas = !(i % 1);
+
+ new_topics[i] = rd_kafka_NewTopic_new(topic,
+ num_parts,
+ set_replicas ? -1 :
+ num_replicas,
+ NULL, 0);
+
+ if (set_config) {
+ /*
+ * Add various (unverified) configuration properties
+ */
+ err = rd_kafka_NewTopic_set_config(new_topics[i],
+ "dummy.doesntexist",
+ "butThere'sNothing "
+ "to verify that");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ err = rd_kafka_NewTopic_set_config(new_topics[i],
+ "try.a.null.value",
+ NULL);
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ err = rd_kafka_NewTopic_set_config(new_topics[i],
+ "or.empty", "");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+ }
+
+
+ if (set_replicas) {
+ int32_t p;
+ int32_t replicas[MY_NEW_TOPICS_CNT];
+ int j;
+
+ for (j = 0 ; j < num_replicas ; j++)
+ replicas[j] = j;
+
+ /*
+ * Set valid replica assignments
+ */
+ for (p = 0 ; p < num_parts ; p++) {
+ /* Try adding an existing out of order,
+ * should fail */
+ if (p == 1) {
+ err = rd_kafka_NewTopic_set_replica_assignment(
+ new_topics[i], p+1,
+ replicas, num_replicas,
+ errstr, sizeof(errstr));
+ TEST_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG,
+ "%s", rd_kafka_err2str(err));
+ }
+
+ err = rd_kafka_NewTopic_set_replica_assignment(
+ new_topics[i], p,
+ replicas, num_replicas,
+ errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "%s", errstr);
+ }
+
+ /* Try to add an existing partition, should fail */
+ err = rd_kafka_NewTopic_set_replica_assignment(
+ new_topics[i], 0,
+ replicas, num_replicas, NULL, 0);
+ TEST_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG,
+ "%s", rd_kafka_err2str(err));
+
+ } else {
+ int32_t dummy_replicas[1] = {1};
+
+ /* Test invalid partition */
+ err = rd_kafka_NewTopic_set_replica_assignment(
+ new_topics[i], num_parts+1, dummy_replicas, 1,
+ errstr, sizeof(errstr));
+ TEST_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG,
+ "%s: %s", rd_kafka_err2str(err),
+ err == RD_KAFKA_RESP_ERR_NO_ERROR ?
+ "" : errstr);
+
+ /* Setting replicas with with default replicas != -1
+ * is an error. */
+ err = rd_kafka_NewTopic_set_replica_assignment(
+ new_topics[i], 0, dummy_replicas, 1,
+ errstr, sizeof(errstr));
+ TEST_ASSERT(err == RD_KAFKA_RESP_ERR__INVALID_ARG,
+ "%s: %s", rd_kafka_err2str(err),
+ err == RD_KAFKA_RESP_ERR_NO_ERROR ?
+ "" : errstr);
+ }
+ }
+
+ if (with_options) {
+ options = rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_ANY);
+
+ exp_timeout = MY_SOCKET_TIMEOUT_MS * 2;
+ err = rd_kafka_AdminOptions_set_request_timeout(
+ options, exp_timeout, errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ my_opaque = (void *)123;
+ rd_kafka_AdminOptions_set_opaque(options, my_opaque);
+ }
+
+ TIMING_START(&timing, "CreateTopics");
+ TEST_SAY("Call CreateTopics, timeout is %dms\n", exp_timeout);
+ rd_kafka_CreateTopics(rk, new_topics, MY_NEW_TOPICS_CNT,
+ options, q);
+ TIMING_ASSERT_LATER(&timing, 0, 50);
+
+ if (with_background_event_cb) {
+ /* Result event will be triggered by callback from
+ * librdkafka background queue thread. */
+ TIMING_START(&timing, "CreateTopics.wait_background_event_cb");
+ rkev = wait_background_event_cb();
+ } else {
+ /* Poll result queue */
+ TIMING_START(&timing, "CreateTopics.queue_poll");
+ rkev = rd_kafka_queue_poll(q, exp_timeout + 1000);
+ }
+
+ TIMING_ASSERT_LATER(&timing, exp_timeout-100, exp_timeout+100);
+ TEST_ASSERT(rkev != NULL, "expected result in %dms",
+ exp_timeout);
+ TEST_SAY("CreateTopics: got %s in %.3fs\n",
+ rd_kafka_event_name(rkev),
+ TIMING_DURATION(&timing) / 1000.0f);
+
+ /* Convert event to proper result */
+ res = rd_kafka_event_CreateTopics_result(rkev);
+ TEST_ASSERT(res, "expected CreateTopics_result, not %s",
+ rd_kafka_event_name(rkev));
+
+ opaque = rd_kafka_event_opaque(rkev);
+ TEST_ASSERT(opaque == my_opaque, "expected opaque to be %p, not %p",
+ my_opaque, opaque);
+
+ /* Expecting error */
+ err = rd_kafka_event_error(rkev);
+ errstr2 = rd_kafka_event_error_string(rkev);
+ TEST_ASSERT(err == RD_KAFKA_RESP_ERR__TIMED_OUT,
+ "expected CreateTopics to return error %s, not %s (%s)",
+ rd_kafka_err2str(RD_KAFKA_RESP_ERR__TIMED_OUT),
+ rd_kafka_err2str(err),
+ err ? errstr2 : "n/a");
+
+ /* Attempt to extract topics anyway, should return NULL. */
+ restopics = rd_kafka_CreateTopics_result_topics(res, &restopic_cnt);
+ TEST_ASSERT(!restopics && restopic_cnt == 0,
+ "expected no result_topics, got %p cnt %"PRIusz,
+ restopics, restopic_cnt);
+
+ rd_kafka_event_destroy(rkev);
+
+ rd_kafka_NewTopic_destroy_array(new_topics, MY_NEW_TOPICS_CNT);
+
+ if (options)
+ rd_kafka_AdminOptions_destroy(options);
+
+ if (!useq)
+ rd_kafka_queue_destroy(q);
+}
+
+
+
+
+
+
+/**
+ * @brief DeleteTopics tests
+ *
+ *
+ *
+ */
+static void do_test_DeleteTopics (const char *what,
+ rd_kafka_t *rk, rd_kafka_queue_t *useq,
+ int with_options) {
+ rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
+#define MY_DEL_TOPICS_CNT 4
+ rd_kafka_DeleteTopic_t *del_topics[MY_DEL_TOPICS_CNT];
+ rd_kafka_AdminOptions_t *options = NULL;
+ int exp_timeout = MY_SOCKET_TIMEOUT_MS;
+ int i;
+ char errstr[512];
+ const char *errstr2;
+ rd_kafka_resp_err_t err;
+ test_timing_t timing;
+ rd_kafka_event_t *rkev;
+ const rd_kafka_DeleteTopics_result_t *res;
+ const rd_kafka_topic_result_t **restopics;
+ size_t restopic_cnt;
+ void *my_opaque = NULL, *opaque;
+
+ TEST_SAY(_C_MAG "[ %s DeleteTopics with %s, timeout %dms ]\n",
+ rd_kafka_name(rk), what, exp_timeout);
+
+ for (i = 0 ; i < MY_DEL_TOPICS_CNT ; i++)
+ del_topics[i] = rd_kafka_DeleteTopic_new(test_mk_topic_name(__FUNCTION__, 1));
+
+ if (with_options) {
+ options = rd_kafka_AdminOptions_new(
+ rk, RD_KAFKA_ADMIN_OP_DELETETOPICS);
+
+ exp_timeout = MY_SOCKET_TIMEOUT_MS * 2;
+ err = rd_kafka_AdminOptions_set_request_timeout(
+ options, exp_timeout, errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ if (useq) {
+ my_opaque = (void *)456;
+ rd_kafka_AdminOptions_set_opaque(options, my_opaque);
+ }
+ }
+
+ TIMING_START(&timing, "DeleteTopics");
+ TEST_SAY("Call DeleteTopics, timeout is %dms\n", exp_timeout);
+ rd_kafka_DeleteTopics(rk, del_topics, MY_DEL_TOPICS_CNT,
+ options, q);
+ TIMING_ASSERT_LATER(&timing, 0, 50);
+
+ /* Poll result queue */
+ TIMING_START(&timing, "DeleteTopics.queue_poll");
+ rkev = rd_kafka_queue_poll(q, exp_timeout + 1000);
+ TIMING_ASSERT_LATER(&timing, exp_timeout-100, exp_timeout+100);
+ TEST_ASSERT(rkev != NULL, "expected result in %dms", exp_timeout);
+ TEST_SAY("DeleteTopics: got %s in %.3fs\n",
+ rd_kafka_event_name(rkev), TIMING_DURATION(&timing) / 1000.0f);
+
+ /* Convert event to proper result */
+ res = rd_kafka_event_DeleteTopics_result(rkev);
+ TEST_ASSERT(res, "expected DeleteTopics_result, not %s",
+ rd_kafka_event_name(rkev));
+
+ opaque = rd_kafka_event_opaque(rkev);
+ TEST_ASSERT(opaque == my_opaque, "expected opaque to be %p, not %p",
+ my_opaque, opaque);
+
+ /* Expecting error */
+ err = rd_kafka_event_error(rkev);
+ errstr2 = rd_kafka_event_error_string(rkev);
+ TEST_ASSERT(err == RD_KAFKA_RESP_ERR__TIMED_OUT,
+ "expected DeleteTopics to return error %s, not %s (%s)",
+ rd_kafka_err2str(RD_KAFKA_RESP_ERR__TIMED_OUT),
+ rd_kafka_err2str(err),
+ err ? errstr2 : "n/a");
+
+ /* Attempt to extract topics anyway, should return NULL. */
+ restopics = rd_kafka_DeleteTopics_result_topics(res, &restopic_cnt);
+ TEST_ASSERT(!restopics && restopic_cnt == 0,
+ "expected no result_topics, got %p cnt %"PRIusz,
+ restopics, restopic_cnt);
+
+ rd_kafka_event_destroy(rkev);
+
+ rd_kafka_DeleteTopic_destroy_array(del_topics, MY_DEL_TOPICS_CNT);
+
+ if (options)
+ rd_kafka_AdminOptions_destroy(options);
+
+ if (!useq)
+ rd_kafka_queue_destroy(q);
+}
+
+
+/**
+ * @brief Test a mix of APIs using the same replyq.
+ *
+ * - Create topics A,B
+ * - Delete topic B
+ * - Create topic C
+ * - Create extra partitions for topic D
+ */
+static void do_test_mix (rd_kafka_t *rk, rd_kafka_queue_t *rkqu) {
+ char *topics[] = { "topicA", "topicB", "topicC" };
+ int cnt = 0;
+ struct waiting {
+ rd_kafka_event_type_t evtype;
+ int seen;
+ };
+ struct waiting id1 = {RD_KAFKA_EVENT_CREATETOPICS_RESULT};
+ struct waiting id2 = {RD_KAFKA_EVENT_DELETETOPICS_RESULT};
+ struct waiting id3 = {RD_KAFKA_EVENT_CREATETOPICS_RESULT};
+ struct waiting id4 = {RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT};
+
+ TEST_SAY(_C_MAG "[ Mixed mode test on %s]\n", rd_kafka_name(rk));
+
+ test_CreateTopics_simple(rk, rkqu, topics, 2, 1, &id1);
+ test_DeleteTopics_simple(rk, rkqu, &topics[1], 1, &id2);
+ test_CreateTopics_simple(rk, rkqu, &topics[2], 1, 1, &id3);
+ test_CreatePartitions_simple(rk, rkqu, "topicD", 15, &id4);
+
+ while (cnt < 4) {
+ rd_kafka_event_t *rkev;
+ struct waiting *w;
+
+ rkev = rd_kafka_queue_poll(rkqu, -1);
+ TEST_ASSERT(rkev);
+
+ TEST_SAY("Got event %s: %s\n",
+ rd_kafka_event_name(rkev),
+ rd_kafka_event_error_string(rkev));
+
+ w = rd_kafka_event_opaque(rkev);
+ TEST_ASSERT(w);
+
+ TEST_ASSERT(w->evtype == rd_kafka_event_type(rkev),
+ "Expected evtype %d, not %d (%s)",
+ w->evtype, rd_kafka_event_type(rkev),
+ rd_kafka_event_name(rkev));
+
+ TEST_ASSERT(w->seen == 0, "Duplicate results");
+
+ w->seen++;
+ cnt++;
+
+ rd_kafka_event_destroy(rkev);
+ }
+}
+
+
+/**
+ * @brief Test AlterConfigs and DescribeConfigs
+ */
+static void do_test_configs (rd_kafka_t *rk, rd_kafka_queue_t *rkqu) {
+#define MY_CONFRES_CNT RD_KAFKA_RESOURCE__CNT + 2
+ rd_kafka_ConfigResource_t *configs[MY_CONFRES_CNT];
+ rd_kafka_AdminOptions_t *options;
+ rd_kafka_event_t *rkev;
+ rd_kafka_resp_err_t err;
+ const rd_kafka_AlterConfigs_result_t *res;
+ const rd_kafka_ConfigResource_t **rconfigs;
+ size_t rconfig_cnt;
+ char errstr[128];
+ int i;
+
+ /* Check invalids */
+ configs[0] = rd_kafka_ConfigResource_new(
+ (rd_kafka_ResourceType_t)-1, "something");
+ TEST_ASSERT(!configs[0]);
+
+ configs[0] = rd_kafka_ConfigResource_new(
+ (rd_kafka_ResourceType_t)0, NULL);
+ TEST_ASSERT(!configs[0]);
+
+
+ for (i = 0 ; i < MY_CONFRES_CNT ; i++) {
+ int set_config = !(i % 2);
+
+ /* librdkafka shall not limit the use of illogical
+ * or unknown settings, they are enforced by the broker. */
+ configs[i] = rd_kafka_ConfigResource_new(
+ (rd_kafka_ResourceType_t)i, "3");
+ TEST_ASSERT(configs[i] != NULL);
+
+ if (set_config) {
+ rd_kafka_ConfigResource_set_config(configs[i],
+ "some.conf",
+ "which remains "
+ "unchecked");
+ rd_kafka_ConfigResource_set_config(configs[i],
+ "some.conf.null",
+ NULL);
+ }
+ }
+
+
+ options = rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_ANY);
+ err = rd_kafka_AdminOptions_set_request_timeout(options, 1000, errstr,
+ sizeof(errstr));
+ TEST_ASSERT(!err, "%s", errstr);
+
+ /* AlterConfigs */
+ rd_kafka_AlterConfigs(rk, configs, MY_CONFRES_CNT,
+ options, rkqu);
+
+ rkev = test_wait_admin_result(rkqu, RD_KAFKA_EVENT_ALTERCONFIGS_RESULT,
+ 2000);
+
+ TEST_ASSERT(rd_kafka_event_error(rkev) == RD_KAFKA_RESP_ERR__TIMED_OUT,
+ "Expected timeout, not %s",
+ rd_kafka_event_error_string(rkev));
+
+ res = rd_kafka_event_AlterConfigs_result(rkev);
+ TEST_ASSERT(res);
+
+ rconfigs = rd_kafka_AlterConfigs_result_resources(res, &rconfig_cnt);
+ TEST_ASSERT(!rconfigs && !rconfig_cnt,
+ "Expected no result resources, got %"PRIusz,
+ rconfig_cnt);
+
+ rd_kafka_event_destroy(rkev);
+
+ /* DescribeConfigs: reuse same configs and options */
+ rd_kafka_DescribeConfigs(rk, configs, MY_CONFRES_CNT,
+ options, rkqu);
+
+ rd_kafka_AdminOptions_destroy(options);
+ rd_kafka_ConfigResource_destroy_array(configs, MY_CONFRES_CNT);
+
+ rkev = test_wait_admin_result(rkqu,
+ RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT,
+ 2000);
+
+ TEST_ASSERT(rd_kafka_event_error(rkev) == RD_KAFKA_RESP_ERR__TIMED_OUT,
+ "Expected timeout, not %s",
+ rd_kafka_event_error_string(rkev));
+
+ res = rd_kafka_event_DescribeConfigs_result(rkev);
+ TEST_ASSERT(res);
+
+ rconfigs = rd_kafka_DescribeConfigs_result_resources(res, &rconfig_cnt);
+ TEST_ASSERT(!rconfigs && !rconfig_cnt,
+ "Expected no result resources, got %"PRIusz,
+ rconfig_cnt);
+
+ rd_kafka_event_destroy(rkev);
+}
+
+
+/**
+ * @brief Verify that an unclean rd_kafka_destroy() does not hang.
+ */
+static void do_test_unclean_destroy (rd_kafka_type_t cltype, int with_mainq) {
+ rd_kafka_t *rk;
+ char errstr[512];
+ rd_kafka_conf_t *conf;
+ rd_kafka_queue_t *q;
+ rd_kafka_event_t *rkev;
+ rd_kafka_DeleteTopic_t *topic;
+ test_timing_t t_destroy;
+
+ test_conf_init(&conf, NULL, 0);
+ /* Remove brokers, if any, since this is a local test and we
+ * rely on the controller not being found. */
+ test_conf_set(conf, "bootstrap.servers", "");
+ test_conf_set(conf, "socket.timeout.ms", "60000");
+
+ rk = rd_kafka_new(cltype, conf, errstr, sizeof(errstr));
+ TEST_ASSERT(rk, "kafka_new(%d): %s", cltype, errstr);
+
+ TEST_SAY(_C_MAG "[ Test unclean destroy for %s using %s]\n", rd_kafka_name(rk),
+ with_mainq ? "mainq" : "tempq");
+
+ if (with_mainq)
+ q = rd_kafka_queue_get_main(rk);
+ else
+ q = rd_kafka_queue_new(rk);
+
+ topic = rd_kafka_DeleteTopic_new("test");
+ rd_kafka_DeleteTopics(rk, &topic, 1, NULL, q);
+ rd_kafka_DeleteTopic_destroy(topic);
+
+ /* We're not expecting a result yet since DeleteTopics will attempt
+ * to look up the controller for socket.timeout.ms (1 minute). */
+ rkev = rd_kafka_queue_poll(q, 100);
+ TEST_ASSERT(!rkev, "Did not expect result: %s", rd_kafka_event_name(rkev));
+
+ rd_kafka_queue_destroy(q);
+
+ TEST_SAY("Giving rd_kafka_destroy() 5s to finish, "
+ "despite Admin API request being processed\n");
+ test_timeout_set(5);
+ TIMING_START(&t_destroy, "rd_kafka_destroy()");
+ rd_kafka_destroy(rk);
+ TIMING_STOP(&t_destroy);
+
+ /* Restore timeout */
+ test_timeout_set(60);
+}
+
+
+/**
+ * @brief Test AdminOptions
+ */
+static void do_test_options (rd_kafka_t *rk) {
+#define _all_apis { RD_KAFKA_ADMIN_OP_CREATETOPICS, \
+ RD_KAFKA_ADMIN_OP_DELETETOPICS, \
+ RD_KAFKA_ADMIN_OP_CREATEPARTITIONS, \
+ RD_KAFKA_ADMIN_OP_ALTERCONFIGS, \
+ RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS, \
+ RD_KAFKA_ADMIN_OP_ANY /* Must be last */}
+ struct {
+ const char *setter;
+ const rd_kafka_admin_op_t valid_apis[8];
+ } matrix[] = {
+ { "request_timeout", _all_apis },
+ { "operation_timeout", { RD_KAFKA_ADMIN_OP_CREATETOPICS,
+ RD_KAFKA_ADMIN_OP_DELETETOPICS,
+ RD_KAFKA_ADMIN_OP_CREATEPARTITIONS } },
+ { "validate_only", { RD_KAFKA_ADMIN_OP_CREATETOPICS,
+ RD_KAFKA_ADMIN_OP_CREATEPARTITIONS,
+ RD_KAFKA_ADMIN_OP_ALTERCONFIGS } },
+ { "broker", _all_apis },
+ { "opaque", _all_apis },
+ { NULL },
+ };
+ int i;
+ rd_kafka_AdminOptions_t *options;
+
+
+ for (i = 0 ; matrix[i].setter ; i++) {
+ static const rd_kafka_admin_op_t all_apis[] = _all_apis;
+ const rd_kafka_admin_op_t *for_api;
+
+ for (for_api = all_apis ; ; for_api++) {
+ rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ rd_kafka_resp_err_t exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ char errstr[512];
+ int fi;
+
+ options = rd_kafka_AdminOptions_new(rk, *for_api);
+ TEST_ASSERT(options,
+ "AdminOptions_new(%d) failed", *for_api);
+
+ if (!strcmp(matrix[i].setter, "request_timeout"))
+ err = rd_kafka_AdminOptions_set_request_timeout(
+ options, 1234, errstr, sizeof(errstr));
+ else if (!strcmp(matrix[i].setter, "operation_timeout"))
+ err = rd_kafka_AdminOptions_set_operation_timeout(
+ options, 12345, errstr, sizeof(errstr));
+ else if (!strcmp(matrix[i].setter, "validate_only"))
+ err = rd_kafka_AdminOptions_set_validate_only(
+ options, 1, errstr, sizeof(errstr));
+ else if (!strcmp(matrix[i].setter, "broker"))
+ err = rd_kafka_AdminOptions_set_broker(
+ options, 5, errstr, sizeof(errstr));
+ else if (!strcmp(matrix[i].setter, "opaque")) {
+ rd_kafka_AdminOptions_set_opaque(
+ options, (void *)options);
+ err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ } else
+ TEST_FAIL("Invalid setter: %s",
+ matrix[i].setter);
+
+
+ TEST_SAYL(3, "AdminOptions_set_%s on "
+ "RD_KAFKA_ADMIN_OP_%d options "
+ "returned %s: %s\n",
+ matrix[i].setter,
+ *for_api,
+ rd_kafka_err2name(err),
+ err ? errstr : "success");
+
+ /* Scan matrix valid_apis to see if this
+ * setter should be accepted or not. */
+ if (exp_err) {
+ /* An expected error is already set */
+ } else if (*for_api != RD_KAFKA_ADMIN_OP_ANY) {
+ exp_err = RD_KAFKA_RESP_ERR__INVALID_ARG;
+
+ for (fi = 0 ; matrix[i].valid_apis[fi] ; fi++) {
+ if (matrix[i].valid_apis[fi] ==
+ *for_api)
+ exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+ } else {
+ exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+
+ if (err != exp_err)
+ TEST_FAIL_LATER("Expected AdminOptions_set_%s "
+ "for RD_KAFKA_ADMIN_OP_%d "
+ "options to return %s, "
+ "not %s",
+ matrix[i].setter,
+ *for_api,
+ rd_kafka_err2name(exp_err),
+ rd_kafka_err2name(err));
+
+ rd_kafka_AdminOptions_destroy(options);
+
+ if (*for_api == RD_KAFKA_ADMIN_OP_ANY)
+ break; /* This was the last one */
+ }
+ }
+
+ /* Try an invalid for_api */
+ options = rd_kafka_AdminOptions_new(rk, (rd_kafka_admin_op_t)1234);
+ TEST_ASSERT(!options, "Expectred AdminOptions_new() to fail "
+ "with an invalid for_api, didn't.");
+
+ TEST_LATER_CHECK();
+}
+
+
+static void do_test_apis (rd_kafka_type_t cltype) {
+ rd_kafka_t *rk;
+ char errstr[512];
+ rd_kafka_queue_t *mainq, *backgroundq;
+ rd_kafka_conf_t *conf;
+
+ mtx_init(&last_event_lock, mtx_plain);
+ cnd_init(&last_event_cnd);
+
+ do_test_unclean_destroy(cltype, 0/*tempq*/);
+ do_test_unclean_destroy(cltype, 1/*mainq*/);
+
+ test_conf_init(&conf, NULL, 0);
+ /* Remove brokers, if any, since this is a local test and we
+ * rely on the controller not being found. */
+ test_conf_set(conf, "bootstrap.servers", "");
+ test_conf_set(conf, "socket.timeout.ms", MY_SOCKET_TIMEOUT_MS_STR);
+ /* For use with the background queue */
+ rd_kafka_conf_set_background_event_cb(conf, background_event_cb);
+
+ rk = rd_kafka_new(cltype, conf, errstr, sizeof(errstr));
+ TEST_ASSERT(rk, "kafka_new(%d): %s", cltype, errstr);
+
+ mainq = rd_kafka_queue_get_main(rk);
+ backgroundq = rd_kafka_queue_get_background(rk);
+
+ do_test_options(rk);
+
+ do_test_CreateTopics("temp queue, no options", rk, NULL, 0, 0);
+ do_test_CreateTopics("temp queue, no options, background_event_cb",
+ rk, backgroundq, 1, 0);
+ do_test_CreateTopics("temp queue, options", rk, NULL, 0, 1);
+ do_test_CreateTopics("main queue, options", rk, mainq, 0, 1);
+
+ do_test_DeleteTopics("temp queue, no options", rk, NULL, 0);
+ do_test_DeleteTopics("temp queue, options", rk, NULL, 1);
+ do_test_DeleteTopics("main queue, options", rk, mainq, 1);
+
+ do_test_mix(rk, mainq);
+
+ do_test_configs(rk, mainq);
+
+ rd_kafka_queue_destroy(backgroundq);
+ rd_kafka_queue_destroy(mainq);
+
+ rd_kafka_destroy(rk);
+
+ mtx_destroy(&last_event_lock);
+ cnd_destroy(&last_event_cnd);
+
+}
+
+
+int main_0080_admin_ut (int argc, char **argv) {
+ do_test_apis(RD_KAFKA_PRODUCER);
+ do_test_apis(RD_KAFKA_CONSUMER);
+ return 0;
+}
diff -Nru librdkafka-0.11.3/tests/0081-admin.c librdkafka-0.11.6/tests/0081-admin.c
--- librdkafka-0.11.3/tests/0081-admin.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0081-admin.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,1175 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+#include "rdkafka.h"
+
+/**
+ * @brief Admin API integration tests.
+ */
+
+
+static int32_t *avail_brokers;
+static size_t avail_broker_cnt;
+
+
+
+
+static void do_test_CreateTopics (const char *what,
+ rd_kafka_t *rk, rd_kafka_queue_t *useq,
+ int op_timeout, rd_bool_t validate_only) {
+ rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
+#define MY_NEW_TOPICS_CNT 6
+ char *topics[MY_NEW_TOPICS_CNT];
+ rd_kafka_NewTopic_t *new_topics[MY_NEW_TOPICS_CNT];
+ rd_kafka_AdminOptions_t *options = NULL;
+ rd_kafka_resp_err_t exp_topicerr[MY_NEW_TOPICS_CNT] = {0};
+ rd_kafka_resp_err_t exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ /* Expected topics in metadata */
+ rd_kafka_metadata_topic_t exp_mdtopics[MY_NEW_TOPICS_CNT] = {{0}};
+ int exp_mdtopic_cnt = 0;
+ /* Not expected topics in metadata */
+ rd_kafka_metadata_topic_t exp_not_mdtopics[MY_NEW_TOPICS_CNT] = {{0}};
+ int exp_not_mdtopic_cnt = 0;
+ int i;
+ char errstr[512];
+ const char *errstr2;
+ rd_kafka_resp_err_t err;
+ test_timing_t timing;
+ rd_kafka_event_t *rkev;
+ const rd_kafka_CreateTopics_result_t *res;
+ const rd_kafka_topic_result_t **restopics;
+ size_t restopic_cnt;
+ int metadata_tmout ;
+ int num_replicas = (int)avail_broker_cnt;
+ int32_t *replicas;
+
+ /* Set up replicas */
+ replicas = rd_alloca(sizeof(*replicas) * num_replicas);
+ for (i = 0 ; i < num_replicas ; i++)
+ replicas[i] = avail_brokers[i];
+
+ TEST_SAY(_C_MAG "[ %s CreateTopics with %s, "
+ "op_timeout %d, validate_only %d ]\n",
+ rd_kafka_name(rk), what, op_timeout, validate_only);
+
+ /**
+ * Construct NewTopic array with different properties for
+ * different partitions.
+ */
+ for (i = 0 ; i < MY_NEW_TOPICS_CNT ; i++) {
+ char *topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
+ int num_parts = i * 7 + 1;
+ int set_config = (i & 1);
+ int add_invalid_config = (i == 1);
+ int set_replicas = !(i % 3);
+ rd_kafka_resp_err_t this_exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ topics[i] = topic;
+ new_topics[i] = rd_kafka_NewTopic_new(topic,
+ num_parts,
+ set_replicas ? -1 :
+ num_replicas,
+ NULL, 0);
+
+ if (set_config) {
+ /*
+ * Add various configuration properties
+ */
+ err = rd_kafka_NewTopic_set_config(
+ new_topics[i], "compression.type", "lz4");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ err = rd_kafka_NewTopic_set_config(
+ new_topics[i], "delete.retention.ms", "900");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+ }
+
+ if (add_invalid_config) {
+ /* Add invalid config property */
+ err = rd_kafka_NewTopic_set_config(
+ new_topics[i],
+ "dummy.doesntexist",
+ "broker is verifying this");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+ this_exp_err = RD_KAFKA_RESP_ERR_INVALID_CONFIG;
+ }
+
+ TEST_SAY("Expected result for topic #%d: %s "
+ "(set_config=%d, add_invalid_config=%d, "
+ "set_replicas=%d)\n",
+ i, rd_kafka_err2name(this_exp_err),
+ set_config, add_invalid_config, set_replicas);
+
+ if (set_replicas) {
+ int32_t p;
+
+ /*
+ * Set valid replica assignments
+ */
+ for (p = 0 ; p < num_parts ; p++) {
+ err = rd_kafka_NewTopic_set_replica_assignment(
+ new_topics[i], p,
+ replicas, num_replicas,
+ errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "%s", errstr);
+ }
+ }
+
+ if (this_exp_err || validate_only) {
+ exp_topicerr[i] = this_exp_err;
+ exp_not_mdtopics[exp_not_mdtopic_cnt++].topic = topic;
+
+ } else {
+ exp_mdtopics[exp_mdtopic_cnt].topic = topic;
+ exp_mdtopics[exp_mdtopic_cnt].partition_cnt =
+ num_parts;
+ exp_mdtopic_cnt++;
+ }
+ }
+
+ if (op_timeout != -1 || validate_only) {
+ options = rd_kafka_AdminOptions_new(
+ rk, RD_KAFKA_ADMIN_OP_CREATETOPICS);
+
+ if (op_timeout != -1) {
+ err = rd_kafka_AdminOptions_set_operation_timeout(
+ options, op_timeout, errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+ }
+
+ if (validate_only) {
+ err = rd_kafka_AdminOptions_set_validate_only(
+ options, validate_only, errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+ }
+ }
+
+ TIMING_START(&timing, "CreateTopics");
+ TEST_SAY("Call CreateTopics\n");
+ rd_kafka_CreateTopics(rk, new_topics, MY_NEW_TOPICS_CNT,
+ options, q);
+ TIMING_ASSERT_LATER(&timing, 0, 50);
+
+ /* Poll result queue for CreateTopics result.
+ * Print but otherwise ignore other event types
+ * (typically generic Error events). */
+ TIMING_START(&timing, "CreateTopics.queue_poll");
+ do {
+ rkev = rd_kafka_queue_poll(q, tmout_multip(20*1000));
+ TEST_SAY("CreateTopics: got %s in %.3fms\n",
+ rd_kafka_event_name(rkev),
+ TIMING_DURATION(&timing) / 1000.0f);
+ if (rd_kafka_event_error(rkev))
+ TEST_SAY("%s: %s\n",
+ rd_kafka_event_name(rkev),
+ rd_kafka_event_error_string(rkev));
+ } while (rd_kafka_event_type(rkev) !=
+ RD_KAFKA_EVENT_CREATETOPICS_RESULT);
+
+ /* Convert event to proper result */
+ res = rd_kafka_event_CreateTopics_result(rkev);
+ TEST_ASSERT(res, "expected CreateTopics_result, not %s",
+ rd_kafka_event_name(rkev));
+
+ /* Expecting error */
+ err = rd_kafka_event_error(rkev);
+ errstr2 = rd_kafka_event_error_string(rkev);
+ TEST_ASSERT(err == exp_err,
+ "expected CreateTopics to return %s, not %s (%s)",
+ rd_kafka_err2str(exp_err),
+ rd_kafka_err2str(err),
+ err ? errstr2 : "n/a");
+
+ TEST_SAY("CreateTopics: returned %s (%s)\n",
+ rd_kafka_err2str(err), err ? errstr2 : "n/a");
+
+ /* Extract topics */
+ restopics = rd_kafka_CreateTopics_result_topics(res, &restopic_cnt);
+
+
+ /* Scan topics for proper fields and expected failures. */
+ for (i = 0 ; i < (int)restopic_cnt ; i++) {
+ const rd_kafka_topic_result_t *terr = restopics[i];
+
+ /* Verify that topic order matches our request. */
+ if (strcmp(rd_kafka_topic_result_name(terr), topics[i]))
+ TEST_FAIL_LATER("Topic result order mismatch at #%d: "
+ "expected %s, got %s",
+ i, topics[i],
+ rd_kafka_topic_result_name(terr));
+
+ TEST_SAY("CreateTopics result: #%d: %s: %s: %s\n",
+ i,
+ rd_kafka_topic_result_name(terr),
+ rd_kafka_err2name(rd_kafka_topic_result_error(terr)),
+ rd_kafka_topic_result_error_string(terr));
+ if (rd_kafka_topic_result_error(terr) != exp_topicerr[i])
+ TEST_FAIL_LATER(
+ "Expected %s, not %d: %s",
+ rd_kafka_err2name(exp_topicerr[i]),
+ rd_kafka_topic_result_error(terr),
+ rd_kafka_err2name(rd_kafka_topic_result_error(
+ terr)));
+ }
+
+ /**
+ * Verify that the expecteded topics are created and the non-expected
+ * are not. Allow it some time to propagate.
+ */
+ if (validate_only) {
+ /* No topics should have been created, give it some time
+ * before checking. */
+ rd_sleep(2);
+ metadata_tmout = 5 * 1000;
+ } else {
+ if (op_timeout > 0)
+ metadata_tmout = op_timeout + 1000;
+ else
+ metadata_tmout = 10 * 1000;
+ }
+
+ test_wait_metadata_update(rk,
+ exp_mdtopics,
+ exp_mdtopic_cnt,
+ exp_not_mdtopics,
+ exp_not_mdtopic_cnt,
+ metadata_tmout);
+
+ rd_kafka_event_destroy(rkev);
+
+ for (i = 0 ; i < MY_NEW_TOPICS_CNT ; i++) {
+ rd_kafka_NewTopic_destroy(new_topics[i]);
+ rd_free(topics[i]);
+ }
+
+ if (options)
+ rd_kafka_AdminOptions_destroy(options);
+
+ if (!useq)
+ rd_kafka_queue_destroy(q);
+
+#undef MY_NEW_TOPICS_CNT
+}
+
+
+
+
+/**
+ * @brief Test deletion of topics
+ *
+ *
+ */
+static void do_test_DeleteTopics (const char *what,
+ rd_kafka_t *rk, rd_kafka_queue_t *useq,
+ int op_timeout) {
+ rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
+ const int skip_topic_cnt = 2;
+#define MY_DEL_TOPICS_CNT 9
+ char *topics[MY_DEL_TOPICS_CNT];
+ rd_kafka_DeleteTopic_t *del_topics[MY_DEL_TOPICS_CNT];
+ rd_kafka_AdminOptions_t *options = NULL;
+ rd_kafka_resp_err_t exp_topicerr[MY_DEL_TOPICS_CNT] = {0};
+ rd_kafka_resp_err_t exp_err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ /* Expected topics in metadata */
+ rd_kafka_metadata_topic_t exp_mdtopics[MY_DEL_TOPICS_CNT] = {{0}};
+ int exp_mdtopic_cnt = 0;
+ /* Not expected topics in metadata */
+ rd_kafka_metadata_topic_t exp_not_mdtopics[MY_DEL_TOPICS_CNT] = {{0}};
+ int exp_not_mdtopic_cnt = 0;
+ int i;
+ char errstr[512];
+ const char *errstr2;
+ rd_kafka_resp_err_t err;
+ test_timing_t timing;
+ rd_kafka_event_t *rkev;
+ const rd_kafka_DeleteTopics_result_t *res;
+ const rd_kafka_topic_result_t **restopics;
+ size_t restopic_cnt;
+ int metadata_tmout;
+
+ TEST_SAY(_C_MAG "[ %s DeleteTopics with %s, op_timeout %d ]\n",
+ rd_kafka_name(rk), what, op_timeout);
+
+ /**
+ * Construct DeleteTopic array
+ */
+ for (i = 0 ; i < MY_DEL_TOPICS_CNT ; i++) {
+ char *topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
+ int notexist_topic = i >= MY_DEL_TOPICS_CNT - skip_topic_cnt;
+
+ topics[i] = topic;
+
+ del_topics[i] = rd_kafka_DeleteTopic_new(topic);
+
+ if (notexist_topic)
+ exp_topicerr[i] =
+ RD_KAFKA_RESP_ERR_UNKNOWN_TOPIC_OR_PART;
+ else {
+ exp_topicerr[i] =
+ RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ exp_mdtopics[exp_mdtopic_cnt++].topic = topic;
+ }
+
+ exp_not_mdtopics[exp_not_mdtopic_cnt++].topic = topic;
+ }
+
+ if (op_timeout != -1) {
+ options = rd_kafka_AdminOptions_new(
+ rk, RD_KAFKA_ADMIN_OP_ANY);
+
+ err = rd_kafka_AdminOptions_set_operation_timeout(
+ options, op_timeout, errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+ }
+
+
+ /* Create the topics first, minus the skip count. */
+ test_CreateTopics_simple(rk, NULL, topics,
+ MY_DEL_TOPICS_CNT-skip_topic_cnt,
+ 2/*num_partitions*/,
+ NULL);
+
+ /* Verify that topics are reported by metadata */
+ test_wait_metadata_update(rk,
+ exp_mdtopics, exp_mdtopic_cnt,
+ NULL, 0,
+ 15*1000);
+
+ TIMING_START(&timing, "DeleteTopics");
+ TEST_SAY("Call DeleteTopics\n");
+ rd_kafka_DeleteTopics(rk, del_topics, MY_DEL_TOPICS_CNT,
+ options, q);
+ TIMING_ASSERT_LATER(&timing, 0, 50);
+
+ /* Poll result queue for DeleteTopics result.
+ * Print but otherwise ignore other event types
+ * (typically generic Error events). */
+ TIMING_START(&timing, "DeleteTopics.queue_poll");
+ while (1) {
+ rkev = rd_kafka_queue_poll(q, tmout_multip(20*1000));
+ TEST_SAY("DeleteTopics: got %s in %.3fms\n",
+ rd_kafka_event_name(rkev),
+ TIMING_DURATION(&timing) / 1000.0f);
+ if (rd_kafka_event_error(rkev))
+ TEST_SAY("%s: %s\n",
+ rd_kafka_event_name(rkev),
+ rd_kafka_event_error_string(rkev));
+
+ if (rd_kafka_event_type(rkev) ==
+ RD_KAFKA_EVENT_DELETETOPICS_RESULT)
+ break;
+
+ rd_kafka_event_destroy(rkev);
+ }
+
+ /* Convert event to proper result */
+ res = rd_kafka_event_DeleteTopics_result(rkev);
+ TEST_ASSERT(res, "expected DeleteTopics_result, not %s",
+ rd_kafka_event_name(rkev));
+
+ /* Expecting error */
+ err = rd_kafka_event_error(rkev);
+ errstr2 = rd_kafka_event_error_string(rkev);
+ TEST_ASSERT(err == exp_err,
+ "expected DeleteTopics to return %s, not %s (%s)",
+ rd_kafka_err2str(exp_err),
+ rd_kafka_err2str(err),
+ err ? errstr2 : "n/a");
+
+ TEST_SAY("DeleteTopics: returned %s (%s)\n",
+ rd_kafka_err2str(err), err ? errstr2 : "n/a");
+
+ /* Extract topics */
+ restopics = rd_kafka_DeleteTopics_result_topics(res, &restopic_cnt);
+
+
+ /* Scan topics for proper fields and expected failures. */
+ for (i = 0 ; i < (int)restopic_cnt ; i++) {
+ const rd_kafka_topic_result_t *terr = restopics[i];
+
+ /* Verify that topic order matches our request. */
+ if (strcmp(rd_kafka_topic_result_name(terr), topics[i]))
+ TEST_FAIL_LATER("Topic result order mismatch at #%d: "
+ "expected %s, got %s",
+ i, topics[i],
+ rd_kafka_topic_result_name(terr));
+
+ TEST_SAY("DeleteTopics result: #%d: %s: %s: %s\n",
+ i,
+ rd_kafka_topic_result_name(terr),
+ rd_kafka_err2name(rd_kafka_topic_result_error(terr)),
+ rd_kafka_topic_result_error_string(terr));
+ if (rd_kafka_topic_result_error(terr) != exp_topicerr[i])
+ TEST_FAIL_LATER(
+ "Expected %s, not %d: %s",
+ rd_kafka_err2name(exp_topicerr[i]),
+ rd_kafka_topic_result_error(terr),
+ rd_kafka_err2name(rd_kafka_topic_result_error(
+ terr)));
+ }
+
+ /**
+ * Verify that the expected topics are deleted and the non-expected
+ * are not. Allow it some time to propagate.
+ */
+ if (op_timeout > 0)
+ metadata_tmout = op_timeout + 1000;
+ else
+ metadata_tmout = 10 * 1000;
+
+ test_wait_metadata_update(rk,
+ NULL, 0,
+ exp_not_mdtopics,
+ exp_not_mdtopic_cnt,
+ metadata_tmout);
+
+ rd_kafka_event_destroy(rkev);
+
+ for (i = 0 ; i < MY_DEL_TOPICS_CNT ; i++) {
+ rd_kafka_DeleteTopic_destroy(del_topics[i]);
+ rd_free(topics[i]);
+ }
+
+ if (options)
+ rd_kafka_AdminOptions_destroy(options);
+
+ if (!useq)
+ rd_kafka_queue_destroy(q);
+
+#undef MY_DEL_TOPICS_CNT
+}
+
+
+
+/**
+ * @brief Test creation of partitions
+ *
+ *
+ */
+static void do_test_CreatePartitions (const char *what,
+ rd_kafka_t *rk, rd_kafka_queue_t *useq,
+ int op_timeout) {
+ rd_kafka_queue_t *q = useq ? useq : rd_kafka_queue_new(rk);
+#define MY_CRP_TOPICS_CNT 9
+ char *topics[MY_CRP_TOPICS_CNT];
+ rd_kafka_NewTopic_t *new_topics[MY_CRP_TOPICS_CNT];
+ rd_kafka_NewPartitions_t *crp_topics[MY_CRP_TOPICS_CNT];
+ rd_kafka_AdminOptions_t *options = NULL;
+ /* Expected topics in metadata */
+ rd_kafka_metadata_topic_t exp_mdtopics[MY_CRP_TOPICS_CNT] = {{0}};
+ rd_kafka_metadata_partition_t exp_mdparts[2] = {{0}};
+ int exp_mdtopic_cnt = 0;
+ int i;
+ char errstr[512];
+ rd_kafka_resp_err_t err;
+ test_timing_t timing;
+ int metadata_tmout;
+ int num_replicas = (int)avail_broker_cnt;
+
+ TEST_SAY(_C_MAG "[ %s CreatePartitions with %s, op_timeout %d ]\n",
+ rd_kafka_name(rk), what, op_timeout);
+
+ /* Set up two expected partitions with different replication sets
+ * so they can be matched by the metadata checker later.
+ * Even partitions use exp_mdparts[0] while odd partitions
+ * use exp_mdparts[1]. */
+
+ /* Set valid replica assignments (even, and odd (reverse) ) */
+ exp_mdparts[0].replicas = rd_alloca(sizeof(*exp_mdparts[0].replicas) *
+ num_replicas);
+ exp_mdparts[1].replicas = rd_alloca(sizeof(*exp_mdparts[1].replicas) *
+ num_replicas);
+ exp_mdparts[0].replica_cnt = num_replicas;
+ exp_mdparts[1].replica_cnt = num_replicas;
+ for (i = 0 ; i < num_replicas ; i++) {
+ exp_mdparts[0].replicas[i] = avail_brokers[i];
+ exp_mdparts[1].replicas[i] = avail_brokers[num_replicas-i-1];
+ }
+
+ /**
+ * Construct CreatePartitions array
+ */
+ for (i = 0 ; i < MY_CRP_TOPICS_CNT ; i++) {
+ char *topic = rd_strdup(test_mk_topic_name(__FUNCTION__, 1));
+ int initial_part_cnt = 1 + (i * 2);
+ int new_part_cnt = 1 + (i / 2);
+ int final_part_cnt = initial_part_cnt + new_part_cnt;
+ int set_replicas = !(i % 2);
+ int pi;
+
+ topics[i] = topic;
+
+ /* Topic to create with initial partition count */
+ new_topics[i] = rd_kafka_NewTopic_new(topic, initial_part_cnt,
+ set_replicas ?
+ -1 : num_replicas,
+ NULL, 0);
+
+ /* .. and later add more partitions to */
+ crp_topics[i] = rd_kafka_NewPartitions_new(topic,
+ final_part_cnt,
+ errstr,
+ sizeof(errstr));
+
+ if (set_replicas) {
+ exp_mdtopics[exp_mdtopic_cnt].partitions =
+ rd_alloca(final_part_cnt *
+ sizeof(*exp_mdtopics[exp_mdtopic_cnt].
+ partitions));
+
+ for (pi = 0 ; pi < final_part_cnt ; pi++) {
+ const rd_kafka_metadata_partition_t *exp_mdp =
+ &exp_mdparts[pi & 1];
+
+ exp_mdtopics[exp_mdtopic_cnt].
+ partitions[pi] = *exp_mdp; /* copy */
+
+ exp_mdtopics[exp_mdtopic_cnt].
+ partitions[pi].id = pi;
+
+ if (pi < initial_part_cnt) {
+ /* Set replica assignment
+ * for initial partitions */
+ err = rd_kafka_NewTopic_set_replica_assignment(
+ new_topics[i], pi,
+ exp_mdp->replicas,
+ (size_t)exp_mdp->replica_cnt,
+ errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "NewTopic_set_replica_assignment: %s",
+ errstr);
+ } else {
+ /* Set replica assignment for new
+ * partitions */
+ err = rd_kafka_NewPartitions_set_replica_assignment(
+ crp_topics[i],
+ pi - initial_part_cnt,
+ exp_mdp->replicas,
+ (size_t)exp_mdp->replica_cnt,
+ errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "NewPartitions_set_replica_assignment: %s",
+ errstr);
+ }
+
+ }
+ }
+
+ TEST_SAY(_C_YEL "Topic %s with %d initial partitions will grow "
+ "by %d to %d total partitions with%s replicas set\n",
+ topics[i],
+ initial_part_cnt, new_part_cnt, final_part_cnt,
+ set_replicas ? "" : "out");
+
+ exp_mdtopics[exp_mdtopic_cnt].topic = topic;
+ exp_mdtopics[exp_mdtopic_cnt].partition_cnt = final_part_cnt;
+
+ exp_mdtopic_cnt++;
+ }
+
+ if (op_timeout != -1) {
+ options = rd_kafka_AdminOptions_new(
+ rk, RD_KAFKA_ADMIN_OP_ANY);
+
+ err = rd_kafka_AdminOptions_set_operation_timeout(
+ options, op_timeout, errstr, sizeof(errstr));
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+ }
+
+ /*
+ * Create topics with initial partition count
+ */
+ TIMING_START(&timing, "CreateTopics");
+ TEST_SAY("Creating topics with initial partition counts\n");
+ rd_kafka_CreateTopics(rk, new_topics, MY_CRP_TOPICS_CNT,
+ options, q);
+ TIMING_ASSERT_LATER(&timing, 0, 50);
+
+ err = test_wait_topic_admin_result(q,
+ RD_KAFKA_EVENT_CREATETOPICS_RESULT,
+ 15000);
+ TEST_ASSERT(!err, "CreateTopics failed: %s", rd_kafka_err2str(err));
+
+ rd_kafka_NewTopic_destroy_array(new_topics, MY_CRP_TOPICS_CNT);
+
+
+ /*
+ * Create new partitions
+ */
+ TIMING_START(&timing, "CreatePartitions");
+ TEST_SAY("Creating partitions\n");
+ rd_kafka_CreatePartitions(rk, crp_topics, MY_CRP_TOPICS_CNT,
+ options, q);
+ TIMING_ASSERT_LATER(&timing, 0, 50);
+
+ err = test_wait_topic_admin_result(q,
+ RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT,
+ 15000);
+ TEST_ASSERT(!err, "CreatePartitions failed: %s", rd_kafka_err2str(err));
+
+ rd_kafka_NewPartitions_destroy_array(crp_topics, MY_CRP_TOPICS_CNT);
+
+
+ /**
+ * Verify that the expected topics are deleted and the non-expected
+ * are not. Allow it some time to propagate.
+ */
+ if (op_timeout > 0)
+ metadata_tmout = op_timeout + 1000;
+ else
+ metadata_tmout = 10 * 1000;
+
+ test_wait_metadata_update(rk,
+ exp_mdtopics,
+ exp_mdtopic_cnt,
+ NULL, 0,
+ metadata_tmout);
+
+ for (i = 0 ; i < MY_CRP_TOPICS_CNT ; i++)
+ rd_free(topics[i]);
+
+ if (options)
+ rd_kafka_AdminOptions_destroy(options);
+
+ if (!useq)
+ rd_kafka_queue_destroy(q);
+
+#undef MY_CRP_TOPICS_CNT
+}
+
+
+
+/**
+ * @brief Print the ConfigEntrys in the provided array.
+ */
+static void
+test_print_ConfigEntry_array (const rd_kafka_ConfigEntry_t **entries,
+ size_t entry_cnt, unsigned int depth) {
+ const char *indent = &" "[4 - (depth > 4 ? 4 : depth)];
+ size_t ei;
+
+ for (ei = 0 ; ei < entry_cnt ; ei++) {
+ const rd_kafka_ConfigEntry_t *e = entries[ei];
+ const rd_kafka_ConfigEntry_t **syns;
+ size_t syn_cnt;
+
+ syns = rd_kafka_ConfigEntry_synonyms(e, &syn_cnt);
+
+#define YN(v) ((v) ? "y" : "n")
+ TEST_SAY("%s#%"PRIusz"/%"PRIusz
+ ": Source %s (%d): \"%s\"=\"%s\" "
+ "[is read-only=%s, default=%s, sensitive=%s, "
+ "synonym=%s] with %"PRIusz" synonym(s)\n",
+ indent,
+ ei, entry_cnt,
+ rd_kafka_ConfigSource_name(
+ rd_kafka_ConfigEntry_source(e)),
+ rd_kafka_ConfigEntry_source(e),
+ rd_kafka_ConfigEntry_name(e),
+ rd_kafka_ConfigEntry_value(e) ?
+ rd_kafka_ConfigEntry_value(e) : "(NULL)",
+ YN(rd_kafka_ConfigEntry_is_read_only(e)),
+ YN(rd_kafka_ConfigEntry_is_default(e)),
+ YN(rd_kafka_ConfigEntry_is_sensitive(e)),
+ YN(rd_kafka_ConfigEntry_is_synonym(e)),
+ syn_cnt);
+#undef YN
+
+ if (syn_cnt > 0)
+ test_print_ConfigEntry_array(syns, syn_cnt, depth+1);
+ }
+}
+
+
+/**
+ * @brief Test AlterConfigs
+ */
+static void do_test_AlterConfigs (rd_kafka_t *rk, rd_kafka_queue_t *rkqu) {
+#define MY_CONFRES_CNT 3
+ char *topics[MY_CONFRES_CNT];
+ rd_kafka_ConfigResource_t *configs[MY_CONFRES_CNT];
+ rd_kafka_AdminOptions_t *options;
+ rd_kafka_resp_err_t exp_err[MY_CONFRES_CNT];
+ rd_kafka_event_t *rkev;
+ rd_kafka_resp_err_t err;
+ const rd_kafka_AlterConfigs_result_t *res;
+ const rd_kafka_ConfigResource_t **rconfigs;
+ size_t rconfig_cnt;
+ char errstr[128];
+ const char *errstr2;
+ int ci = 0;
+ int i;
+ int fails = 0;
+
+ /*
+ * Only create one topic, the others will be non-existent.
+ */
+ for (i = 0 ; i < MY_CONFRES_CNT ; i++)
+ rd_strdupa(&topics[i], test_mk_topic_name(__FUNCTION__, 1));
+
+ test_CreateTopics_simple(rk, NULL, topics, 1, 1, NULL);
+
+ /*
+ * ConfigResource #0: valid topic config
+ */
+ configs[ci] = rd_kafka_ConfigResource_new(
+ RD_KAFKA_RESOURCE_TOPIC, topics[ci]);
+
+ err = rd_kafka_ConfigResource_set_config(configs[ci],
+ "compression.type", "gzip");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ err = rd_kafka_ConfigResource_set_config(configs[ci],
+ "flush.ms", "12345678");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ exp_err[ci] = RD_KAFKA_RESP_ERR_NO_ERROR;
+ ci++;
+
+
+ if (test_broker_version >= TEST_BRKVER(1, 1, 0, 0)) {
+ /*
+ * ConfigResource #1: valid broker config
+ */
+ configs[ci] = rd_kafka_ConfigResource_new(
+ RD_KAFKA_RESOURCE_BROKER,
+ tsprintf("%"PRId32, avail_brokers[0]));
+
+ err = rd_kafka_ConfigResource_set_config(
+ configs[ci],
+ "sasl.kerberos.min.time.before.relogin", "58000");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ exp_err[ci] = RD_KAFKA_RESP_ERR_NO_ERROR;
+ ci++;
+ } else {
+ TEST_WARN("Skipping RESOURCE_BROKER test on unsupported "
+ "broker version\n");
+ }
+
+ /*
+ * ConfigResource #2: valid topic config, non-existent topic
+ */
+ configs[ci] = rd_kafka_ConfigResource_new(
+ RD_KAFKA_RESOURCE_TOPIC, topics[ci]);
+
+ err = rd_kafka_ConfigResource_set_config(configs[ci],
+ "compression.type", "lz4");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ err = rd_kafka_ConfigResource_set_config(configs[ci],
+ "offset.metadata.max.bytes",
+ "12345");
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+
+ exp_err[ci] = RD_KAFKA_RESP_ERR_UNKNOWN;
+ ci++;
+
+
+ /*
+ * Timeout options
+ */
+ options = rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_ALTERCONFIGS);
+ err = rd_kafka_AdminOptions_set_request_timeout(options, 10000, errstr,
+ sizeof(errstr));
+ TEST_ASSERT(!err, "%s", errstr);
+
+
+ /*
+ * Fire off request
+ */
+ rd_kafka_AlterConfigs(rk, configs, ci, options, rkqu);
+
+ rd_kafka_AdminOptions_destroy(options);
+
+ /*
+ * Wait for result
+ */
+ rkev = test_wait_admin_result(rkqu, RD_KAFKA_EVENT_ALTERCONFIGS_RESULT,
+ 10000+1000);
+
+ /*
+ * Extract result
+ */
+ res = rd_kafka_event_AlterConfigs_result(rkev);
+ TEST_ASSERT(res, "Expected AlterConfigs result, not %s",
+ rd_kafka_event_name(rkev));
+
+ err = rd_kafka_event_error(rkev);
+ errstr2 = rd_kafka_event_error_string(rkev);
+ TEST_ASSERT(!err,
+ "Expected success, not %s: %s",
+ rd_kafka_err2name(err), errstr2);
+
+ rconfigs = rd_kafka_AlterConfigs_result_resources(res, &rconfig_cnt);
+ TEST_ASSERT((int)rconfig_cnt == ci,
+ "Expected %d result resources, got %"PRIusz"\n",
+ ci, rconfig_cnt);
+
+ /*
+ * Verify status per resource
+ */
+ for (i = 0 ; i < (int)rconfig_cnt ; i++) {
+ const rd_kafka_ConfigEntry_t **entries;
+ size_t entry_cnt;
+
+ err = rd_kafka_ConfigResource_error(rconfigs[i]);
+ errstr2 = rd_kafka_ConfigResource_error_string(rconfigs[i]);
+
+ entries = rd_kafka_ConfigResource_configs(rconfigs[i],
+ &entry_cnt);
+
+ TEST_SAY("ConfigResource #%d: type %s (%d), \"%s\": "
+ "%"PRIusz" ConfigEntries, error %s (%s)\n",
+ i,
+ rd_kafka_ResourceType_name(
+ rd_kafka_ConfigResource_type(rconfigs[i])),
+ rd_kafka_ConfigResource_type(rconfigs[i]),
+ rd_kafka_ConfigResource_name(rconfigs[i]),
+ entry_cnt,
+ rd_kafka_err2name(err), errstr2 ? errstr2 : "");
+
+ test_print_ConfigEntry_array(entries, entry_cnt, 1);
+
+ if (rd_kafka_ConfigResource_type(rconfigs[i]) !=
+ rd_kafka_ConfigResource_type(configs[i]) ||
+ strcmp(rd_kafka_ConfigResource_name(rconfigs[i]),
+ rd_kafka_ConfigResource_name(configs[i]))) {
+ TEST_FAIL_LATER(
+ "ConfigResource #%d: "
+ "expected type %s name %s, "
+ "got type %s name %s",
+ i,
+ rd_kafka_ResourceType_name(rd_kafka_ConfigResource_type(configs[i])),
+ rd_kafka_ConfigResource_name(configs[i]),
+ rd_kafka_ResourceType_name(rd_kafka_ConfigResource_type(rconfigs[i])),
+ rd_kafka_ConfigResource_name(rconfigs[i]));
+ fails++;
+ continue;
+ }
+
+
+ if (err != exp_err[i]) {
+ TEST_FAIL_LATER("ConfigResource #%d: "
+ "expected %s (%d), got %s (%s)",
+ i,
+ rd_kafka_err2name(exp_err[i]),
+ exp_err[i],
+ rd_kafka_err2name(err),
+ errstr2 ? errstr2 : "");
+ fails++;
+ }
+ }
+
+ TEST_ASSERT(!fails, "See %d previous failure(s)", fails);
+
+ rd_kafka_event_destroy(rkev);
+
+ rd_kafka_ConfigResource_destroy_array(configs, ci);
+
+#undef MY_CONFRES_CNT
+}
+
+
+
+/**
+ * @brief Test DescribeConfigs
+ */
+static void do_test_DescribeConfigs (rd_kafka_t *rk, rd_kafka_queue_t *rkqu) {
+#define MY_CONFRES_CNT 3
+ char *topics[MY_CONFRES_CNT];
+ rd_kafka_ConfigResource_t *configs[MY_CONFRES_CNT];
+ rd_kafka_AdminOptions_t *options;
+ rd_kafka_resp_err_t exp_err[MY_CONFRES_CNT];
+ rd_kafka_event_t *rkev;
+ rd_kafka_resp_err_t err;
+ const rd_kafka_DescribeConfigs_result_t *res;
+ const rd_kafka_ConfigResource_t **rconfigs;
+ size_t rconfig_cnt;
+ char errstr[128];
+ const char *errstr2;
+ int ci = 0;
+ int i;
+ int fails = 0;
+
+ /*
+ * Only create one topic, the others will be non-existent.
+ */
+ rd_strdupa(&topics[0], test_mk_topic_name("DescribeConfigs_exist", 1));
+ for (i = 1 ; i < MY_CONFRES_CNT ; i++)
+ rd_strdupa(&topics[i],
+ test_mk_topic_name("DescribeConfigs_notexist", 1));
+
+ test_CreateTopics_simple(rk, NULL, topics, 1, 1, NULL);
+
+ /*
+ * ConfigResource #0: topic config, no config entries.
+ */
+ configs[ci] = rd_kafka_ConfigResource_new(
+ RD_KAFKA_RESOURCE_TOPIC, topics[ci]);
+ exp_err[ci] = RD_KAFKA_RESP_ERR_NO_ERROR;
+ ci++;
+
+ /*
+ * ConfigResource #1:broker config, no config entries
+ */
+ configs[ci] = rd_kafka_ConfigResource_new(
+ RD_KAFKA_RESOURCE_BROKER,
+ tsprintf("%"PRId32, avail_brokers[0]));
+
+ exp_err[ci] = RD_KAFKA_RESP_ERR_NO_ERROR;
+ ci++;
+
+ /*
+ * ConfigResource #2: topic config, non-existent topic, no config entr.
+ */
+ configs[ci] = rd_kafka_ConfigResource_new(
+ RD_KAFKA_RESOURCE_TOPIC, topics[ci]);
+ /* FIXME: This is a bug in the broker (
+#include
+#include
+#include "testcpp.h"
+
+/**
+ * @brief Test fetch.max.bytes
+ *
+ * - Produce 1*10 Megs to 3 partitions (~<1 Meg per message)
+ * - Set max.partition.fetch.bytes to 5 Meg
+ * - Set fetch.max.bytes to 2 Meg
+ * - Verify all messages are consumed without error.
+ */
+
+
+static void do_test_fetch_max_bytes (void) {
+ const int partcnt = 3;
+ int msgcnt = 10 * partcnt;
+ const int msgsize = 900*1024; /* Less than 1 Meg to account
+ * for batch overhead */
+ std::string errstr;
+ RdKafka::ErrorCode err;
+
+ std::string topic = Test::mk_topic_name("0081-fetch_max_bytes", 1);
+
+ /* Produce messages to partitions */
+ for (int32_t p = 0 ; p < (int32_t)partcnt ; p++)
+ test_produce_msgs_easy_size(topic.c_str(), 0, p, msgcnt, msgsize);
+
+ /* Create consumer */
+ RdKafka::Conf *conf;
+ Test::conf_init(&conf, NULL, 10);
+ Test::conf_set(conf, "group.id", topic);
+ Test::conf_set(conf, "auto.offset.reset", "earliest");
+ /* We try to fetch 20 Megs per partition, but only allow 1 Meg as total
+ * response size, this ends up serving the first batch from the
+ * first partition.
+ * receive.message.max.bytes is set low to trigger the original bug,
+ * but this value is now adjusted upwards automatically by rd_kafka_new()
+ * to hold both fetch.max.bytes and the protocol / batching overhead.
+ * Prior to the introduction of fetch.max.bytes the fetcher code
+ * would use receive.message.max.bytes to limit the total Fetch response,
+ * but due to batching overhead it would result in situations where
+ * the consumer asked for 1000000 bytes and got 1000096 bytes batch, which
+ * was higher than the 1000000 limit.
+ * See https://github.com/edenhill/librdkafka/issues/1616
+ */
+ Test::conf_set(conf, "max.partition.fetch.bytes", "20000000"); /* ~20MB */
+ Test::conf_set(conf, "fetch.max.bytes", "1000000"); /* ~1MB */
+ Test::conf_set(conf, "receive.message.max.bytes", "1000000"); /* ~1MB+ */
+
+ RdKafka::KafkaConsumer *c = RdKafka::KafkaConsumer::create(conf, errstr);
+ if (!c)
+ Test::Fail("Failed to create KafkaConsumer: " + errstr);
+ delete conf;
+
+ /* Subscribe */
+ std::vector topics;
+ topics.push_back(topic);
+ if ((err = c->subscribe(topics)))
+ Test::Fail("subscribe failed: " + RdKafka::err2str(err));
+
+ /* Start consuming */
+ Test::Say("Consuming topic " + topic + "\n");
+ int cnt = 0;
+ while (cnt < msgcnt) {
+ RdKafka::Message *msg = c->consume(tmout_multip(1000));
+ switch (msg->err())
+ {
+ case RdKafka::ERR__TIMED_OUT:
+ case RdKafka::ERR__PARTITION_EOF:
+ break;
+
+ case RdKafka::ERR_NO_ERROR:
+ cnt++;
+ break;
+
+ default:
+ Test::Fail("Consume error: " + msg->errstr());
+ break;
+ }
+
+ delete msg;
+ }
+ Test::Say("Done\n");
+
+ c->close();
+ delete c;
+}
+
+extern "C" {
+ int main_0082_fetch_max_bytes (int argc, char **argv) {
+ do_test_fetch_max_bytes();
+ return 0;
+ }
+}
diff -Nru librdkafka-0.11.3/tests/0083-cb_event.c librdkafka-0.11.6/tests/0083-cb_event.c
--- librdkafka-0.11.3/tests/0083-cb_event.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0083-cb_event.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,210 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * Tests the queue callback IO event signalling.
+ */
+
+
+#include "test.h"
+
+/* Typical include path would be , but this program
+ * is built from within the librdkafka source tree and thus differs. */
+#include "rdkafka.h" /* for Kafka driver */
+
+
+/**
+ * @brief Thread safe event counter */
+static struct {
+ mtx_t lock;
+ int count;
+} event_receiver;
+
+/**
+ * @brief Event callback function. Check the opaque pointer and
+ * increase the count of received event. */
+static void event_cb(rd_kafka_t *rk_p, void *opaque) {
+ TEST_ASSERT(opaque == (void*)0x1234,
+ "Opaque pointer is not as expected (got: %p)", opaque);
+ mtx_lock(&event_receiver.lock);
+ event_receiver.count += 1;
+ mtx_unlock(&event_receiver.lock);
+}
+
+/**
+ * @brief Wait for one or more events to be received.
+ * Return 0 if no event was received within the timeout. */
+static int wait_event_cb(int timeout_secs) {
+ int event_count = 0;
+ for (; timeout_secs >= 0; timeout_secs--) {
+ mtx_lock(&event_receiver.lock);
+ event_count = event_receiver.count;
+ event_receiver.count = 0;
+ mtx_unlock(&event_receiver.lock);
+ if (event_count > 0 || timeout_secs == 0)
+ return event_count;
+ rd_sleep(1);
+ }
+ return 0;
+}
+
+
+int main_0083_cb_event (int argc, char **argv) {
+ rd_kafka_conf_t *conf;
+ rd_kafka_topic_conf_t *tconf;
+ rd_kafka_t *rk_p, *rk_c;
+ const char *topic;
+ rd_kafka_topic_t *rkt_p;
+ rd_kafka_queue_t *queue;
+ uint64_t testid;
+ int msgcnt = 100;
+ int recvd = 0;
+ int wait_multiplier = 1;
+ rd_kafka_resp_err_t err;
+ enum {
+ _NOPE,
+ _YEP,
+ _REBALANCE
+ } expecting_io = _REBALANCE;
+ int callback_event_count;
+ rd_kafka_event_t *rkev;
+ int eventcnt = 0;
+
+ testid = test_id_generate();
+ topic = test_mk_topic_name(__FUNCTION__, 1);
+
+ rk_p = test_create_producer();
+ rkt_p = test_create_producer_topic(rk_p, topic, NULL);
+ err = test_auto_create_topic_rkt(rk_p, rkt_p, tmout_multip(5000));
+ TEST_ASSERT(!err, "Topic auto creation failed: %s",
+ rd_kafka_err2str(err));
+
+ test_conf_init(&conf, &tconf, 0);
+ rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
+ test_conf_set(conf, "session.timeout.ms", "6000");
+ test_conf_set(conf, "enable.partition.eof", "false");
+ /* Speed up propagation of new topics */
+ test_conf_set(conf, "metadata.max.age.ms", "5000");
+ test_topic_conf_set(tconf, "auto.offset.reset", "earliest");
+ rk_c = test_create_consumer(topic, NULL, conf, tconf);
+
+ queue = rd_kafka_queue_get_consumer(rk_c);
+
+ test_consumer_subscribe(rk_c, topic);
+
+ rd_kafka_queue_cb_event_enable(queue, event_cb, (void *)0x1234);
+
+ /**
+ * 1) Wait for rebalance event
+ * 2) Wait 1 interval (1s) expecting no IO (nothing produced).
+ * 3) Produce half the messages
+ * 4) Expect CB
+ * 5) Consume the available messages
+ * 6) Wait 1 interval expecting no CB.
+ * 7) Produce remaing half
+ * 8) Expect CB
+ * 9) Done.
+ */
+ while (recvd < msgcnt) {
+ TEST_SAY("Waiting for event\n");
+ callback_event_count = wait_event_cb(1 * wait_multiplier);
+ TEST_ASSERT(callback_event_count <= 1, "Event cb called %d times", callback_event_count);
+
+ if (callback_event_count == 1) {
+ TEST_SAY("Events received: %d\n", callback_event_count);
+
+ while ((rkev = rd_kafka_queue_poll(queue, 0))) {
+ eventcnt++;
+ switch (rd_kafka_event_type(rkev))
+ {
+ case RD_KAFKA_EVENT_REBALANCE:
+ TEST_SAY("Got %s: %s\n", rd_kafka_event_name(rkev),
+ rd_kafka_err2str(rd_kafka_event_error(rkev)));
+ if (expecting_io != _REBALANCE)
+ TEST_FAIL("Got Rebalance when expecting message\n");
+ if (rd_kafka_event_error(rkev) == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) {
+ rd_kafka_assign(rk_c, rd_kafka_event_topic_partition_list(rkev));
+ expecting_io = _NOPE;
+ } else
+ rd_kafka_assign(rk_c, NULL);
+ break;
+
+ case RD_KAFKA_EVENT_FETCH:
+ if (expecting_io != _YEP)
+ TEST_FAIL("Did not expect more messages at %d/%d\n",
+ recvd, msgcnt);
+ recvd++;
+ if (recvd == (msgcnt / 2) || recvd == msgcnt)
+ expecting_io = _NOPE;
+ break;
+
+ case RD_KAFKA_EVENT_ERROR:
+ TEST_FAIL("Error: %s\n", rd_kafka_event_error_string(rkev));
+ break;
+
+ default:
+ TEST_SAY("Ignoring event %s\n", rd_kafka_event_name(rkev));
+ }
+
+ rd_kafka_event_destroy(rkev);
+ }
+ TEST_SAY("%d events, Consumed %d/%d messages\n", eventcnt, recvd, msgcnt);
+
+ wait_multiplier = 1;
+
+ } else {
+ if (expecting_io == _REBALANCE) {
+ continue;
+ } else if (expecting_io == _YEP) {
+ TEST_FAIL("Did not see expected IO after %d/%d msgs\n",
+ recvd, msgcnt);
+ }
+
+ TEST_SAY("Event wait timeout (good)\n");
+ TEST_SAY("Got idle period, producing\n");
+ test_produce_msgs(rk_p, rkt_p, testid, 0, recvd, msgcnt/2,
+ NULL, 10);
+
+ expecting_io = _YEP;
+ /* When running slowly (e.g., valgrind) it might take
+ * some time before the first message is received
+ * after producing. */
+ wait_multiplier = 3;
+ }
+ }
+ TEST_SAY("Done\n");
+
+ rd_kafka_topic_destroy(rkt_p);
+ rd_kafka_destroy(rk_p);
+
+ rd_kafka_queue_destroy(queue);
+ rd_kafka_consumer_close(rk_c);
+ rd_kafka_destroy(rk_c);
+
+ return 0;
+}
diff -Nru librdkafka-0.11.3/tests/0084-destroy_flags.c librdkafka-0.11.6/tests/0084-destroy_flags.c
--- librdkafka-0.11.3/tests/0084-destroy_flags.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0084-destroy_flags.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,205 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2018, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @name Test rd_kafka_destroy_flags()
+ */
+
+
+#include "test.h"
+
+
+static RD_TLS int rebalance_cnt = 0;
+
+static void destroy_flags_rebalance_cb (rd_kafka_t *rk, rd_kafka_resp_err_t err,
+ rd_kafka_topic_partition_list_t *parts,
+ void *opaque) {
+ rebalance_cnt++;
+
+ TEST_SAY("rebalance_cb: %s with %d partition(s)\n",
+ rd_kafka_err2str(err), parts->cnt);
+
+ switch (err)
+ {
+ case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS:
+ test_consumer_assign("rebalance", rk, parts);
+ break;
+
+ case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS:
+ test_consumer_unassign("rebalance", rk);
+ break;
+
+ default:
+ TEST_FAIL("rebalance_cb: error: %s", rd_kafka_err2str(err));
+ }
+}
+
+struct df_args {
+ rd_kafka_type_t client_type;
+ int produce_cnt;
+ int consumer_subscribe;
+ int consumer_unsubscribe;
+};
+
+static void do_test_destroy_flags (const char *topic,
+ int destroy_flags,
+ int local_mode,
+ const struct df_args *args) {
+ rd_kafka_t *rk;
+ rd_kafka_conf_t *conf;
+ test_timing_t t_destroy;
+
+ TEST_SAY(_C_MAG "[ test destroy_flags 0x%x for client_type %d, "
+ "produce_cnt %d, subscribe %d, unsubscribe %d, "
+ "%s mode ]\n" _C_CLR,
+ destroy_flags, args->client_type,
+ args->produce_cnt, args->consumer_subscribe,
+ args->consumer_unsubscribe,
+ local_mode ? "local" : "broker");
+
+ test_conf_init(&conf, NULL, 20);
+
+ if (local_mode)
+ test_conf_set(conf, "bootstrap.servers", "");
+
+ if (args->client_type == RD_KAFKA_PRODUCER) {
+
+ rk = test_create_handle(args->client_type, conf);
+
+ if (args->produce_cnt > 0) {
+ rd_kafka_topic_t *rkt;
+ int msgcounter = 0;
+
+ rkt = test_create_producer_topic(rk, topic, NULL);
+ test_produce_msgs_nowait(rk, rkt, 0,
+ RD_KAFKA_PARTITION_UA,
+ 0, 10000, NULL, 100,
+ &msgcounter);
+ rd_kafka_topic_destroy(rkt);
+ }
+
+ } else {
+ int i;
+
+ TEST_ASSERT(args->client_type == RD_KAFKA_CONSUMER);
+
+ rk = test_create_consumer(topic, destroy_flags_rebalance_cb,
+ conf, NULL);
+
+ if (args->consumer_subscribe) {
+ test_consumer_subscribe(rk, topic);
+
+ if (!local_mode) {
+ TEST_SAY("Waiting for assignment\n");
+ while (rebalance_cnt == 0)
+ test_consumer_poll_once(rk, NULL, 1000);
+ }
+ }
+
+ for (i = 0 ; i < 5 ; i++)
+ test_consumer_poll_once(rk, NULL, 100);
+
+ if (args->consumer_unsubscribe) {
+ /* Test that calling rd_kafka_unsubscribe immediately
+ * prior to rd_kafka_destroy_flags doesn't cause the
+ * latter to hang. */
+ TEST_SAY(_C_YEL"Calling rd_kafka_unsubscribe\n"_C_CLR);
+ rd_kafka_unsubscribe(rk);
+ }
+ }
+
+ rebalance_cnt = 0;
+ TEST_SAY(_C_YEL "Calling rd_kafka_destroy_flags(0x%x)\n" _C_CLR,
+ destroy_flags);
+ TIMING_START(&t_destroy, "rd_kafka_destroy_flags(0x%x)", destroy_flags);
+ rd_kafka_destroy_flags(rk, destroy_flags);
+ TIMING_STOP(&t_destroy);
+
+ if (destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE)
+ TIMING_ASSERT_LATER(&t_destroy, 0, 200);
+ else
+ TIMING_ASSERT_LATER(&t_destroy, 0, 1000);
+
+ if (args->consumer_subscribe &&
+ !(destroy_flags & RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE)) {
+ if (!local_mode)
+ TEST_ASSERT(rebalance_cnt > 0,
+ "expected final rebalance callback");
+ } else
+ TEST_ASSERT(rebalance_cnt == 0,
+ "expected no rebalance callbacks, got %d",
+ rebalance_cnt);
+
+ TEST_SAY(_C_GRN "[ test destroy_flags 0x%x for client_type %d, "
+ "produce_cnt %d, subscribe %d, unsubscribe %d, "
+ "%s mode: PASS ]\n" _C_CLR,
+ destroy_flags, args->client_type,
+ args->produce_cnt, args->consumer_subscribe,
+ args->consumer_unsubscribe,
+ local_mode ? "local" : "broker");
+}
+
+
+/**
+ * @brief Destroy with flags
+ */
+static void destroy_flags (int local_mode) {
+ const struct df_args args[] = {
+ { RD_KAFKA_PRODUCER, 0, 0, 0 },
+ { RD_KAFKA_PRODUCER, 10000, 0, 0 },
+ { RD_KAFKA_CONSUMER, 0, 1, 0 },
+ { RD_KAFKA_CONSUMER, 0, 1, 1 },
+ { RD_KAFKA_CONSUMER, 0, 0, 0 }
+ };
+ const int flag_combos[] = { 0,
+ RD_KAFKA_DESTROY_F_NO_CONSUMER_CLOSE };
+ const char *topic = test_mk_topic_name(__FUNCTION__, 1);
+ int i, j;
+
+ for (i = 0 ; i < (int)RD_ARRAYSIZE(args) ; i++) {
+ for (j = 0 ; j < (int)RD_ARRAYSIZE(flag_combos) ; j++) {
+ do_test_destroy_flags(topic,
+ flag_combos[j],
+ local_mode,
+ &args[i]);
+ }
+ }
+
+}
+
+
+
+int main_0084_destroy_flags_local (int argc, char **argv) {
+ destroy_flags(1/*no brokers*/);
+ return 0;
+}
+
+int main_0084_destroy_flags (int argc, char **argv) {
+ destroy_flags(0/*with brokers*/);
+ return 0;
+}
diff -Nru librdkafka-0.11.3/tests/0088-produce_metadata_timeout.c librdkafka-0.11.6/tests/0088-produce_metadata_timeout.c
--- librdkafka-0.11.3/tests/0088-produce_metadata_timeout.c 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/0088-produce_metadata_timeout.c 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,160 @@
+/*
+ * librdkafka - Apache Kafka C library
+ *
+ * Copyright (c) 2012-2015, Magnus Edenhill
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "test.h"
+
+#if WITH_SOCKEM
+#include "rdkafka.h"
+
+#include
+
+/**
+ * @name Verify #1985:
+ *
+ * Previously known topic transitions to UNKNOWN when metadata times out,
+ * new messages are put on UA, when brokers come up again and metadata
+ * is retrieved the UA messages must be produced.
+ */
+
+static rd_atomic32_t refuse_connect;
+
+
+/**
+ * @brief Sockem connect, called from **internal librdkafka thread** through
+ * librdkafka's connect_cb
+ */
+static int connect_cb (struct test *test, sockem_t *skm, const char *id) {
+ if (rd_atomic32_get(&refuse_connect) > 0)
+ return -1;
+ else
+ return 0;
+}
+
+static int is_fatal_cb (rd_kafka_t *rk, rd_kafka_resp_err_t err,
+ const char *reason) {
+ /* Ignore connectivity errors since we'll be bringing down
+ * .. connectivity.
+ * SASL auther will think a connection-down even in the auth
+ * state means the broker doesn't support SASL PLAIN. */
+ TEST_SAY("is_fatal?: %s: %s\n", rd_kafka_err2str(err), reason);
+ if (err == RD_KAFKA_RESP_ERR__TRANSPORT ||
+ err == RD_KAFKA_RESP_ERR__ALL_BROKERS_DOWN ||
+ err == RD_KAFKA_RESP_ERR__AUTHENTICATION ||
+ err == RD_KAFKA_RESP_ERR__TIMED_OUT)
+ return 0;
+ return 1;
+}
+
+static int msg_dr_cnt = 0;
+static int msg_dr_fail_cnt = 0;
+
+static void dr_msg_cb (rd_kafka_t *rk, const rd_kafka_message_t *rkmessage,
+ void *opaque) {
+ msg_dr_cnt++;
+ TEST_SAYL(3, "Delivery for message %.*s: %s\n",
+ (int)rkmessage->len, (const char *)rkmessage->payload,
+ rd_kafka_err2name(rkmessage->err));
+
+ if (rkmessage->err) {
+ TEST_FAIL_LATER("Expected message to succeed, got %s",
+ rd_kafka_err2str(rkmessage->err));
+ msg_dr_fail_cnt++;
+ }
+}
+
+
+
+int main_0088_produce_metadata_timeout (int argc, char **argv) {
+ int64_t testid;
+ rd_kafka_t *rk;
+ rd_kafka_topic_t *rkt;
+ const char *topic = test_mk_topic_name("0088_produce_metadata_timeout",
+ 1);
+ int msgcnt = 0;
+ rd_kafka_conf_t *conf;
+
+ testid = test_id_generate();
+
+ /* Create topic with single partition, for simplicity. */
+ test_create_topic(topic, 1, 1);
+
+ test_conf_init(&conf, NULL, 15*60*2); // msgcnt * 2);
+ rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);
+ test_conf_set(conf, "metadata.max.age.ms", "10000");
+ test_conf_set(conf, "topic.metadata.refresh.interval.ms", "-1");
+ test_conf_set(conf, "linger.ms", "5000");
+ test_conf_set(conf, "batch.num.messages", "5");
+
+ test_socket_enable(conf);
+ test_curr->connect_cb = connect_cb;
+ test_curr->is_fatal_cb = is_fatal_cb;
+
+ rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
+ rkt = rd_kafka_topic_new(rk, topic, NULL);
+
+ /* Produce first set of messages and wait for delivery */
+ test_produce_msgs_nowait(rk, rkt, testid, RD_KAFKA_PARTITION_UA,
+ msgcnt, 20, NULL, 0, &msgcnt);
+ while (msg_dr_cnt < 5)
+ rd_kafka_poll(rk, 1000);
+
+ TEST_SAY(_C_YEL "Disconnecting sockets and "
+ "refusing future connections\n");
+ rd_atomic32_set(&refuse_connect, 1);
+ test_socket_close_all(test_curr, 1/*reinit*/);
+
+
+ /* Wait for metadata timeout */
+ TEST_SAY("Waiting for metadata timeout\n");
+ rd_sleep(10+5);
+
+ /* These messages will be put on the UA queue */
+ test_produce_msgs_nowait(rk, rkt, testid, RD_KAFKA_PARTITION_UA,
+ msgcnt, 20, NULL, 0, &msgcnt);
+
+ /* Restore the connection(s) when metadata has timed out. */
+ TEST_SAY(_C_YEL "Allowing connections\n");
+ rd_atomic32_set(&refuse_connect, 0);
+
+ rd_sleep(3);
+ test_produce_msgs_nowait(rk, rkt, testid, RD_KAFKA_PARTITION_UA,
+ msgcnt, 20, NULL, 0, &msgcnt);
+
+ test_flush(rk, 2*5*1000); /* linger.ms * 2 */
+
+ TEST_ASSERT(msg_dr_cnt == msgcnt,
+ "expected %d, got %d", msgcnt, msg_dr_cnt);
+ TEST_ASSERT(msg_dr_fail_cnt == 0,
+ "expected %d dr failures, got %d", 0, msg_dr_fail_cnt);
+
+ rd_kafka_topic_destroy(rkt);
+ rd_kafka_destroy(rk);
+
+ return 0;
+}
+#endif
diff -Nru librdkafka-0.11.3/tests/autotest.sh librdkafka-0.11.6/tests/autotest.sh
--- librdkafka-0.11.3/tests/autotest.sh 1970-01-01 00:00:00.000000000 +0000
+++ librdkafka-0.11.6/tests/autotest.sh 2018-10-10 06:54:38.000000000 +0000
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# autotest.sh runs the integration tests using a temporary Kafka cluster.
+# This is intended to be used on CI.
+#
+
+set -e
+
+KAFKA_VERSION=$1
+
+if [[ -z $KAFKA_VERSION ]]; then
+ echo "Usage: $0 "
+ exit 1
+fi
+
+set -x
+
+pushd tests
+
+[[ -d _venv ]] || virtualenv _venv
+source _venv/bin/activate
+
+# Install trivup that is used to bring up a cluster.
+pip install -U trivup
+
+# Run tests that automatically spin up their clusters
+export KAFKA_VERSION
+
+echo "## Running full test suite for broker version $KAFKA_VERSION ##"
+time make full
+
+
+popd # tests
diff -Nru librdkafka-0.11.3/tests/broker_version_tests.py librdkafka-0.11.6/tests/broker_version_tests.py
--- librdkafka-0.11.3/tests/broker_version_tests.py 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/broker_version_tests.py 2018-10-10 06:54:38.000000000 +0000
@@ -46,16 +46,14 @@
cluster.start(timeout=30)
if conf.get('test_mode', '') == 'bash':
- cmd_env = 'export KAFKA_PATH="%s" RDKAFKA_TEST_CONF="%s" ZK_ADDRESS="%s" BROKERS="%s" TEST_KAFKA_VERSION="%s" TRIVUP_ROOT="%s"; ' % \
- (cluster.get_all('destdir', '', KafkaBrokerApp)[0], rdkafka.test_conf_file, cluster.get_all('address', '', ZookeeperApp)[0], cluster.bootstrap_servers(), version, cluster.instance_path())
cmd = 'bash --rcfile <(cat ~/.bashrc; echo \'PS1="[TRIVUP:%s@%s] \\u@\\h:\w$ "\')' % (cluster.name, version)
- subprocess.call('%s %s' % (cmd_env, cmd), shell=True, executable='/bin/bash')
+ subprocess.call(cmd, env=rdkafka.env, shell=True, executable='/bin/bash')
report = None
else:
rdkafka.start()
print('# librdkafka regression tests started, logs in %s' % rdkafka.root_path())
- rdkafka.wait_stopped(timeout=60*10)
+ rdkafka.wait_stopped(timeout=60*30)
report = rdkafka.report()
report['root_path'] = rdkafka.root_path()
@@ -63,7 +61,7 @@
if report.get('tests_failed', 0) > 0 and interact:
print('# Connect to cluster with bootstrap.servers %s' % cluster.bootstrap_servers())
print('# Exiting the shell will bring down the cluster. Good luck.')
- subprocess.call('bash --rcfile <(cat ~/.bashrc; echo \'PS1="[TRIVUP:%s@%s] \\u@\\h:\w$ "\')' % (cluster.name, version), shell=True, executable='/bin/bash')
+ subprocess.call('bash --rcfile <(cat ~/.bashrc; echo \'PS1="[TRIVUP:%s@%s] \\u@\\h:\w$ "\')' % (cluster.name, version), env=rdkafka.env, shell=True, executable='/bin/bash')
cluster.stop(force=True)
@@ -201,6 +199,10 @@
print('\033[41m#### Version %s, suite %s: FAILED: %s\033[0m' %
(version, suite['name'], reason))
fail_cnt += 1
+
+ # Emit hopefully relevant parts of the log on failure
+ subprocess.call("grep --color=always -B100 -A10 FAIL %s" % (os.path.join(report['root_path'], 'stderr.log')), shell=True)
+
print('#### Test output: %s/stderr.log' % (report['root_path']))
suite['version'][version] = report
diff -Nru librdkafka-0.11.3/tests/CMakeLists.txt librdkafka-0.11.6/tests/CMakeLists.txt
--- librdkafka-0.11.3/tests/CMakeLists.txt 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/CMakeLists.txt 2018-10-10 06:54:38.000000000 +0000
@@ -64,15 +64,36 @@
0068-produce_timeout.c
0069-consumer_add_parts.c
0070-null_empty.cpp
+ 0072-headers_ut.c
+ 0073-headers.c
0074-producev.c
+ 0075-retry.c
+ 0076-produce_retry.c
+ 0077-compaction.c
+ 0078-c_from_cpp.cpp
+ 0079-fork.c
+ 0080-admin_ut.c
+ 0081-admin.c
+ 0082-fetch_max_bytes.cpp
+ 0083-cb_event.c
+ 0084-destroy_flags.c
+ 0088-produce_metadata_timeout.c
8000-idle.cpp
test.c
testcpp.cpp
- sockem.c
)
+if(NOT WIN32)
+ list(APPEND sources sockem.c)
+else()
+ list(APPEND sources ../src/tinycthread.c ../src/tinycthread_extra.c)
+endif()
+
add_executable(rdkafka_test ${sources})
target_link_libraries(rdkafka_test PUBLIC rdkafka++)
+if(WIN32)
+ target_compile_definitions(rdkafka_test PRIVATE LIBRDKAFKACPP_EXPORTS=0)
+endif(WIN32)
add_test(NAME RdKafkaTestInParallel COMMAND rdkafka_test -p5)
add_test(NAME RdKafkaTestSequentially COMMAND rdkafka_test -p1)
diff -Nru librdkafka-0.11.3/tests/gen-ssl-certs.sh librdkafka-0.11.6/tests/gen-ssl-certs.sh
--- librdkafka-0.11.3/tests/gen-ssl-certs.sh 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/gen-ssl-certs.sh 2018-10-10 06:54:38.000000000 +0000
@@ -60,7 +60,7 @@
#Step 1
echo "############ Generating key"
- keytool -storepass "$PASS" -keypass "$PASS" -keystore ${PFX}server.keystore.jks -alias localhost -validity $VALIDITY -genkey <use.delay + skm->use.jitter == 0)
+static int sockem_calc_waittime (sockem_t *skm, int64_t now) {
+ const sockem_buf_t *sb;
+
+ if (!(sb = TAILQ_FIRST(&skm->bufs)))
return 1000;
+ else if (now > sb->sb_at)
+ return 0;
else
- return skm->use.jitter < skm->use.delay ?
- skm->use.jitter : skm->use.delay;
+ return (sb->sb_at - now) / 1000;
+}
+
+
+/**
+ * @brief Unlink and destroy a buffer
+ */
+static void sockem_buf_destroy (sockem_t *skm, sockem_buf_t *sb) {
+ skm->bufs_size -= sb->sb_size - sb->sb_of;
+ TAILQ_REMOVE(&skm->bufs, sb, sb_link);
+ free(sb);
+}
+
+/**
+ * @brief Add delayed buffer to transmit.
+ */
+static sockem_buf_t *sockem_buf_add (sockem_t *skm,
+ size_t size, const void *data) {
+ sockem_buf_t *sb;
+
+ skm->bufs_size += size;
+ if (skm->bufs_size > skm->bufs_size_max) {
+ /* No more buffer space, halt recv fd until
+ * queued buffers drop below threshold. */
+ skm->poll_fd_cnt = 1;
+ }
+
+ sb = malloc(sizeof(*sb) + size);
+
+ sb->sb_of = 0;
+ sb->sb_size = size;
+ sb->sb_data = (char *)(sb+1);
+ sb->sb_at = sockem_clock() +
+ ((skm->use.delay +
+ (skm->use.jitter / 2)/*FIXME*/) * 1000);
+ memcpy(sb->sb_data, data, size);
+
+ TAILQ_INSERT_TAIL(&skm->bufs, sb, sb_link);
+
+ return sb;
+}
+
+
+/**
+ * @brief Forward any delayed buffers that have passed their deadline
+ * @remark lock must be held but will be released momentarily while
+ * performing send syscall.
+ */
+static int sockem_fwd_bufs (sockem_t *skm, int ofd) {
+ sockem_buf_t *sb;
+ int64_t now = sockem_clock();
+ size_t to_write;
+ int64_t elapsed = now - skm->ts_last_fwd;
+
+ if (!elapsed)
+ return 0;
+
+ /* Calculate how many bytes to send to adhere to rate-limit */
+ to_write = (size_t)((double)skm->use.tx_thruput *
+ ((double)elapsed / 1000000.0));
+
+ while (to_write > 0 &&
+ (sb = TAILQ_FIRST(&skm->bufs)) && sb->sb_at <= now) {
+ ssize_t r;
+ size_t remain = sb->sb_size - sb->sb_of;
+ size_t wr = to_write < remain ? to_write : remain;
+
+ if (wr == 0)
+ break;
+
+ mtx_unlock(&skm->lock);
+
+ r = send(ofd, sb->sb_data+sb->sb_of, wr, 0);
+
+ mtx_lock(&skm->lock);
+
+ if (r == -1) {
+ if (errno == ENOBUFS || errno == EAGAIN ||
+ errno == EWOULDBLOCK)
+ return 0;
+ return -1;
+ }
+
+ skm->ts_last_fwd = now;
+
+ sb->sb_of += r;
+ to_write -= r;
+
+ if (sb->sb_of < sb->sb_size)
+ break;
+
+ sockem_buf_destroy(skm, sb);
+
+ now = sockem_clock();
+ }
+
+ /* Re-enable app fd poll if queued buffers are below threshold */
+ if (skm->bufs_size < skm->bufs_size_max)
+ skm->poll_fd_cnt = 2;
+
+ return 0;
}
@@ -201,11 +324,10 @@
*
* @returns the number of bytes forwarded, or -1 on error.
*/
-static int sockem_recv_fwd (sockem_t *skm, int ifd, int ofd) {
+static int sockem_recv_fwd (sockem_t *skm, int ifd, int ofd, int direct) {
ssize_t r, wr;
- int64_t delay;
- r = recv(ifd, skm->buf, skm->bufsz, MSG_DONTWAIT);
+ r = recv(ifd, skm->recv_buf, skm->recv_bufsz, MSG_DONTWAIT);
if (r == -1) {
int serr = socket_errno();
if (serr == EAGAIN || serr == EWOULDBLOCK)
@@ -217,16 +339,17 @@
return -1;
}
- /* FIXME: proper */
- delay = skm->use.delay + (skm->use.jitter / 2);
- if (delay)
- usleep(delay * 1000);
-
- wr = send(ofd, skm->buf, r, 0);
- if (wr < r)
- return -1;
+ if (direct) {
+ /* No delay or rate limit, send right away */
+ wr = send(ofd, skm->recv_buf, r, 0);
+ if (wr < r)
+ return -1;
- return wr;
+ return wr;
+ } else {
+ sockem_buf_add(skm, r, skm->recv_buf);
+ return r;
+ }
}
@@ -254,13 +377,22 @@
}
+/**
+ * @brief Copy desired (app) config to internally use(d) configuration.
+ * @remark lock must be held
+ */
+static __inline void sockem_conf_use (sockem_t *skm) {
+ skm->use = skm->conf;
+ /* Figure out if direct forward is to be used */
+ skm->use.direct = !(skm->use.delay || skm->use.jitter ||
+ (skm->use.tx_thruput < (1 << 30)));
+}
/**
* @brief sockem internal per-socket forwarder thread
*/
static void *sockem_run (void *arg) {
sockem_t *skm = arg;
- int waittime;
int cs = -1;
int ls;
struct pollfd pfd[2];
@@ -268,12 +400,12 @@
mtx_lock(&skm->lock);
if (skm->run == SOCKEM_START)
skm->run = SOCKEM_RUN;
- skm->use = skm->conf;
+ sockem_conf_use(skm);
ls = skm->ls;
mtx_unlock(&skm->lock);
- skm->bufsz = skm->use.bufsz;
- skm->buf = malloc(skm->bufsz);
+ skm->recv_bufsz = skm->use.recv_bufsz;
+ skm->recv_buf = malloc(skm->recv_bufsz);
/* Accept connection from sockfd in sockem_connect() */
cs = accept(ls, NULL, 0);
@@ -290,35 +422,51 @@
/* Set up poll (blocking IO) */
memset(pfd, 0, sizeof(pfd));
- pfd[0].fd = cs;
- pfd[0].events = POLLIN;
+ pfd[1].fd = cs;
+ pfd[1].events = POLLIN;
mtx_lock(&skm->lock);
- pfd[1].fd = skm->ps;
+ pfd[0].fd = skm->ps;
mtx_unlock(&skm->lock);
- pfd[1].events = POLLIN;
+ pfd[0].events = POLLIN;
- waittime = sockem_calc_waittime(skm);
+ skm->poll_fd_cnt = 2;
mtx_lock(&skm->lock);
while (skm->run == SOCKEM_RUN) {
int r;
int i;
+ int waittime = sockem_calc_waittime(skm, sockem_clock());
mtx_unlock(&skm->lock);
- r = poll(pfd, 2, waittime);
-
+ r = poll(pfd, skm->poll_fd_cnt, waittime);
if (r == -1)
break;
+ /* Send/forward delayed buffers */
+ mtx_lock(&skm->lock);
+ sockem_conf_use(skm);
+
+ if (sockem_fwd_bufs(skm, skm->ps) == -1) {
+ mtx_unlock(&skm->lock);
+ skm->run = SOCKEM_TERM;
+ break;
+ }
+ mtx_unlock(&skm->lock);
+
for (i = 0 ; r > 0 && i < 2 ; i++) {
if (pfd[i].revents & (POLLHUP|POLLERR)) {
skm->run = SOCKEM_TERM;
} else if (pfd[i].revents & POLLIN) {
- if (sockem_recv_fwd(skm,
- pfd[i].fd,
- pfd[i^1].fd) == -1) {
+ if (sockem_recv_fwd(
+ skm,
+ pfd[i].fd,
+ pfd[i^1].fd,
+ /* direct mode for app socket
+ * with delay, and always for
+ * peer socket (receive channel) */
+ i == 0 || skm->use.direct) == -1) {
skm->run = SOCKEM_TERM;
break;
}
@@ -326,7 +474,6 @@
}
mtx_lock(&skm->lock);
- skm->use = skm->conf;
}
done:
if (cs != -1)
@@ -334,7 +481,7 @@
sockem_close_all(skm);
mtx_unlock(&skm->lock);
- free(skm->buf);
+ free(skm->recv_buf);
return NULL;
@@ -418,14 +565,17 @@
skm->as = sockfd;
skm->ls = ls;
skm->ps = ps;
+ skm->bufs_size_max = 16 * 1024 * 1024; /* 16kb of queue buffer */
+ TAILQ_INIT(&skm->bufs);
mtx_init(&skm->lock);
- /* Default config*/
+ /* Default config */
skm->conf.rx_thruput = 1 << 30;
skm->conf.tx_thruput = 1 << 30;
skm->conf.delay = 0;
skm->conf.jitter = 0;
- skm->conf.bufsz = 1024*1024;
+ skm->conf.recv_bufsz = 1024*1024;
+ skm->conf.direct = 1;
/* Apply passed configuration */
va_start(ap, addrlen);
@@ -464,6 +614,18 @@
return skm;
}
+
+/**
+ * @brief Purge/drop all queued buffers
+ */
+static void sockem_bufs_purge (sockem_t *skm) {
+ sockem_buf_t *sb;
+
+ while ((sb = TAILQ_FIRST(&skm->bufs)))
+ sockem_buf_destroy(skm, sb);
+}
+
+
void sockem_close (sockem_t *skm) {
@@ -485,6 +647,8 @@
thrd_join(skm->thrd, NULL);
+ sockem_bufs_purge(skm);
+
mtx_destroy(&skm->lock);
@@ -509,7 +673,7 @@
else if (!strcmp(key, "jitter"))
skm->conf.jitter = val;
else if (!strcmp(key, "rx.bufsz"))
- skm->conf.bufsz = val;
+ skm->conf.recv_bufsz = val;
else if (!strcmp(key, "debug"))
skm->conf.debug = val;
else if (!strcmp(key, "true"))
@@ -534,6 +698,7 @@
}
} else
return -1;
+
return 0;
}
diff -Nru librdkafka-0.11.3/tests/sockem.h librdkafka-0.11.6/tests/sockem.h
--- librdkafka-0.11.3/tests/sockem.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/sockem.h 2018-10-10 06:54:38.000000000 +0000
@@ -26,7 +26,8 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _RD_SOCKEM_H_
+#define _RD_SOCKEM_H_
#include
#include
@@ -80,3 +81,5 @@
* @remark Application is responsible for locking.
*/
sockem_t *sockem_find (int sockfd);
+
+#endif /* _RD_SOCKEM_H_ */
diff -Nru librdkafka-0.11.3/tests/test.c librdkafka-0.11.6/tests/test.c
--- librdkafka-0.11.3/tests/test.c 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/test.c 2018-10-10 06:54:38.000000000 +0000
@@ -34,6 +34,9 @@
#include
#include
+#ifdef _MSC_VER
+#include /* _getcwd */
+#endif
/* Typical include path would be , but this program
* is built from within the librdkafka source tree and thus differs. */
@@ -60,7 +63,8 @@
static int test_delete_topics_between = 0;
static const char *test_git_version = "HEAD";
static const char *test_sockem_conf = "";
-
+int test_on_ci = 0; /* Tests are being run on CI, be more forgiving
+ * with regards to timeouts, etc. */
static int show_summary = 1;
static int test_summary (int do_lock);
@@ -121,6 +125,7 @@
_TEST_DECL(0036_partial_fetch);
_TEST_DECL(0037_destroy_hang_local);
_TEST_DECL(0038_performance);
+_TEST_DECL(0039_event_dr);
_TEST_DECL(0039_event);
_TEST_DECL(0040_io_event);
_TEST_DECL(0041_fetch_max_bytes);
@@ -137,7 +142,8 @@
_TEST_DECL(0050_subscribe_adds);
_TEST_DECL(0051_assign_adds);
_TEST_DECL(0052_msg_timestamps);
-_TEST_DECL(0053_stats_cb);
+_TEST_DECL(0053_stats_timing);
+_TEST_DECL(0053_stats);
_TEST_DECL(0054_offset_time);
_TEST_DECL(0055_producer_latency);
_TEST_DECL(0056_balanced_group_mt);
@@ -155,8 +161,21 @@
_TEST_DECL(0068_produce_timeout);
_TEST_DECL(0069_consumer_add_parts);
_TEST_DECL(0070_null_empty);
+_TEST_DECL(0072_headers_ut);
+_TEST_DECL(0073_headers);
_TEST_DECL(0074_producev);
-
+_TEST_DECL(0075_retry);
+_TEST_DECL(0076_produce_retry);
+_TEST_DECL(0077_compaction);
+_TEST_DECL(0078_c_from_cpp);
+_TEST_DECL(0079_fork);
+_TEST_DECL(0080_admin_ut);
+_TEST_DECL(0081_admin);
+_TEST_DECL(0082_fetch_max_bytes);
+_TEST_DECL(0083_cb_event);
+_TEST_DECL(0084_destroy_flags_local);
+_TEST_DECL(0084_destroy_flags);
+_TEST_DECL(0088_produce_metadata_timeout);
/* Manual tests */
_TEST_DECL(8000_idle);
@@ -202,7 +221,8 @@
_TEST(0036_partial_fetch, 0),
_TEST(0037_destroy_hang_local, TEST_F_LOCAL),
_TEST(0038_performance, 0),
- _TEST(0039_event, 0),
+ _TEST(0039_event_dr, 0),
+ _TEST(0039_event, TEST_F_LOCAL),
_TEST(0040_io_event, 0, TEST_BRKVER(0,9,0,0)),
_TEST(0041_fetch_max_bytes, 0),
_TEST(0042_many_topics, 0),
@@ -217,12 +237,13 @@
_TEST(0047_partial_buf_tmout, TEST_F_KNOWN_ISSUE),
_TEST(0048_partitioner, 0),
#if WITH_SOCKEM
- _TEST(0049_consume_conn_close, 0, TEST_BRKVER(0,9,0,0)),
+ _TEST(0049_consume_conn_close, TEST_F_SOCKEM, TEST_BRKVER(0,9,0,0)),
#endif
_TEST(0050_subscribe_adds, 0, TEST_BRKVER(0,9,0,0)),
_TEST(0051_assign_adds, 0, TEST_BRKVER(0,9,0,0)),
_TEST(0052_msg_timestamps, 0, TEST_BRKVER(0,10,0,0)),
- _TEST(0053_stats_cb, TEST_F_LOCAL),
+ _TEST(0053_stats_timing, TEST_F_LOCAL),
+ _TEST(0053_stats, 0),
_TEST(0054_offset_time, 0, TEST_BRKVER(0,10,1,0)),
_TEST(0055_producer_latency, TEST_F_KNOWN_ISSUE_WIN32),
_TEST(0056_balanced_group_mt, 0, TEST_BRKVER(0,9,0,0)),
@@ -240,13 +261,32 @@
.extra = "dynamic loading of tests might not be fixed for this platform"),
_TEST(0067_empty_topic, 0),
#if WITH_SOCKEM
- _TEST(0068_produce_timeout, 0),
+ _TEST(0068_produce_timeout, TEST_F_SOCKEM),
#endif
_TEST(0069_consumer_add_parts, TEST_F_KNOWN_ISSUE_WIN32,
TEST_BRKVER(0,9,0,0)),
_TEST(0070_null_empty, 0),
+ _TEST(0072_headers_ut, TEST_F_LOCAL),
+ _TEST(0073_headers, 0, TEST_BRKVER(0,11,0,0)),
_TEST(0074_producev, TEST_F_LOCAL),
-
+#if WITH_SOCKEM
+ _TEST(0075_retry, TEST_F_SOCKEM),
+#endif
+ _TEST(0076_produce_retry, TEST_F_SOCKEM),
+ _TEST(0077_compaction, 0, TEST_BRKVER(0,9,0,0)),
+ _TEST(0078_c_from_cpp, TEST_F_LOCAL),
+ _TEST(0079_fork, TEST_F_LOCAL|TEST_F_KNOWN_ISSUE,
+ .extra = "using a fork():ed rd_kafka_t is not supported and will "
+ "most likely hang"),
+ _TEST(0080_admin_ut, TEST_F_LOCAL),
+ _TEST(0081_admin, 0, TEST_BRKVER(0,10,2,0)),
+ _TEST(0082_fetch_max_bytes, 0, TEST_BRKVER(0,10,1,0)),
+ _TEST(0083_cb_event, 0, TEST_BRKVER(0,9,0,0)),
+ _TEST(0084_destroy_flags_local, TEST_F_LOCAL),
+ _TEST(0084_destroy_flags, 0),
+#if WITH_SOCKEM
+ _TEST(0088_produce_metadata_timeout, TEST_F_SOCKEM),
+#endif
/* Manual tests */
_TEST(8000_idle, TEST_F_MANUAL),
@@ -270,15 +310,35 @@
}
static void test_socket_del (struct test *test, sockem_t *skm, int do_lock) {
- void *p;
if (do_lock)
TEST_LOCK();
- p = rd_list_remove(&test->sockets, skm);
- assert(p);
+ /* Best effort, skm might not have been added if connect_cb failed */
+ rd_list_remove(&test->sockets, skm);
if (do_lock)
TEST_UNLOCK();
}
+int test_socket_sockem_set_all (const char *key, int val) {
+ int i;
+ sockem_t *skm;
+ int cnt = 0;
+
+ TEST_LOCK();
+
+ cnt = rd_list_cnt(&test_curr->sockets);
+ TEST_SAY("Setting sockem %s=%d on %s%d socket(s)\n", key, val,
+ cnt > 0 ? "" : _C_RED, cnt);
+
+ RD_LIST_FOREACH(skm, &test_curr->sockets, i) {
+ if (sockem_set(skm, key, val, NULL) == -1)
+ TEST_FAIL("sockem_set(%s, %d) failed", key, val);
+ }
+
+ TEST_UNLOCK();
+
+ return cnt;
+}
+
void test_socket_close_all (struct test *test, int reinit) {
TEST_LOCK();
rd_list_destroy(&test->sockets);
@@ -488,7 +548,7 @@
rd_kafka_topic_conf_t *topic_conf,
int *timeoutp) {
FILE *fp;
- char buf[512];
+ char buf[1024];
int line = 0;
#ifndef _MSC_VER
@@ -710,7 +770,7 @@
size_t of;
of = rd_snprintf(dest, dest_size,
- "testid=%"PRIu64", partition=%"PRId32", msg=%i",
+ "testid=%"PRIu64", partition=%"PRId32", msg=%i\n",
testid, partition, msgid);
if (of < dest_size - 1) {
memset(dest+of, '!', dest_size-of);
@@ -741,33 +801,40 @@
/**
* Parse a message token
*/
-void test_msg_parse0 (const char *func, int line,
- uint64_t testid, rd_kafka_message_t *rkmessage,
- int32_t exp_partition, int *msgidp) {
- char buf[128];
- uint64_t in_testid;
- int in_part;
-
- if (!rkmessage->key)
- TEST_FAIL("%s:%i: Message (%s [%"PRId32"] @ %"PRId64") "
- "has empty key\n",
- func, line,
- rd_kafka_topic_name(rkmessage->rkt),
- rkmessage->partition, rkmessage->offset);
-
- rd_snprintf(buf, sizeof(buf), "%.*s", (int)rkmessage->key_len,
- (char *)rkmessage->key);
-
- if (sscanf(buf, "testid=%"SCNu64", partition=%i, msg=%i",
- &in_testid, &in_part, msgidp) != 3)
- TEST_FAIL("%s:%i: Incorrect key format: %s", func, line, buf);
-
+void test_msg_parse00 (const char *func, int line,
+ uint64_t testid, int32_t exp_partition, int *msgidp,
+ const char *topic, int32_t partition, int64_t offset,
+ const char *key, size_t key_size) {
+ char buf[128];
+ uint64_t in_testid;
+ int in_part;
+
+ if (!key)
+ TEST_FAIL("%s:%i: Message (%s [%"PRId32"] @ %"PRId64") "
+ "has empty key\n",
+ func, line, topic, partition, offset);
+
+ rd_snprintf(buf, sizeof(buf), "%.*s", (int)key_size, key);
+
+ if (sscanf(buf, "testid=%"SCNu64", partition=%i, msg=%i\n",
+ &in_testid, &in_part, msgidp) != 3)
+ TEST_FAIL("%s:%i: Incorrect key format: %s", func, line, buf);
+
+
+ if (testid != in_testid ||
+ (exp_partition != -1 && exp_partition != in_part))
+ TEST_FAIL("%s:%i: Our testid %"PRIu64", part %i did "
+ "not match message: \"%s\"\n",
+ func, line, testid, (int)exp_partition, buf);
+}
- if (testid != in_testid ||
- (exp_partition != -1 && exp_partition != in_part))
- TEST_FAIL("%s:%i: Our testid %"PRIu64", part %i did "
- "not match message: \"%s\"\n",
- func, line, testid, (int)exp_partition, buf);
+void test_msg_parse0 (const char *func, int line,
+ uint64_t testid, rd_kafka_message_t *rkmessage,
+ int32_t exp_partition, int *msgidp) {
+ test_msg_parse00(func, line, testid, exp_partition, msgidp,
+ rd_kafka_topic_name(rkmessage->rkt),
+ rkmessage->partition, rkmessage->offset,
+ (const char *)rkmessage->key, rkmessage->key_len);
}
@@ -844,14 +911,8 @@
}
}
-#ifndef _MSC_VER
- if (test_delete_topics_between && test_concurrent_max == 1) {
- if (system("./delete-test-topics.sh $ZK_ADDRESS "
- "$KAFKA_PATH/bin/kafka-topics.sh") != 0) {
- /* ignore failures but avoid unused-result warning */
- }
- }
-#endif
+ if (test_delete_topics_between && test_concurrent_max == 1)
+ test_delete_all_test_topics(60*1000);
return r;
}
@@ -1099,11 +1160,15 @@
break;
}
- if (show_summary && test->state != TEST_SKIPPED)
- printf("|%s %-40s | %10s | %7.3fs %s|%s\n",
- color,
- test->name, test_states[test->state],
- (double)duration/1000000.0, _C_CLR, extra);
+ if (show_summary && test->state != TEST_SKIPPED) {
+ printf("|%s %-40s | %10s | %7.3fs %s|",
+ color,
+ test->name, test_states[test->state],
+ (double)duration/1000000.0, _C_CLR);
+ if (test->state == TEST_FAILED)
+ printf(_C_RED " %s" _C_CLR, test->failstr);
+ printf("%s\n", extra);
+ }
if (report_fp) {
int i;
@@ -1223,6 +1288,10 @@
test_broker_version_str);
test_git_version = test_getenv("RDKAFKA_GITVER", "HEAD");
+ /* Are we running on CI? */
+ if (test_getenv("CI", NULL))
+ test_on_ci = 1;
+
test_conf_init(NULL, NULL, 10);
for (i = 1 ; i < argc ; i++) {
@@ -1238,14 +1307,14 @@
test_flags |= TEST_F_KNOWN_ISSUE;
else if (!strcmp(argv[i], "-K"))
test_neg_flags |= TEST_F_KNOWN_ISSUE;
+ else if (!strcmp(argv[i], "-E"))
+ test_neg_flags |= TEST_F_SOCKEM;
else if (!strcmp(argv[i], "-V") && i+1 < argc)
test_broker_version_str = argv[++i];
else if (!strcmp(argv[i], "-S"))
show_summary = 0;
-#ifndef _MSC_VER
else if (!strcmp(argv[i], "-D"))
test_delete_topics_between = 1;
-#endif
else if (*argv[i] != '-')
tests_to_run = argv[i];
else {
@@ -1256,12 +1325,11 @@
" -p Run N tests in parallel\n"
" -l/-L Only/dont run local tests (no broker needed)\n"
" -k/-K Only/dont run tests with known issues\n"
+ " -E Don't run sockem tests\n"
" -a Assert on failures\n"
" -S Dont show test summary\n"
" -V Broker version.\n"
-#ifndef _MSC_VER
- " -D Run delete-test-topics.sh between each test (requires -p1)\n"
-#endif
+ " -D Delete all test topics between each test (-p1) or after all tests\n"
"\n"
"Environment variables:\n"
" TESTS - substring matched test to run (e.g., 0033)\n"
@@ -1306,6 +1374,12 @@
test_curr->state = TEST_PASSED;
test_curr->start = test_clock();
+ if (test_on_ci) {
+ TEST_LOCK();
+ test_timeout_multiplier += 2;
+ TEST_UNLOCK();
+ }
+
if (!strcmp(test_mode, "helgrind") ||
!strcmp(test_mode, "drd")) {
TEST_LOCK();
@@ -1325,7 +1399,8 @@
if ((test_broker_version & 0xffff0000) == 0x00090000)
test_timeout_multiplier += 3;
- test_timeout_multiplier += (double)test_concurrent_max / 3;
+ if (test_concurrent_max > 1)
+ test_timeout_multiplier += (double)test_concurrent_max / 3;
TEST_SAY("Tests to run: %s\n", tests_to_run ? tests_to_run : "all");
TEST_SAY("Test mode : %s\n", test_mode);
@@ -1335,6 +1410,17 @@
TEST_SAY("Action on test failure: %s\n",
test_assert_on_fail ? "assert crash" : "continue other tests");
+ {
+ char cwd[512], *pcwd;
+#ifdef _MSC_VER
+ pcwd = _getcwd(cwd, sizeof(cwd) - 1);
+#else
+ pcwd = getcwd(cwd, sizeof(cwd) - 1);
+#endif
+ if (pcwd)
+ TEST_SAY("Current directory: %s\n", cwd);
+ }
+
test_timeout_set(20);
TIMING_START(&t_all, "ALL-TESTS");
@@ -1393,12 +1479,15 @@
TEST_UNLOCK();
+ if (test_delete_topics_between)
+ test_delete_all_test_topics(60*1000);
+
+ r = test_summary(1/*lock*/) ? 1 : 0;
+
/* Wait for everything to be cleaned up since broker destroys are
* handled in its own thread. */
test_wait_exit(0);
- r = test_summary(1/*lock*/) ? 1 : 0;
-
/* If we havent failed at this point then
* there were no threads leaked */
if (r == 0)
@@ -1426,15 +1515,21 @@
rd_kafka_resp_err_t err, void *opaque, void *msg_opaque) {
int *remainsp = msg_opaque;
- if (err != RD_KAFKA_RESP_ERR_NO_ERROR)
- TEST_FAIL("Message delivery failed: %s\n",
- rd_kafka_err2str(err));
+ TEST_SAYL(4, "Delivery report: %s\n", rd_kafka_err2str(err));
+
+ if (!test_curr->produce_sync && err != test_curr->exp_dr_err)
+ TEST_FAIL("Message delivery failed: expected %s, got %s",
+ rd_kafka_err2str(test_curr->exp_dr_err),
+ rd_kafka_err2str(err));
if (*remainsp == 0)
TEST_FAIL("Too many messages delivered (remains %i)",
*remainsp);
(*remainsp)--;
+
+ if (test_curr->produce_sync)
+ test_curr->produce_sync_err = err;
}
@@ -1561,10 +1656,11 @@
const char *payload, size_t size,
int *msgcounterp) {
int msg_id;
- test_timing_t t_all;
+ test_timing_t t_all, t_poll;
char key[128];
void *buf;
int64_t tot_bytes = 0;
+ int64_t tot_time_poll = 0;
if (payload)
buf = (void *)payload;
@@ -1578,8 +1674,10 @@
rd_kafka_topic_name(rkt), partition, msg_base, msg_base+cnt);
TIMING_START(&t_all, "PRODUCE");
+ TIMING_START(&t_poll, "SUM(POLL)");
for (msg_id = msg_base ; msg_id < msg_base + cnt ; msg_id++) {
+
if (!payload)
test_prepare_msg(testid, partition, msg_id,
buf, size, key, sizeof(key));
@@ -1599,7 +1697,9 @@
(*msgcounterp)++;
tot_bytes += size;
+ TIMING_RESTART(&t_poll);
rd_kafka_poll(rk, 0);
+ tot_time_poll = TIMING_DURATION(&t_poll);
if (TIMING_EVERY(&t_all, 3*1000000))
TEST_SAY("produced %3d%%: %d/%d messages "
@@ -1615,6 +1715,8 @@
if (!payload)
free(buf);
+ t_poll.duration = tot_time_poll;
+ TIMING_STOP(&t_poll);
TIMING_STOP(&t_all);
}
@@ -1642,6 +1744,10 @@
TIMING_STOP(&t_all);
+ TEST_ASSERT(*msgcounterp == 0,
+ "Not all messages delivered: msgcounter still at %d, "
+ "outq_len %d",
+ *msgcounterp, rd_kafka_outq_len(rk));
}
/**
@@ -1666,8 +1772,8 @@
* destroy consumer, and returns the used testid.
*/
uint64_t
-test_produce_msgs_easy (const char *topic, uint64_t testid,
- int32_t partition, int msgcnt) {
+test_produce_msgs_easy_size (const char *topic, uint64_t testid,
+ int32_t partition, int msgcnt, size_t size) {
rd_kafka_t *rk;
rd_kafka_topic_t *rkt;
test_timing_t t_produce;
@@ -1678,7 +1784,7 @@
rkt = test_create_producer_topic(rk, topic, NULL);
TIMING_START(&t_produce, "PRODUCE");
- test_produce_msgs(rk, rkt, testid, partition, 0, msgcnt, NULL, 0);
+ test_produce_msgs(rk, rkt, testid, partition, 0, msgcnt, NULL, size);
TIMING_STOP(&t_produce);
rd_kafka_topic_destroy(rkt);
rd_kafka_destroy(rk);
@@ -1686,6 +1792,14 @@
return testid;
}
+rd_kafka_resp_err_t test_produce_sync (rd_kafka_t *rk, rd_kafka_topic_t *rkt,
+ uint64_t testid, int32_t partition) {
+ test_curr->produce_sync = 1;
+ test_produce_msgs(rk, rkt, testid, partition, 0, 1, NULL, 0);
+ test_curr->produce_sync = 0;
+ return test_curr->produce_sync_err;
+}
+
rd_kafka_t *test_create_consumer (const char *group_id,
void (*rebalance_cb) (
@@ -2116,6 +2230,8 @@
*/
static RD_INLINE struct test_mv_m *test_mv_mvec_get (struct test_mv_mvec *mvec,
int mi) {
+ if (mi >= mvec->cnt)
+ return NULL;
return &mvec->m[mi];
}
@@ -2142,6 +2258,49 @@
/**
+ * @brief Adds a message to the msgver service.
+ *
+ * @returns 1 if message is from the expected testid, else 0 (not added)
+ */
+int test_msgver_add_msg00 (const char *func, int line, test_msgver_t *mv,
+ uint64_t testid,
+ const char *topic, int32_t partition,
+ int64_t offset, int64_t timestamp,
+ rd_kafka_resp_err_t err, int msgnum) {
+ struct test_mv_p *p;
+ struct test_mv_m *m;
+
+ if (testid != mv->testid)
+ return 0; /* Ignore message */
+
+ p = test_msgver_p_get(mv, topic, partition, 1);
+
+ if (err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
+ p->eof_offset = offset;
+ return 1;
+ }
+
+ m = test_mv_mvec_add(&p->mvec);
+
+ m->offset = offset;
+ m->msgid = msgnum;
+ m->timestamp = timestamp;
+
+ if (test_level > 2) {
+ TEST_SAY("%s:%d: "
+ "Recv msg %s [%"PRId32"] offset %"PRId64" msgid %d "
+ "timestamp %"PRId64"\n",
+ func, line,
+ p->topic, p->partition, m->offset, m->msgid,
+ m->timestamp);
+ }
+
+ mv->msgcnt++;
+
+ return 1;
+}
+
+/**
* Adds a message to the msgver service.
*
* Message must be a proper message or PARTITION_EOF.
@@ -2152,10 +2311,13 @@
test_msgver_t *mv, rd_kafka_message_t *rkmessage) {
uint64_t in_testid;
int in_part;
- int in_msgnum;
+ int in_msgnum = -1;
char buf[128];
- struct test_mv_p *p;
- struct test_mv_m *m;
+ const void *val;
+ size_t valsize;
+
+ if (mv->fwd)
+ test_msgver_add_msg(mv->fwd, rkmessage);
if (rkmessage->err) {
if (rkmessage->err != RD_KAFKA_RESP_ERR__PARTITION_EOF)
@@ -2164,47 +2326,47 @@
in_testid = mv->testid;
} else {
- rd_snprintf(buf, sizeof(buf), "%.*s",
- (int)rkmessage->len, (char *)rkmessage->payload);
-
- if (sscanf(buf, "testid=%"SCNu64", partition=%i, msg=%i",
- &in_testid, &in_part, &in_msgnum) != 3)
- TEST_FAIL("%s:%d: Incorrect format at offset %"PRId64
- ": %s",
- func, line, rkmessage->offset, buf);
- }
-
- if (mv->fwd)
- test_msgver_add_msg(mv->fwd, rkmessage);
-
- if (in_testid != mv->testid)
- return 0; /* Ignore message */
- p = test_msgver_p_get(mv, rd_kafka_topic_name(rkmessage->rkt),
- rkmessage->partition, 1);
-
- if (rkmessage->err == RD_KAFKA_RESP_ERR__PARTITION_EOF) {
- p->eof_offset = rkmessage->offset;
- return 1;
- }
+ if (!mv->msgid_hdr) {
+ rd_snprintf(buf, sizeof(buf), "%.*s",
+ (int)rkmessage->len,
+ (char *)rkmessage->payload);
+ val = buf;
+ } else {
+ /* msgid is in message header */
+ rd_kafka_headers_t *hdrs;
- m = test_mv_mvec_add(&p->mvec);
+ if (rd_kafka_message_headers(rkmessage, &hdrs) ||
+ rd_kafka_header_get_last(hdrs, mv->msgid_hdr,
+ &val, &valsize)) {
+ TEST_SAYL(3,
+ "%s:%d: msgid expected in header %s "
+ "but %s exists for "
+ "message at offset %"PRId64
+ " has no headers",
+ func, line, mv->msgid_hdr,
+ hdrs ? "no such header" : "no headers",
+ rkmessage->offset);
- m->offset = rkmessage->offset;
- m->msgid = in_msgnum;
- m->timestamp = rd_kafka_message_timestamp(rkmessage, NULL);
-
- if (test_level > 2) {
- TEST_SAY("%s:%d: "
- "Recv msg %s [%"PRId32"] offset %"PRId64" msgid %d "
- "timestamp %"PRId64"\n",
- func, line,
- p->topic, p->partition, m->offset, m->msgid,
- m->timestamp);
- }
+ return 0;
+ }
+ }
- mv->msgcnt++;
+ if (sscanf(val, "testid=%"SCNu64", partition=%i, msg=%i\n",
+ &in_testid, &in_part, &in_msgnum) != 3)
+ TEST_FAIL("%s:%d: Incorrect format at offset %"PRId64
+ ": %s",
+ func, line, rkmessage->offset,
+ (const char *)val);
+ }
+ return test_msgver_add_msg00(func, line, mv, in_testid,
+ rd_kafka_topic_name(rkmessage->rkt),
+ rkmessage->partition,
+ rkmessage->offset,
+ rd_kafka_message_timestamp(rkmessage, NULL),
+ rkmessage->err,
+ in_msgnum);
return 1;
}
@@ -2250,6 +2412,84 @@
}
+/**
+ * @brief Verify that messages correspond to 'correct' msgver.
+ */
+static int test_mv_mvec_verify_corr (test_msgver_t *mv, int flags,
+ struct test_mv_p *p,
+ struct test_mv_mvec *mvec,
+ struct test_mv_vs *vs) {
+ int mi;
+ int fails = 0;
+ struct test_mv_p *corr_p = NULL;
+ struct test_mv_mvec *corr_mvec;
+
+ TEST_ASSERT(vs->corr);
+
+ /* Get correct mvec for comparison. */
+ if (p)
+ corr_p = test_msgver_p_get(vs->corr, p->topic, p->partition, 0);
+ if (!corr_p) {
+ TEST_MV_WARN(mv,
+ " %s [%"PRId32"]: "
+ "no corresponding correct partition found\n",
+ p ? p->topic : "*",
+ p ? p->partition : -1);
+ return 1;
+ }
+
+ corr_mvec = &corr_p->mvec;
+
+ for (mi = 0 ; mi < mvec->cnt ; mi++) {
+ struct test_mv_m *this = test_mv_mvec_get(mvec, mi);
+ const struct test_mv_m *corr = test_mv_mvec_get(corr_mvec, mi);
+
+ if (0)
+ TEST_MV_WARN(mv,
+ "msg #%d: msgid %d, offset %"PRId64"\n",
+ mi, this->msgid, this->offset);
+ if (!corr) {
+ TEST_MV_WARN(
+ mv,
+ " %s [%"PRId32"] msg rcvidx #%d/%d: "
+ "out of range: correct mvec has %d messages: "
+ "message offset %"PRId64", msgid %d\n",
+ p ? p->topic : "*",
+ p ? p->partition : -1,
+ mi, mvec->cnt, corr_mvec->cnt,
+ this->offset, this->msgid);
+ fails++;
+ continue;
+ }
+
+ if (((flags & TEST_MSGVER_BY_OFFSET) &&
+ this->offset != corr->offset) ||
+ ((flags & TEST_MSGVER_BY_MSGID) &&
+ this->msgid != corr->msgid) ||
+ ((flags & TEST_MSGVER_BY_TIMESTAMP) &&
+ this->timestamp != corr->timestamp)) {
+ TEST_MV_WARN(
+ mv,
+ " %s [%"PRId32"] msg rcvidx #%d/%d: "
+ "did not match correct msg: "
+ "offset %"PRId64" vs %"PRId64", "
+ "msgid %d vs %d, "
+ "timestamp %"PRId64" vs %"PRId64" (fl 0x%x)\n",
+ p ? p->topic : "*",
+ p ? p->partition : -1,
+ mi, mvec->cnt,
+ this->offset, corr->offset,
+ this->msgid, corr->msgid,
+ this->timestamp, corr->timestamp,
+ flags);
+ fails++;
+ }
+ }
+
+ return fails;
+}
+
+
static int test_mv_m_cmp_offset (const void *_a, const void *_b) {
const struct test_mv_m *a = _a, *b = _b;
@@ -2658,7 +2898,7 @@
rd_snprintf(buf, sizeof(buf), "%.*s",
(int)rkmessage->len, (char *)rkmessage->payload);
- if (sscanf(buf, "testid=%"SCNu64", partition=%i, msg=%i",
+ if (sscanf(buf, "testid=%"SCNu64", partition=%i, msg=%i\n",
&in_testid, &in_part, &in_msgnum) != 3)
TEST_FAIL("Incorrect format: %s", buf);
@@ -2687,6 +2927,51 @@
/**
+ * @brief Verify that \p mv is identical to \p corr according to flags.
+ */
+void test_msgver_verify_compare0 (const char *func, int line,
+ const char *what, test_msgver_t *mv,
+ test_msgver_t *corr, int flags) {
+ struct test_mv_vs vs;
+ int fails = 0;
+
+ memset(&vs, 0, sizeof(vs));
+
+ TEST_SAY("%s:%d: %s: Verifying %d received messages (flags 0x%x) by "
+ "comparison to correct msgver (%d messages)\n",
+ func, line, what, mv->msgcnt, flags, corr->msgcnt);
+
+ vs.corr = corr;
+
+ /* Per-partition checks */
+ fails += test_mv_p_verify_f(mv, flags,
+ test_mv_mvec_verify_corr, &vs);
+
+ if (mv->log_suppr_cnt > 0)
+ TEST_WARN("%s:%d: %s: %d message warning logs suppressed\n",
+ func, line, what, mv->log_suppr_cnt);
+
+ if (corr->msgcnt != mv->msgcnt) {
+ TEST_WARN("%s:%d: %s: expected %d messages, got %d\n",
+ func, line, what, corr->msgcnt, mv->msgcnt);
+ fails++;
+ }
+
+ if (fails)
+ TEST_FAIL("%s:%d: %s: Verification of %d received messages "
+ "failed: expected %d messages: see previous errors\n",
+ func, line, what,
+ mv->msgcnt, corr->msgcnt);
+ else
+ TEST_SAY("%s:%d: %s: Verification of %d received messages "
+ "succeeded: matching %d messages from correct msgver\n",
+ func, line, what,
+ mv->msgcnt, corr->msgcnt);
+
+}
+
+
+/**
* Consumer poll but dont expect any proper messages for \p timeout_ms.
*/
void test_consumer_poll_no_msgs (const char *what, rd_kafka_t *rk,
@@ -2813,7 +3098,7 @@
TIMING_START(&t_cons, "CONSUME");
- while ((exp_eof_cnt == -1 || eof_cnt < exp_eof_cnt) &&
+ while ((exp_eof_cnt <= 0 || eof_cnt < exp_eof_cnt) &&
(exp_cnt == -1 || cnt < exp_cnt)) {
rd_kafka_message_t *rkmessage;
@@ -2830,6 +3115,7 @@
rd_kafka_topic_name(rkmessage->rkt),
rkmessage->partition,
rkmessage->offset);
+ TEST_ASSERT(exp_eof_cnt != 0, "expected no EOFs");
if (mv)
test_msgver_add_msg(mv, rkmessage);
eof_cnt++;
@@ -2884,9 +3170,10 @@
err = rd_kafka_flush(rk, timeout_ms);
TIMING_STOP(&timing);
if (err)
- TEST_FAIL("Failed to flush(%s, %d): %s\n",
+ TEST_FAIL("Failed to flush(%s, %d): %s: len() = %d\n",
rd_kafka_name(rk), timeout_ms,
- rd_kafka_err2str(err));
+ rd_kafka_err2str(err),
+ rd_kafka_outq_len(rk));
}
@@ -3018,11 +3305,90 @@
}
+
+/**
+ * @brief Create topic using Topic Admin API
+ */
+static void test_admin_create_topic (rd_kafka_t *use_rk,
+ const char *topicname, int partition_cnt,
+ int replication_factor) {
+ rd_kafka_t *rk;
+ rd_kafka_NewTopic_t *newt[1];
+ const size_t newt_cnt = 1;
+ rd_kafka_AdminOptions_t *options;
+ rd_kafka_queue_t *rkqu;
+ rd_kafka_event_t *rkev;
+ const rd_kafka_CreateTopics_result_t *res;
+ const rd_kafka_topic_result_t **terr;
+ int timeout_ms = tmout_multip(10000);
+ size_t res_cnt;
+ rd_kafka_resp_err_t err;
+ char errstr[512];
+ test_timing_t t_create;
+
+ if (!(rk = use_rk))
+ rk = test_create_producer();
+
+ rkqu = rd_kafka_queue_new(rk);
+
+ newt[0] = rd_kafka_NewTopic_new(topicname, partition_cnt,
+ replication_factor,
+ errstr, sizeof(errstr));
+ TEST_ASSERT(newt[0] != NULL, "%s", errstr);
+
+ options = rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_CREATETOPICS);
+ err = rd_kafka_AdminOptions_set_operation_timeout(options, timeout_ms,
+ errstr,
+ sizeof(errstr));
+ TEST_ASSERT(!err, "%s", errstr);
+
+ TEST_SAY("Creating topic \"%s\" "
+ "(partitions=%d, replication_factor=%d, timeout=%d)\n",
+ topicname, partition_cnt, replication_factor, timeout_ms);
+
+ TIMING_START(&t_create, "CreateTopics");
+ rd_kafka_CreateTopics(rk, newt, newt_cnt, options, rkqu);
+
+ /* Wait for result */
+ rkev = rd_kafka_queue_poll(rkqu, timeout_ms + 2000);
+ TEST_ASSERT(rkev, "Timed out waiting for CreateTopics result");
+
+ TIMING_STOP(&t_create);
+
+ res = rd_kafka_event_CreateTopics_result(rkev);
+ TEST_ASSERT(res, "Expected CreateTopics_result, not %s",
+ rd_kafka_event_name(rkev));
+
+ terr = rd_kafka_CreateTopics_result_topics(res, &res_cnt);
+ TEST_ASSERT(terr, "CreateTopics_result_topics returned NULL");
+ TEST_ASSERT(res_cnt == newt_cnt,
+ "CreateTopics_result_topics returned %"PRIusz" topics, "
+ "not the expected %"PRIusz,
+ res_cnt, newt_cnt);
+
+ TEST_ASSERT(!rd_kafka_topic_result_error(terr[0]),
+ "Topic %s result error: %s",
+ rd_kafka_topic_result_name(terr[0]),
+ rd_kafka_topic_result_error_string(terr[0]));
+
+ rd_kafka_event_destroy(rkev);
+
+ rd_kafka_queue_destroy(rkqu);
+
+ rd_kafka_AdminOptions_destroy(options);
+
+ rd_kafka_NewTopic_destroy(newt[0]);
+
+ if (!use_rk)
+ rd_kafka_destroy(rk);
+}
+
+
/**
* @brief Create topic using kafka-topics.sh --create
*/
-void test_create_topic (const char *topicname, int partition_cnt,
- int replication_factor) {
+static void test_create_topic_sh (const char *topicname, int partition_cnt,
+ int replication_factor) {
test_kafka_topics("--create --topic \"%s\" "
"--replication-factor %d --partitions %d",
topicname, replication_factor, partition_cnt);
@@ -3030,33 +3396,119 @@
/**
+ * @brief Create topic
+ */
+void test_create_topic (const char *topicname, int partition_cnt,
+ int replication_factor) {
+ if (test_broker_version < TEST_BRKVER(0,10,2,0))
+ test_create_topic_sh(topicname, partition_cnt,
+ replication_factor);
+ else
+ test_admin_create_topic(NULL, topicname, partition_cnt,
+ replication_factor);
+}
+
+
+int test_get_partition_count (rd_kafka_t *rk, const char *topicname,
+ int timeout_ms) {
+ rd_kafka_t *use_rk;
+ rd_kafka_resp_err_t err;
+ rd_kafka_topic_t *rkt;
+ int64_t abs_timeout = test_clock() + (timeout_ms * 1000);
+
+ if (!rk)
+ use_rk = test_create_producer();
+ else
+ use_rk = rk;
+
+ rkt = rd_kafka_topic_new(use_rk, topicname, NULL);
+
+ do {
+ const struct rd_kafka_metadata *metadata;
+
+ err = rd_kafka_metadata(use_rk, 0, rkt, &metadata,
+ tmout_multip(15000));
+ if (err)
+ TEST_WARN("metadata() for %s failed: %s\n",
+ rkt ? rd_kafka_topic_name(rkt) :
+ "(all-local)",
+ rd_kafka_err2str(err));
+ else {
+ if (metadata->topic_cnt == 1) {
+ if (metadata->topics[0].err == 0 ||
+ metadata->topics[0].partition_cnt > 0) {
+ int32_t cnt;
+ cnt = metadata->topics[0].partition_cnt;
+ rd_kafka_metadata_destroy(metadata);
+ rd_kafka_topic_destroy(rkt);
+ return (int)cnt;
+ }
+ TEST_SAY("metadata(%s) returned %s: retrying\n",
+ rd_kafka_topic_name(rkt),
+ rd_kafka_err2str(metadata->
+ topics[0].err));
+ }
+ rd_kafka_metadata_destroy(metadata);
+ rd_sleep(1);
+ }
+ } while (test_clock() < abs_timeout);
+
+ rd_kafka_topic_destroy(rkt);
+
+ if (!rk)
+ rd_kafka_destroy(use_rk);
+
+ return -1;
+}
+
+/**
* @brief Let the broker auto-create the topic for us.
*/
rd_kafka_resp_err_t test_auto_create_topic_rkt (rd_kafka_t *rk,
- rd_kafka_topic_t *rkt) {
+ rd_kafka_topic_t *rkt,
+ int timeout_ms) {
const struct rd_kafka_metadata *metadata;
rd_kafka_resp_err_t err;
test_timing_t t;
+ int64_t abs_timeout = test_clock() + (timeout_ms * 1000);
- TIMING_START(&t, "auto_create_topic");
- err = rd_kafka_metadata(rk, 0, rkt, &metadata, tmout_multip(15000));
- TIMING_STOP(&t);
- if (err)
- TEST_WARN("metadata() for %s failed: %s",
- rkt ? rd_kafka_topic_name(rkt) : "(all-local)",
- rd_kafka_err2str(err));
- else
- rd_kafka_metadata_destroy(metadata);
+ do {
+ TIMING_START(&t, "auto_create_topic");
+ err = rd_kafka_metadata(rk, 0, rkt, &metadata,
+ tmout_multip(15000));
+ TIMING_STOP(&t);
+ if (err)
+ TEST_WARN("metadata() for %s failed: %s\n",
+ rkt ? rd_kafka_topic_name(rkt) :
+ "(all-local)",
+ rd_kafka_err2str(err));
+ else {
+ if (metadata->topic_cnt == 1) {
+ if (metadata->topics[0].err == 0 ||
+ metadata->topics[0].partition_cnt > 0) {
+ rd_kafka_metadata_destroy(metadata);
+ return 0;
+ }
+ TEST_SAY("metadata(%s) returned %s: retrying\n",
+ rd_kafka_topic_name(rkt),
+ rd_kafka_err2str(metadata->
+ topics[0].err));
+ }
+ rd_kafka_metadata_destroy(metadata);
+ rd_sleep(1);
+ }
+ } while (test_clock() < abs_timeout);
return err;
}
-rd_kafka_resp_err_t test_auto_create_topic (rd_kafka_t *rk, const char *name) {
+rd_kafka_resp_err_t test_auto_create_topic (rd_kafka_t *rk, const char *name,
+ int timeout_ms) {
rd_kafka_topic_t *rkt = rd_kafka_topic_new(rk, name, NULL);
rd_kafka_resp_err_t err;
if (!rkt)
return rd_kafka_last_error();
- err = test_auto_create_topic_rkt(rk, rkt);
+ err = test_auto_create_topic_rkt(rk, rkt, timeout_ms);
rd_kafka_topic_destroy(rkt);
return err;
}
@@ -3074,7 +3526,7 @@
test_conf_init(&conf, NULL, 0);
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
- err = test_auto_create_topic(rk, topic);
+ err = test_auto_create_topic(rk, topic, tmout_multip(5000));
if (err)
TEST_SAY("Auto topic creation of \"%s\" failed: %s\n",
topic, rd_kafka_err2str(err));
@@ -3161,7 +3613,7 @@
int test_can_create_topics (int skip) {
#ifdef _MSC_VER
if (skip)
- TEST_SKIP("Cannot create topics on Win32");
+ TEST_SKIP("Cannot create topics on Win32\n");
return 0;
#else
@@ -3228,7 +3680,871 @@
TEST_SAYL(level, "%s", str);
}
+void test_SKIP (const char *file, int line, const char *str) {
+ TEST_WARN("SKIPPING TEST: %s", str);
+ TEST_LOCK();
+ test_curr->state = TEST_SKIPPED;
+ TEST_UNLOCK();
+}
const char *test_curr_name (void) {
return test_curr->name;
}
+
+
+/**
+ * @brief Dump/print message haders
+ */
+void test_headers_dump (const char *what, int lvl,
+ const rd_kafka_headers_t *hdrs) {
+ size_t idx = 0;
+ const char *name, *value;
+ size_t size;
+
+ while (!rd_kafka_header_get_all(hdrs, idx++, &name,
+ (const void **)&value, &size))
+ TEST_SAYL(lvl, "%s: Header #%"PRIusz": %s='%s'\n",
+ what, idx-1, name,
+ value ? value : "(NULL)");
+}
+
+
+/**
+ * @brief Retrieve and return the list of broker ids in the cluster.
+ *
+ * @param rk Optional instance to use.
+ * @param cntp Will be updated to the number of brokers returned.
+ *
+ * @returns a malloc:ed list of int32_t broker ids.
+ */
+int32_t *test_get_broker_ids (rd_kafka_t *use_rk, size_t *cntp) {
+ int32_t *ids;
+ rd_kafka_t *rk;
+ const rd_kafka_metadata_t *md;
+ rd_kafka_resp_err_t err;
+ size_t i;
+
+ if (!(rk = use_rk))
+ rk = test_create_producer();
+
+ err = rd_kafka_metadata(rk, 0, NULL, &md, tmout_multip(5000));
+ TEST_ASSERT(!err, "%s", rd_kafka_err2str(err));
+ TEST_ASSERT(md->broker_cnt > 0,
+ "%d brokers, expected > 0", md->broker_cnt);
+
+ ids = malloc(sizeof(*ids) * md->broker_cnt);
+
+ for (i = 0 ; i < (size_t)md->broker_cnt ; i++)
+ ids[i] = md->brokers[i].id;
+
+ *cntp = md->broker_cnt;
+
+ rd_kafka_metadata_destroy(md);
+
+ if (!use_rk)
+ rd_kafka_destroy(rk);
+
+ return ids;
+}
+
+
+
+/**
+ * @brief Verify that all topics in \p topics are reported in metadata,
+ * and that none of the topics in \p not_topics are reported.
+ *
+ * @returns the number of failures (but does not FAIL).
+ */
+static int verify_topics_in_metadata (rd_kafka_t *rk,
+ rd_kafka_metadata_topic_t *topics,
+ size_t topic_cnt,
+ rd_kafka_metadata_topic_t *not_topics,
+ size_t not_topic_cnt) {
+ const rd_kafka_metadata_t *md;
+ rd_kafka_resp_err_t err;
+ int ti;
+ size_t i;
+ int fails = 0;
+
+ /* Mark topics with dummy error which is overwritten
+ * when topic is found in metadata, allowing us to check
+ * for missed topics. */
+ for (i = 0 ; i < topic_cnt ; i++)
+ topics[i].err = 12345;
+
+ err = rd_kafka_metadata(rk, 1/*all_topics*/, NULL, &md,
+ tmout_multip(5000));
+ TEST_ASSERT(!err, "metadata failed: %s", rd_kafka_err2str(err));
+
+ for (ti = 0 ; ti < md->topic_cnt ; ti++) {
+ const rd_kafka_metadata_topic_t *mdt = &md->topics[ti];
+
+ for (i = 0 ; i < topic_cnt ; i++) {
+ int pi;
+ rd_kafka_metadata_topic_t *exp_mdt;
+
+ if (strcmp(topics[i].topic, mdt->topic))
+ continue;
+
+ exp_mdt = &topics[i];
+
+ exp_mdt->err = mdt->err; /* indicate found */
+ if (mdt->err) {
+ TEST_SAY("metadata: "
+ "Topic %s has error %s\n",
+ mdt->topic,
+ rd_kafka_err2str(mdt->err));
+ fails++;
+ }
+
+ if (exp_mdt->partition_cnt != 0 &&
+ mdt->partition_cnt != exp_mdt->partition_cnt) {
+ TEST_SAY("metadata: "
+ "Topic %s, expected %d partitions"
+ ", not %d\n",
+ mdt->topic,
+ exp_mdt->partition_cnt,
+ mdt->partition_cnt);
+ fails++;
+ continue;
+ }
+
+ /* Verify per-partition values */
+ for (pi = 0 ; exp_mdt->partitions &&
+ pi < exp_mdt->partition_cnt ; pi++) {
+ const rd_kafka_metadata_partition_t *mdp =
+ &mdt->partitions[pi];
+ const rd_kafka_metadata_partition_t *exp_mdp =
+ &exp_mdt->partitions[pi];
+
+ if (mdp->id != exp_mdp->id) {
+ TEST_SAY("metadata: "
+ "Topic %s, "
+ "partition %d, "
+ "partition list out of order,"
+ " expected %d, not %d\n",
+ mdt->topic, pi,
+ exp_mdp->id, mdp->id);
+ fails++;
+ continue;
+ }
+
+ if (exp_mdp->replicas) {
+ if (mdp->replica_cnt !=
+ exp_mdp->replica_cnt) {
+ TEST_SAY("metadata: "
+ "Topic %s, "
+ "partition %d, "
+ "expected %d replicas,"
+ " not %d\n",
+ mdt->topic, pi,
+ exp_mdp->replica_cnt,
+ mdp->replica_cnt);
+ fails++;
+ } else if (memcmp(mdp->replicas,
+ exp_mdp->replicas,
+ mdp->replica_cnt *
+ sizeof(*mdp->replicas))) {
+ int ri;
+
+ TEST_SAY("metadata: "
+ "Topic %s, "
+ "partition %d, "
+ "replica mismatch:\n",
+ mdt->topic, pi);
+
+ for (ri = 0 ;
+ ri < mdp->replica_cnt ;
+ ri++) {
+ TEST_SAY(" #%d: "
+ "expected "
+ "replica %d, "
+ "not %d\n",
+ ri,
+ exp_mdp->
+ replicas[ri],
+ mdp->
+ replicas[ri]);
+ }
+
+ fails++;
+ }
+
+ }
+ }
+ }
+
+ for (i = 0 ; i < not_topic_cnt ; i++) {
+ if (strcmp(not_topics[i].topic, mdt->topic))
+ continue;
+
+ TEST_SAY("metadata: "
+ "Topic %s found in metadata, unexpected\n",
+ mdt->topic);
+ fails++;
+ }
+
+ }
+
+ for (i = 0 ; i < topic_cnt ; i++) {
+ if ((int)topics[i].err == 12345) {
+ TEST_SAY("metadata: "
+ "Topic %s not seen in metadata\n",
+ topics[i].topic);
+ fails++;
+ }
+ }
+
+ if (fails > 0)
+ TEST_SAY("Metadata verification for %"PRIusz" topics failed "
+ "with %d errors (see above)\n",
+ topic_cnt, fails);
+ else
+ TEST_SAY("Metadata verification succeeded: "
+ "%"PRIusz" desired topics seen, "
+ "%"PRIusz" undesired topics not seen\n",
+ topic_cnt, not_topic_cnt);
+
+ rd_kafka_metadata_destroy(md);
+
+ return fails;
+}
+
+
+
+/**
+ * @brief Wait for metadata to reflect expected and not expected topics
+ */
+void test_wait_metadata_update (rd_kafka_t *rk,
+ rd_kafka_metadata_topic_t *topics,
+ size_t topic_cnt,
+ rd_kafka_metadata_topic_t *not_topics,
+ size_t not_topic_cnt,
+ int tmout) {
+ int64_t abs_timeout;
+ test_timing_t t_md;
+
+ abs_timeout = test_clock() + (tmout * 1000);
+
+ test_timeout_set(10 + (tmout/1000));
+
+ TEST_SAY("Waiting for up to %dms for metadata update\n", tmout);
+
+ TIMING_START(&t_md, "METADATA.WAIT");
+ do {
+ int md_fails;
+
+ md_fails = verify_topics_in_metadata(
+ rk,
+ topics, topic_cnt,
+ not_topics, not_topic_cnt);
+
+ if (!md_fails) {
+ TEST_SAY("All expected topics (not?) "
+ "seen in metadata\n");
+ abs_timeout = 0;
+ break;
+ }
+
+ rd_sleep(1);
+ } while (test_clock() < abs_timeout);
+ TIMING_STOP(&t_md);
+
+ if (abs_timeout)
+ TEST_FAIL("Expected topics not seen in given time.");
+}
+
+
+
+/**
+ * @brief Wait for up to \p tmout for any type of admin result.
+ * @returns the event
+ */
+rd_kafka_event_t *
+test_wait_admin_result (rd_kafka_queue_t *q,
+ rd_kafka_event_type_t evtype,
+ int tmout) {
+ rd_kafka_event_t *rkev;
+
+ while (1) {
+ rkev = rd_kafka_queue_poll(q, tmout);
+ if (!rkev)
+ TEST_FAIL("Timed out waiting for admin result (%d)\n",
+ evtype);
+
+ if (rd_kafka_event_type(rkev) == evtype)
+ return rkev;
+
+
+ if (rd_kafka_event_type(rkev) == RD_KAFKA_EVENT_ERROR) {
+ TEST_WARN("Received error event while waiting for %d: "
+ "%s: ignoring",
+ evtype, rd_kafka_event_error_string(rkev));
+ continue;
+ }
+
+
+ TEST_ASSERT(rd_kafka_event_type(rkev) == evtype,
+ "Expected event type %d, got %d (%s)",
+ evtype,
+ rd_kafka_event_type(rkev),
+ rd_kafka_event_name(rkev));
+ }
+
+ return NULL;
+}
+
+
+
+/**
+ * @brief Wait for up to \p tmout for a
+ * CreateTopics/DeleteTopics/CreatePartitions result
+ * and return the distilled error code.
+ */
+rd_kafka_resp_err_t
+test_wait_topic_admin_result (rd_kafka_queue_t *q,
+ rd_kafka_event_type_t evtype,
+ int tmout) {
+ rd_kafka_event_t *rkev;
+ size_t i;
+ const rd_kafka_topic_result_t **terr = NULL;
+ size_t terr_cnt = 0;
+ int errcnt = 0;
+ rd_kafka_resp_err_t err;
+
+ rkev = test_wait_admin_result(q, evtype, tmout);
+
+ if ((err = rd_kafka_event_error(rkev))) {
+ TEST_WARN("%s failed: %s\n",
+ rd_kafka_event_name(rkev),
+ rd_kafka_event_error_string(rkev));
+ rd_kafka_event_destroy(rkev);
+ return err;
+ }
+
+ if (evtype == RD_KAFKA_EVENT_CREATETOPICS_RESULT) {
+ const rd_kafka_CreateTopics_result_t *res;
+ if (!(res = rd_kafka_event_CreateTopics_result(rkev)))
+ TEST_FAIL("Expected a CreateTopics result, not %s",
+ rd_kafka_event_name(rkev));
+
+ terr = rd_kafka_CreateTopics_result_topics(res, &terr_cnt);
+
+ } else if (evtype == RD_KAFKA_EVENT_DELETETOPICS_RESULT) {
+ const rd_kafka_DeleteTopics_result_t *res;
+ if (!(res = rd_kafka_event_DeleteTopics_result(rkev)))
+ TEST_FAIL("Expected a DeleteTopics result, not %s",
+ rd_kafka_event_name(rkev));
+
+ terr = rd_kafka_DeleteTopics_result_topics(res, &terr_cnt);
+
+ } else if (evtype == RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT) {
+ const rd_kafka_CreatePartitions_result_t *res;
+ if (!(res = rd_kafka_event_CreatePartitions_result(rkev)))
+ TEST_FAIL("Expected a CreatePartitions result, not %s",
+ rd_kafka_event_name(rkev));
+
+ terr = rd_kafka_CreatePartitions_result_topics(res, &terr_cnt);
+
+ } else {
+ TEST_FAIL("Bad evtype: %d", evtype);
+ RD_NOTREACHED();
+ }
+
+ for (i = 0 ; i < terr_cnt ; i++) {
+ if (rd_kafka_topic_result_error(terr[i])) {
+ TEST_WARN("..Topics result: %s: error: %s\n",
+ rd_kafka_topic_result_name(terr[i]),
+ rd_kafka_topic_result_error_string(terr[i]));
+ if (!(errcnt++))
+ err = rd_kafka_topic_result_error(terr[i]);
+ }
+ }
+
+ rd_kafka_event_destroy(rkev);
+
+ return err;
+}
+
+
+
+/**
+ * @brief Topic Admin API helpers
+ *
+ * @param useq Makes the call async and posts the response in this queue.
+ * If NULL this call will be synchronous and return the error
+ * result.
+ *
+ * @remark Fails the current test on failure.
+ */
+
+rd_kafka_resp_err_t
+test_CreateTopics_simple (rd_kafka_t *rk,
+ rd_kafka_queue_t *useq,
+ char **topics, size_t topic_cnt,
+ int num_partitions,
+ void *opaque) {
+ rd_kafka_NewTopic_t **new_topics;
+ rd_kafka_AdminOptions_t *options;
+ rd_kafka_queue_t *q;
+ size_t i;
+ const int tmout = 30 * 1000;
+ rd_kafka_resp_err_t err;
+
+ new_topics = malloc(sizeof(*new_topics) * topic_cnt);
+
+ for (i = 0 ; i < topic_cnt ; i++) {
+ char errstr[512];
+ new_topics[i] = rd_kafka_NewTopic_new(topics[i],
+ num_partitions, 1,
+ errstr, sizeof(errstr));
+ TEST_ASSERT(new_topics[i],
+ "Failed to NewTopic(\"%s\", %d) #%"PRIusz": %s",
+ topics[i], num_partitions, i, errstr);
+ }
+
+ options = rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_CREATETOPICS);
+ rd_kafka_AdminOptions_set_opaque(options, opaque);
+
+ if (!useq) {
+ char errstr[512];
+
+ err = rd_kafka_AdminOptions_set_request_timeout(options,
+ tmout,
+ errstr,
+ sizeof(errstr));
+ TEST_ASSERT(!err, "set_request_timeout: %s", errstr);
+ err = rd_kafka_AdminOptions_set_operation_timeout(options,
+ tmout-5000,
+ errstr,
+ sizeof(errstr));
+ TEST_ASSERT(!err, "set_operation_timeout: %s", errstr);
+
+ q = rd_kafka_queue_new(rk);
+ } else {
+ q = useq;
+ }
+
+ TEST_SAY("Creating %"PRIusz" topics\n", topic_cnt);
+
+ rd_kafka_CreateTopics(rk, new_topics, topic_cnt, options, q);
+
+ rd_kafka_AdminOptions_destroy(options);
+
+ rd_kafka_NewTopic_destroy_array(new_topics, topic_cnt);
+ free(new_topics);
+
+ if (useq)
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+
+ err = test_wait_topic_admin_result(q,
+ RD_KAFKA_EVENT_CREATETOPICS_RESULT,
+ tmout+5000);
+
+ rd_kafka_queue_destroy(q);
+
+ if (err)
+ TEST_FAIL("Failed to create %d topic(s): %s",
+ (int)topic_cnt, rd_kafka_err2str(err));
+
+ return err;
+}
+
+
+rd_kafka_resp_err_t
+test_CreatePartitions_simple (rd_kafka_t *rk,
+ rd_kafka_queue_t *useq,
+ const char *topic,
+ size_t total_part_cnt,
+ void *opaque) {
+ rd_kafka_NewPartitions_t *newp[1];
+ rd_kafka_AdminOptions_t *options;
+ rd_kafka_queue_t *q;
+ const int tmout = 30 * 1000;
+ rd_kafka_resp_err_t err;
+ char errstr[512];
+
+ newp[0] = rd_kafka_NewPartitions_new(topic, total_part_cnt, errstr,
+ sizeof(errstr));
+ TEST_ASSERT(newp[0],
+ "Failed to NewPartitions(\"%s\", %"PRIusz"): %s",
+ topic, total_part_cnt, errstr);
+
+ options = rd_kafka_AdminOptions_new(rk,
+ RD_KAFKA_ADMIN_OP_CREATEPARTITIONS);
+ rd_kafka_AdminOptions_set_opaque(options, opaque);
+
+ if (!useq) {
+ char errstr[512];
+
+ err = rd_kafka_AdminOptions_set_request_timeout(options,
+ tmout,
+ errstr,
+ sizeof(errstr));
+ TEST_ASSERT(!err, "set_request_timeout: %s", errstr);
+ err = rd_kafka_AdminOptions_set_operation_timeout(options,
+ tmout-5000,
+ errstr,
+ sizeof(errstr));
+ TEST_ASSERT(!err, "set_operation_timeout: %s", errstr);
+
+ q = rd_kafka_queue_new(rk);
+ } else {
+ q = useq;
+ }
+
+ TEST_SAY("Creating (up to) %"PRIusz" partitions for topic \"%s\"\n",
+ total_part_cnt, topic);
+
+ rd_kafka_CreatePartitions(rk, newp, 1, options, q);
+
+ rd_kafka_AdminOptions_destroy(options);
+
+ rd_kafka_NewPartitions_destroy(newp[0]);
+
+ if (useq)
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+
+ err = test_wait_topic_admin_result(
+ q, RD_KAFKA_EVENT_CREATEPARTITIONS_RESULT, tmout+5000);
+
+ rd_kafka_queue_destroy(q);
+
+ if (err)
+ TEST_FAIL("Failed to create partitions: %s",
+ rd_kafka_err2str(err));
+
+ return err;
+}
+
+
+rd_kafka_resp_err_t
+test_DeleteTopics_simple (rd_kafka_t *rk,
+ rd_kafka_queue_t *useq,
+ char **topics, size_t topic_cnt,
+ void *opaque) {
+ rd_kafka_queue_t *q;
+ rd_kafka_DeleteTopic_t **del_topics;
+ rd_kafka_AdminOptions_t *options;
+ size_t i;
+ rd_kafka_resp_err_t err;
+ const int tmout = 30*1000;
+
+ del_topics = malloc(sizeof(*del_topics) * topic_cnt);
+
+ for (i = 0 ; i < topic_cnt ; i++) {
+ del_topics[i] = rd_kafka_DeleteTopic_new(topics[i]);
+ TEST_ASSERT(del_topics[i]);
+ }
+
+ options = rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_DELETETOPICS);
+ rd_kafka_AdminOptions_set_opaque(options, opaque);
+
+ if (!useq) {
+ char errstr[512];
+
+ err = rd_kafka_AdminOptions_set_request_timeout(options,
+ tmout,
+ errstr,
+ sizeof(errstr));
+ TEST_ASSERT(!err, "set_request_timeout: %s", errstr);
+ err = rd_kafka_AdminOptions_set_operation_timeout(options,
+ tmout-5000,
+ errstr,
+ sizeof(errstr));
+ TEST_ASSERT(!err, "set_operation_timeout: %s", errstr);
+
+ q = rd_kafka_queue_new(rk);
+ } else {
+ q = useq;
+ }
+
+ TEST_SAY("Deleting %"PRIusz" topics\n", topic_cnt);
+
+ rd_kafka_DeleteTopics(rk, del_topics, topic_cnt, options, useq);
+
+ rd_kafka_AdminOptions_destroy(options);
+
+ rd_kafka_DeleteTopic_destroy_array(del_topics, topic_cnt);
+
+ free(del_topics);
+
+ if (useq)
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+
+ err = test_wait_topic_admin_result(q,
+ RD_KAFKA_EVENT_CREATETOPICS_RESULT,
+ tmout+5000);
+
+ rd_kafka_queue_destroy(q);
+
+ if (err)
+ TEST_FAIL("Failed to delete topics: %s",
+ rd_kafka_err2str(err));
+
+ return err;
+}
+
+
+static void test_free_string_array (char **strs, size_t cnt) {
+ size_t i;
+ for (i = 0 ; i < cnt ; i++)
+ free(strs[i]);
+ free(strs);
+}
+
+
+/**
+ * @return an array of all topics in the cluster matching our the
+ * rdkafka test prefix.
+ */
+static rd_kafka_resp_err_t
+test_get_all_test_topics (rd_kafka_t *rk, char ***topicsp, size_t *topic_cntp) {
+ size_t test_topic_prefix_len = strlen(test_topic_prefix);
+ const rd_kafka_metadata_t *md;
+ char **topics = NULL;
+ size_t topic_cnt = 0;
+ int i;
+ rd_kafka_resp_err_t err;
+
+ *topic_cntp = 0;
+ if (topicsp)
+ *topicsp = NULL;
+
+ /* Retrieve list of topics */
+ err = rd_kafka_metadata(rk, 1/*all topics*/, NULL, &md,
+ tmout_multip(10000));
+ if (err) {
+ TEST_WARN("%s: Failed to acquire metadata: %s: "
+ "not deleting any topics\n",
+ __FUNCTION__, rd_kafka_err2str(err));
+ return err;
+ }
+
+ if (md->topic_cnt == 0) {
+ TEST_WARN("%s: No topics in cluster\n", __FUNCTION__);
+ rd_kafka_metadata_destroy(md);
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+
+ if (topicsp)
+ topics = malloc(sizeof(*topics) * md->topic_cnt);
+
+ for (i = 0 ; i < md->topic_cnt ; i++) {
+ if (strlen(md->topics[i].topic) >= test_topic_prefix_len &&
+ !strncmp(md->topics[i].topic,
+ test_topic_prefix, test_topic_prefix_len)) {
+ if (topicsp)
+ topics[topic_cnt++] =
+ rd_strdup(md->topics[i].topic);
+ else
+ topic_cnt++;
+ }
+ }
+
+ if (topic_cnt == 0) {
+ TEST_SAY("%s: No topics (out of %d) matching our "
+ "test prefix (%s)\n",
+ __FUNCTION__, md->topic_cnt, test_topic_prefix);
+ rd_kafka_metadata_destroy(md);
+ if (topics)
+ test_free_string_array(topics, topic_cnt);
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+
+ rd_kafka_metadata_destroy(md);
+
+ if (topicsp)
+ *topicsp = topics;
+ *topic_cntp = topic_cnt;
+
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+}
+
+/**
+ * @brief Delete all test topics using the Kafka Admin API.
+ */
+rd_kafka_resp_err_t test_delete_all_test_topics (int timeout_ms) {
+ rd_kafka_t *rk;
+ char **topics;
+ size_t topic_cnt = 0;
+ rd_kafka_resp_err_t err;
+ int i;
+ rd_kafka_AdminOptions_t *options;
+ rd_kafka_queue_t *q;
+ char errstr[256];
+ int64_t abs_timeout = test_clock() + (timeout_ms * 1000);
+
+ rk = test_create_producer();
+
+ err = test_get_all_test_topics(rk, &topics, &topic_cnt);
+ if (err) {
+ /* Error already reported by test_get_all_test_topics() */
+ rd_kafka_destroy(rk);
+ return err;
+ }
+
+ if (topic_cnt == 0) {
+ rd_kafka_destroy(rk);
+ return RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+
+ q = rd_kafka_queue_get_main(rk);
+
+ options = rd_kafka_AdminOptions_new(rk, RD_KAFKA_ADMIN_OP_DELETETOPICS);
+ if (rd_kafka_AdminOptions_set_operation_timeout(options, 2*60*1000,
+ errstr,
+ sizeof(errstr)))
+ TEST_SAY(_C_YEL "Failed to set DeleteTopics timeout: %s: "
+ "ignoring\n",
+ errstr);
+
+ TEST_SAY(_C_MAG "====> Deleting all test topics with <===="
+ "a timeout of 2 minutes\n");
+
+ test_DeleteTopics_simple(rk, q, topics, topic_cnt, options);
+
+ rd_kafka_AdminOptions_destroy(options);
+
+ while (1) {
+ rd_kafka_event_t *rkev;
+ const rd_kafka_DeleteTopics_result_t *res;
+
+ rkev = rd_kafka_queue_poll(q, -1);
+
+ res = rd_kafka_event_DeleteTopics_result(rkev);
+ if (!res) {
+ TEST_SAY("%s: Ignoring event: %s: %s\n",
+ __FUNCTION__, rd_kafka_event_name(rkev),
+ rd_kafka_event_error_string(rkev));
+ rd_kafka_event_destroy(rkev);
+ continue;
+ }
+
+ if (rd_kafka_event_error(rkev)) {
+ TEST_WARN("%s: DeleteTopics for %"PRIusz" topics "
+ "failed: %s\n",
+ __FUNCTION__, topic_cnt,
+ rd_kafka_event_error_string(rkev));
+ err = rd_kafka_event_error(rkev);
+ } else {
+ const rd_kafka_topic_result_t **terr;
+ size_t tcnt;
+ int okcnt = 0;
+
+ terr = rd_kafka_DeleteTopics_result_topics(res, &tcnt);
+
+ for(i = 0 ; i < (int)tcnt ; i++) {
+ if (!rd_kafka_topic_result_error(terr[i])) {
+ okcnt++;
+ continue;
+ }
+
+ TEST_WARN("%s: Failed to delete topic %s: %s\n",
+ __FUNCTION__,
+ rd_kafka_topic_result_name(terr[i]),
+ rd_kafka_topic_result_error_string(
+ terr[i]));
+ }
+
+ TEST_SAY("%s: DeleteTopics "
+ "succeeded for %d/%"PRIusz" topics\n",
+ __FUNCTION__, okcnt, topic_cnt);
+ err = RD_KAFKA_RESP_ERR_NO_ERROR;
+ }
+
+ rd_kafka_event_destroy(rkev);
+ break;
+ }
+
+ rd_kafka_queue_destroy(q);
+
+ test_free_string_array(topics, topic_cnt);
+
+ /* Wait for topics to be fully deleted */
+ while (1) {
+ err = test_get_all_test_topics(rk, NULL, &topic_cnt);
+
+ if (!err && topic_cnt == 0)
+ break;
+
+ if (abs_timeout < test_clock()) {
+ TEST_WARN("%s: Timed out waiting for "
+ "remaining %"PRIusz" deleted topics "
+ "to disappear from cluster metadata\n",
+ __FUNCTION__, topic_cnt);
+ break;
+ }
+
+ TEST_SAY("Waiting for remaining %"PRIusz" delete topics "
+ "to disappear from cluster metadata\n", topic_cnt);
+
+ rd_sleep(1);
+ }
+
+ rd_kafka_destroy(rk);
+
+ return err;
+}
+
+
+
+void test_fail0 (const char *file, int line, const char *function,
+ int do_lock, int fail_now, const char *fmt, ...) {
+ char buf[512];
+ int is_thrd = 0;
+ size_t of;
+ va_list ap;
+ char *t;
+ char timestr[32];
+ time_t tnow = time(NULL);
+
+#ifdef _MSC_VER
+ ctime_s(timestr, sizeof(timestr), &tnow);
+#else
+ ctime_r(&tnow, timestr);
+#endif
+ t = strchr(timestr, '\n');
+ if (t)
+ *t = '\0';
+
+ of = rd_snprintf(buf, sizeof(buf), "%s():%i: ", function, line);
+ rd_assert(of < sizeof(buf));
+
+ va_start(ap, fmt);
+ rd_vsnprintf(buf+of, sizeof(buf)-of, fmt, ap);
+ va_end(ap);
+
+ /* Remove trailing newline */
+ if ((t = strchr(buf, '\n')) && !*(t+1))
+ *t = '\0';
+
+ TEST_SAYL(0, "TEST FAILURE\n");
+ fprintf(stderr, "\033[31m### Test \"%s\" failed at %s:%i:%s() at %s: "
+ "###\n"
+ "%s\n",
+ test_curr->name, file, line, function, timestr, buf+of);
+ if (do_lock)
+ TEST_LOCK();
+ test_curr->state = TEST_FAILED;
+ test_curr->failcnt += 1;
+
+ if (!*test_curr->failstr) {
+ strncpy(test_curr->failstr, buf, sizeof(test_curr->failstr));
+ test_curr->failstr[sizeof(test_curr->failstr)-1] = '\0';
+ }
+ if (fail_now && test_curr->mainfunc) {
+ tests_running_cnt--;
+ is_thrd = 1;
+ }
+ if (do_lock)
+ TEST_UNLOCK();
+ if (!fail_now)
+ return;
+ if (test_assert_on_fail || !is_thrd)
+ assert(0);
+ else
+ thrd_exit(0);
+}
diff -Nru librdkafka-0.11.3/tests/testcpp.h librdkafka-0.11.6/tests/testcpp.h
--- librdkafka-0.11.3/tests/testcpp.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/testcpp.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _TESTCPP_H_
+#define _TESTCPP_H_
#include
@@ -34,8 +35,10 @@
extern "C" {
#ifdef _MSC_VER
/* Win32/Visual Studio */
+#include "../src/win32_config.h"
#include "../src/rdwin32.h"
#else
+#include "../config.h"
/* POSIX / UNIX based systems */
#include "../src/rdposix.h"
#endif
@@ -56,6 +59,11 @@
+#define TestMessageVerify(testid,exp_partition,msgidp,msg) \
+ test_msg_parse00(__FUNCTION__, __LINE__, testid, exp_partition, \
+ msgidp, (msg)->topic_name().c_str(), \
+ (msg)->partition(), (msg)->offset(), \
+ (const char *)(msg)->key_pointer(), (msg)->key_len())
namespace Test {
@@ -69,6 +77,9 @@
static RD_UNUSED void FailLater (std::string str) {
test_FAIL(__FILE__, __LINE__, 0, str.c_str());
}
+ static RD_UNUSED void Skip (std::string str) {
+ test_SKIP(__FILE__, __LINE__, str.c_str());
+ }
static RD_UNUSED void Say (int level, std::string str) {
test_SAY(__FILE__, __LINE__, level, str.c_str());
}
@@ -123,3 +134,5 @@
static DeliveryReportCb DrCb;
};
+
+#endif /* _TESTCPP_H_ */
diff -Nru librdkafka-0.11.3/tests/test.h librdkafka-0.11.6/tests/test.h
--- librdkafka-0.11.3/tests/test.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/test.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _TEST_H_
+#define _TEST_H_
#include "../src/rd.h"
@@ -102,6 +103,7 @@
* total test run status. */
#define TEST_F_MANUAL 0x4 /**< Manual test, only started when specifically
* stated */
+#define TEST_F_SOCKEM 0x8 /**< Test requires socket emulation. */
int minver; /**< Limit tests to broker version range. */
int maxver;
@@ -111,6 +113,10 @@
int report_cnt;
int report_size;
+ rd_kafka_resp_err_t exp_dr_err; /* Expected error in test_dr_cb */
+ int produce_sync; /**< test_produce_sync() call in action */
+ rd_kafka_resp_err_t produce_sync_err; /**< DR error */
+
/**
* Runtime
*/
@@ -120,6 +126,8 @@
FILE *stats_fp;
int64_t timeout;
test_state_t state;
+ int failcnt; /**< Number of failures, useful with FAIL_LATER */
+ char failstr[512];/**< First test failure reason */
#if WITH_SOCKEM
rd_list_t sockets;
@@ -142,32 +150,12 @@
#define TEST_F_KNOWN_ISSUE_OSX 0
#endif
+void test_fail0 (const char *file, int line, const char *function,
+ int do_lock, int fail_now, const char *fmt, ...);
-#define TEST_FAIL0(file,line,do_lock,fail_now,...) do { \
- int is_thrd = 0; \
- TEST_SAYL(0, "TEST FAILURE\n"); \
- fprintf(stderr, "\033[31m### Test \"%s\" failed at %s:%i:%s(): ###\n", \
- test_curr->name, \
- file, line,__FUNCTION__); \
- fprintf(stderr, __VA_ARGS__); \
- fprintf(stderr, "\n"); \
- fprintf(stderr, "### Test random seed was %i ###\033[0m\n", \
- test_seed); \
- if (do_lock) \
- TEST_LOCK(); \
- test_curr->state = TEST_FAILED; \
- if (fail_now && test_curr->mainfunc) { \
- tests_running_cnt--; \
- is_thrd = 1; \
- } \
- if (do_lock) \
- TEST_UNLOCK(); \
- if (!fail_now) break; \
- if (test_assert_on_fail || !is_thrd) \
- assert(0); \
- else \
- thrd_exit(0); \
- } while (0)
+#define TEST_FAIL0(file,line,do_lock,fail_now,...) \
+ test_fail0(__FILE__, __LINE__, __FUNCTION__, \
+ do_lock, fail_now, __VA_ARGS__)
/* Whine and abort test */
#define TEST_FAIL(...) TEST_FAIL0(__FILE__,__LINE__,1,1,__VA_ARGS__)
@@ -175,6 +163,15 @@
/* Whine right away, mark the test as failed, but continue the test. */
#define TEST_FAIL_LATER(...) TEST_FAIL0(__FILE__,__LINE__,1,0,__VA_ARGS__)
+/* Whine right away, maybe mark the test as failed, but continue the test. */
+#define TEST_FAIL_LATER0(LATER,...) TEST_FAIL0(__FILE__,__LINE__,1,!(LATER),__VA_ARGS__)
+
+#define TEST_FAILCNT() (test_curr->failcnt)
+
+#define TEST_LATER_CHECK(...) do { \
+ if (test_curr->state == TEST_FAILED) \
+ TEST_FAIL("See previous errors. " __VA_ARGS__); \
+ } while (0)
#define TEST_PERROR(call) do { \
if (!(call)) \
@@ -220,12 +217,15 @@
} while (0)
/* Skip the current test. Argument is textual reason (printf format) */
-#define TEST_SKIP(...) do { \
- TEST_WARN("SKIPPING TEST: " __VA_ARGS__); \
- TEST_LOCK(); \
- test_curr->state = TEST_SKIPPED; \
- TEST_UNLOCK(); \
- } while (0)
+#define TEST_SKIP(...) do { \
+ TEST_WARN("SKIPPING TEST: " __VA_ARGS__); \
+ TEST_LOCK(); \
+ test_curr->state = TEST_SKIPPED; \
+ if (!*test_curr->failstr) \
+ rd_snprintf(test_curr->failstr, \
+ sizeof(test_curr->failstr), __VA_ARGS__); \
+ TEST_UNLOCK(); \
+ } while (0)
void test_conf_init (rd_kafka_conf_t **conf, rd_kafka_topic_conf_t **topic_conf,
@@ -234,9 +234,6 @@
void test_wait_exit (int timeout);
-uint64_t test_id_generate (void);
-char *test_str_id_generate (char *dest, size_t dest_size);
-const char *test_str_id_generate_tmp (void);
@@ -294,6 +291,9 @@
int log_cnt; /* Current number of warning logs */
int log_max; /* Max warning logs before suppressing. */
int log_suppr_cnt; /* Number of suppressed log messages. */
+
+ const char *msgid_hdr; /**< msgid string is in header by this name,
+ * rather than in the payload (default). */
} test_msgver_t;
/* Message */
@@ -331,11 +331,19 @@
int64_t timestamp_max;
struct test_mv_mvec mvec;
+
+ /* Correct msgver for comparison */
+ test_msgver_t *corr;
} vs;
void test_msgver_init (test_msgver_t *mv, uint64_t testid);
void test_msgver_clear (test_msgver_t *mv);
+int test_msgver_add_msg00 (const char *func, int line, test_msgver_t *mv,
+ uint64_t testid,
+ const char *topic, int32_t partition,
+ int64_t offset, int64_t timestamp,
+ rd_kafka_resp_err_t err, int msgnum);
int test_msgver_add_msg0 (const char *func, int line,
test_msgver_t *mv, rd_kafka_message_t *rkm);
#define test_msgver_add_msg(mv,rkm) \
@@ -385,6 +393,12 @@
.exp_cnt = expcnt})
+void test_msgver_verify_compare0 (const char *func, int line,
+ const char *what, test_msgver_t *mv,
+ test_msgver_t *corr, int flags);
+#define test_msgver_verify_compare(what,mv,corr,flags) \
+ test_msgver_verify_compare0(__FUNCTION__,__LINE__, what, mv, corr, flags)
+
rd_kafka_t *test_create_handle (int mode, rd_kafka_conf_t *conf);
/**
@@ -407,6 +421,8 @@
uint64_t testid, int32_t partition,
int msg_base, int cnt,
const char *payload, size_t size);
+rd_kafka_resp_err_t test_produce_sync (rd_kafka_t *rk, rd_kafka_topic_t *rkt,
+ uint64_t testid, int32_t partition);
rd_kafka_t *test_create_consumer (const char *group_id,
void (*rebalance_cb) (
@@ -485,10 +501,15 @@
void test_create_topic (const char *topicname, int partition_cnt,
int replication_factor);
rd_kafka_resp_err_t test_auto_create_topic_rkt (rd_kafka_t *rk,
- rd_kafka_topic_t *rkt);
-rd_kafka_resp_err_t test_auto_create_topic (rd_kafka_t *rk, const char *name);
+ rd_kafka_topic_t *rkt,
+ int timeout_ms);
+rd_kafka_resp_err_t test_auto_create_topic (rd_kafka_t *rk, const char *name,
+ int timeout_ms);
int test_check_auto_create_topic (void);
+int test_get_partition_count (rd_kafka_t *rk, const char *topicname,
+ int timeout_ms);
+
int test_check_builtin (const char *feature);
char *tsprintf (const char *fmt, ...) RD_FORMAT(printf, 1, 2);
@@ -507,5 +528,50 @@
#if WITH_SOCKEM
void test_socket_enable (rd_kafka_conf_t *conf);
void test_socket_close_all (struct test *test, int reinit);
+int test_socket_sockem_set_all (const char *key, int val);
#endif
+void test_headers_dump (const char *what, int lvl,
+ const rd_kafka_headers_t *hdrs);
+
+int32_t *test_get_broker_ids (rd_kafka_t *use_rk, size_t *cntp);
+
+void test_wait_metadata_update (rd_kafka_t *rk,
+ rd_kafka_metadata_topic_t *topics,
+ size_t topic_cnt,
+ rd_kafka_metadata_topic_t *not_topics,
+ size_t not_topic_cnt,
+ int tmout);
+
+rd_kafka_event_t *
+test_wait_admin_result (rd_kafka_queue_t *q,
+ rd_kafka_event_type_t evtype,
+ int tmout);
+
+rd_kafka_resp_err_t
+test_wait_topic_admin_result (rd_kafka_queue_t *q,
+ rd_kafka_event_type_t evtype,
+ int tmout);
+
+rd_kafka_resp_err_t
+test_CreateTopics_simple (rd_kafka_t *rk,
+ rd_kafka_queue_t *useq,
+ char **topics, size_t topic_cnt,
+ int num_partitions,
+ void *opaque);
+rd_kafka_resp_err_t
+test_CreatePartitions_simple (rd_kafka_t *rk,
+ rd_kafka_queue_t *useq,
+ const char *topic,
+ size_t total_part_cnt,
+ void *opaque);
+
+rd_kafka_resp_err_t
+test_DeleteTopics_simple (rd_kafka_t *rk,
+ rd_kafka_queue_t *useq,
+ char **topics, size_t topic_cnt,
+ void *opaque);
+
+rd_kafka_resp_err_t test_delete_all_test_topics (int timeout_ms);
+
+#endif /* _TEST_H_ */
diff -Nru librdkafka-0.11.3/tests/testshared.h librdkafka-0.11.6/tests/testshared.h
--- librdkafka-0.11.3/tests/testshared.h 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/testshared.h 2018-10-10 06:54:38.000000000 +0000
@@ -25,7 +25,8 @@
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
-#pragma once
+#ifndef _TESTSHARED_H_
+#define _TESTSHARED_H_
/**
* C variables and functions shared with C++ tests
@@ -43,22 +44,41 @@
(((V) >> (24-((I)*8))) & 0xff)
extern int test_broker_version;
-
+extern int test_on_ci;
const char *test_mk_topic_name (const char *suffix, int randomized);
uint64_t
-test_produce_msgs_easy (const char *topic, uint64_t testid,
- int32_t partition, int msgcnt);
+test_produce_msgs_easy_size (const char *topic, uint64_t testid,
+ int32_t partition, int msgcnt, size_t size);
+#define test_produce_msgs_easy(topic,testid,partition,msgcnt) \
+ test_produce_msgs_easy_size(topic,testid,partition,msgcnt,0)
void test_FAIL (const char *file, int line, int fail_now, const char *str);
void test_SAY (const char *file, int line, int level, const char *str);
+void test_SKIP (const char *file, int line, const char *str);
void test_timeout_set (int timeout);
int test_set_special_conf (const char *name, const char *val, int *timeoutp);
const char *test_conf_get_path (void);
const char *test_getenv (const char *env, const char *def);
+uint64_t test_id_generate (void);
+char *test_str_id_generate (char *dest, size_t dest_size);
+const char *test_str_id_generate_tmp (void);
+
+void test_prepare_msg (uint64_t testid, int32_t partition, int msg_id,
+ char *val, size_t val_size,
+ char *key, size_t key_size);
+/**
+ * Parse a message token
+ */
+void test_msg_parse00 (const char *func, int line,
+ uint64_t testid, int32_t exp_partition, int *msgidp,
+ const char *topic, int32_t partition, int64_t offset,
+ const char *key, size_t key_size);
+
+
/**
* @returns the current test's name (thread-local)
*/
@@ -96,7 +116,12 @@
gettimeofday(&tv, NULL);
return ((int64_t)tv.tv_sec * 1000000LLU) + (int64_t)tv.tv_usec;
#elif _MSC_VER
- return (int64_t)GetTickCount64() * 1000LLU;
+ LARGE_INTEGER now;
+ static RD_TLS LARGE_INTEGER freq;
+ if (!freq.QuadPart)
+ QueryPerformanceFrequency(&freq);
+ QueryPerformanceCounter(&now);
+ return (now.QuadPart * 1000000) / freq.QuadPart;
#else
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
@@ -116,19 +141,29 @@
/**
* @brief Start timing, Va-Argument is textual name (printf format)
*/
-#define TIMING_START(TIMING,...) do { \
- rd_snprintf((TIMING)->name, sizeof((TIMING)->name), __VA_ARGS__); \
+#define TIMING_RESTART(TIMING) do { \
(TIMING)->ts_start = test_clock(); \
(TIMING)->duration = 0; \
+ } while (0)
+
+#define TIMING_START(TIMING,...) do { \
+ rd_snprintf((TIMING)->name, sizeof((TIMING)->name), __VA_ARGS__); \
+ TIMING_RESTART(TIMING); \
(TIMING)->ts_every = (TIMING)->ts_start; \
} while (0)
+#define TIMING_STOPPED(TIMING) ((TIMING)->duration != 0)
+
#ifndef __cplusplus
#define TIMING_STOP(TIMING) do { \
(TIMING)->duration = test_clock() - (TIMING)->ts_start; \
TEST_SAY("%s: duration %.3fms\n", \
(TIMING)->name, (float)(TIMING)->duration / 1000.0f); \
} while (0)
+#define TIMING_REPORT(TIMING) \
+ TEST_SAY("%s: duration %.3fms\n", \
+ (TIMING)->name, (float)(TIMING)->duration / 1000.0f); \
+
#else
#define TIMING_STOP(TIMING) do { \
char _str[128]; \
@@ -143,6 +178,27 @@
#define TIMING_DURATION(TIMING) ((TIMING)->duration ? (TIMING)->duration : \
(test_clock() - (TIMING)->ts_start))
+#define TIMING_ASSERT0(TIMING,DO_FAIL_LATER,TMIN_MS,TMAX_MS) do { \
+ if (!TIMING_STOPPED(TIMING)) \
+ TIMING_STOP(TIMING); \
+ int _dur_ms = (int)TIMING_DURATION(TIMING) / 1000; \
+ if (TMIN_MS <= _dur_ms && _dur_ms <= TMAX_MS) \
+ break; \
+ if (test_on_ci || strcmp(test_mode, "bare")) \
+ TEST_WARN("%s: expected duration %d <= %d <= %d ms%s\n", \
+ (TIMING)->name, TMIN_MS, _dur_ms, TMAX_MS, \
+ ": not FAILING test on CI"); \
+ else \
+ TEST_FAIL_LATER0(DO_FAIL_LATER, \
+ "%s: expected duration %d <= %d <= %d ms", \
+ (TIMING)->name, TMIN_MS, _dur_ms, TMAX_MS); \
+ } while (0)
+
+#define TIMING_ASSERT(TIMING,TMIN_MS,TMAX_MS) \
+ TIMING_ASSERT0(TIMING,0,TMIN_MS,TMAX_MS)
+#define TIMING_ASSERT_LATER(TIMING,TMIN_MS,TMAX_MS) \
+ TIMING_ASSERT0(TIMING,1,TMIN_MS,TMAX_MS)
+
/* Trigger something every US microseconds. */
static RD_UNUSED int TIMING_EVERY (test_timing_t *timing, int us) {
int64_t now = test_clock();
@@ -159,3 +215,11 @@
#else
#define rd_sleep(S) Sleep((S)*1000)
#endif
+
+/* Make sure __SANITIZE_ADDRESS__ (gcc) is defined if compiled with asan */
+#if !defined(__SANITIZE_ADDRESS__) && defined(__has_feature)
+ #if __has_feature(address_sanitizer)
+ #define __SANITIZE_ADDRESS__ 1
+ #endif
+#endif
+#endif /* _TESTSHARED_H_ */
diff -Nru librdkafka-0.11.3/tests/until-fail.sh librdkafka-0.11.6/tests/until-fail.sh
--- librdkafka-0.11.3/tests/until-fail.sh 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/until-fail.sh 2018-10-10 06:54:38.000000000 +0000
@@ -54,7 +54,16 @@
if [[ "$DELETE_TOPICS" == "y" ]]; then
- ./delete-test-topics.sh $ZK_ADDRESS ~/src/kafka/bin/kafka-topics.sh || true
+ # Delete topics using Admin API, which is very fast
+ # leads to sub-sequent test failures because of the background
+ # deletes in Kafka still taking a long time:
+ #
+ #make delete_topics
+
+ # Delete topic-by-topic using kafka-topics for each one,
+ # very slow but topics are properly deleted before the script
+ # returns.
+ ./delete-test-topics.sh $ZK_ADDRESS ~/src/kafka/bin/kafka-topics.sh || true
fi
done
diff -Nru librdkafka-0.11.3/tests/xxxx-metadata.cpp librdkafka-0.11.6/tests/xxxx-metadata.cpp
--- librdkafka-0.11.3/tests/xxxx-metadata.cpp 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/tests/xxxx-metadata.cpp 2018-10-10 06:54:38.000000000 +0000
@@ -63,7 +63,7 @@
RdKafka::Metadata *metadata;
RdKafka::ErrorCode err;
- int msgcnt = 10000;
+ int msgcnt = test_on_ci ? 1000 : 10000;
int partition_cnt = 2;
int i;
uint64_t testid;
diff -Nru librdkafka-0.11.3/.travis.yml librdkafka-0.11.6/.travis.yml
--- librdkafka-0.11.3/.travis.yml 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/.travis.yml 2018-10-10 06:54:38.000000000 +0000
@@ -10,8 +10,6 @@
- osx
dist: trusty
sudo: false
-before_install:
- - if [[ "${TRAVIS_OS_NAME}_$CC" == "linux_gcc" ]]; then sudo make -C packaging/rpm MOCK_CONFIG=el7-x86_64 prepare_ubuntu ; fi
before_script:
- ccache -s || echo "CCache is not available."
script:
@@ -22,14 +20,15 @@
- make -j2 all examples check && make -C tests run_local
- make install
- (cd dest && tar cvzf ../artifacts/librdkafka.tar.gz .)
-- if [[ "${TRAVIS_OS_NAME}_$CC" == "linux_gcc" ]]; then sudo make -C packaging/rpm MOCK_CONFIG=el7-x86_64 all copy-artifacts ; fi
+- if [[ "${TRAVIS_OS_NAME}_$CC" == "linux_gcc" ]]; then packaging/rpm/mock-on-docker.sh ; fi
- if [[ "${TRAVIS_OS_NAME}_$CC" == "linux_gcc" ]]; then docker run -it -v $PWD:/v microsoft/dotnet:2-sdk /v/packaging/tools/build-debian.sh /v /v/artifacts/librdkafka-debian9.tgz; fi
+- if [[ "${TRAVIS_OS_NAME}_$CC" == "linux_clang" ]]; then packaging/alpine/test-build.sh ; fi
deploy:
provider: s3
access_key_id:
- secure: "i7IAO5z6xYeVaOnFiJOz0wluH7CfGf39TiABQG/yQIIp53fRu+X94FBUS7xq2PziKV08ak8pNWWjenwRB/viPTGqFQDS/ypymhSGGQVTmhgG372ypODhqbnvzskMNe1wp+UFIl8LFVEaHMVPEwWVuYpXhWNMHGrr4uSQhF+5r0Y="
+ secure: "nGcknL5JZ5XYCEJ96UeDtnLOOidWsfXrk2x91Z9Ip2AyrUtdfZBc8BX16C7SAQbBeb4PQu/OjRBQWTIRqU64ZEQU1Z0lHjxCiGEt5HO0YlXWvZ8OJGAQ0wSmrQED850lWjGW2z5MpDqqxbZyATE8VksW5dtGiHgNuITinVW8Lok="
secret_access_key:
- secure: q7rWZkwQqyIFu3i32y/rJsKF1AJpwIqQe3AhBFEqjCz80mdEYysvcHBfl956EAAr4qbR3umIKbkhRwbBVyDGlMaraZiR7YFgAO+v0DRFeMiF5XFE3KHNIpCQHSKnzOiL65TpJHZIoB+6w0lAbBVzj87MBN0axAeArHKiO1y8Eec=
+ secure: "J+LygNeoXQImN9E7EARNmcgLpqm6hoRjxwHJaen9opeuSDowKDpZxP7ixSml3BEn2pJJ4kpsdj5A8t5uius+qC4nu9mqSAZcmdKeSmliCbH7kj4J9MR7LBcXk3Uf515QGm7y4nzw+c1PmpteYL5S06Kgqp+KkPRLKTS2NevVZuY="
bucket: librdkafka-ci-packages
region: us-west-1
skip_cleanup: true
diff -Nru librdkafka-0.11.3/win32/librdkafka.master.testing.targets librdkafka-0.11.6/win32/librdkafka.master.testing.targets
--- librdkafka-0.11.3/win32/librdkafka.master.testing.targets 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/win32/librdkafka.master.testing.targets 2018-10-10 06:54:38.000000000 +0000
@@ -1,13 +1,13 @@
- $(MSBuildThisFileDirectory)..\..\package-win\runtimes\$(Configuration)\win7-$(Platform)\native\librdkafka.lib;%(AdditionalDependencies)
+ $(MSBuildThisFileDirectory)..\..\package-win\runtimes\$(Configuration)\win-$(Platform)\native\librdkafka.lib;%(AdditionalDependencies)
$(MSBuildThisFileDirectory)include;%(AdditionalIncludeDirectories)
-
+
diff -Nru librdkafka-0.11.3/win32/librdkafka.sln librdkafka-0.11.6/win32/librdkafka.sln
--- librdkafka-0.11.3/win32/librdkafka.sln 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/win32/librdkafka.sln 2018-10-10 06:54:38.000000000 +0000
@@ -43,9 +43,6 @@
EndProjectSection
EndProject
Global
- GlobalSection(Performance) = preSolution
- HasPerformanceSessions = true
- EndGlobalSection
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|Mixed Platforms = Debug|Mixed Platforms
diff -Nru librdkafka-0.11.3/win32/librdkafka.vcxproj librdkafka-0.11.6/win32/librdkafka.vcxproj
--- librdkafka-0.11.3/win32/librdkafka.vcxproj 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/win32/librdkafka.vcxproj 2018-10-10 06:54:38.000000000 +0000
@@ -98,12 +98,15 @@
+
+
+
@@ -115,7 +118,6 @@
-
@@ -134,6 +136,7 @@
+
@@ -146,6 +149,7 @@
+
@@ -159,6 +163,7 @@
+
@@ -191,8 +196,13 @@
+
+
+
+
+
@@ -200,6 +210,7 @@
+
diff -Nru librdkafka-0.11.3/win32/packages.config librdkafka-0.11.6/win32/packages.config
--- librdkafka-0.11.3/win32/packages.config 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/win32/packages.config 2018-10-10 06:54:38.000000000 +0000
@@ -3,4 +3,4 @@
-
\ No newline at end of file
+
diff -Nru librdkafka-0.11.3/win32/tests/tests.vcxproj librdkafka-0.11.6/win32/tests/tests.vcxproj
--- librdkafka-0.11.3/win32/tests/tests.vcxproj 2017-12-04 09:29:22.000000000 +0000
+++ librdkafka-0.11.6/win32/tests/tests.vcxproj 2018-10-10 06:54:38.000000000 +0000
@@ -6,13 +6,11 @@
tests
8.1
-
+
Application
-
- 8e214174
-
+
Console
@@ -134,6 +132,7 @@
+
@@ -155,11 +154,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+