diff -Nru nutcracker-0.4.0+dfsg/ChangeLog nutcracker-0.4.1+dfsg/ChangeLog --- nutcracker-0.4.0+dfsg/ChangeLog 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/ChangeLog 2015-06-23 06:06:05.000000000 +0000 @@ -1,3 +1,36 @@ + 2015-22-06 Manju Rajashekhar + * twemproxy: version 0.4.1 release + redis_auth is only valid for a redis pool + getaddrinfo returns non-zero +ve value on error + fix-hang-when-command-only (charsyam) + fix bug crash when get command without key and whitespace (charsyam) + mark server as failed on protocol level transiet failures like -OOM, -LOADING, etc + implemented support for parsing fine grained redis error response + remove redundant conditional judgement in rbtree deletion (leo ma) + fix bug mset has invalid pair (charsyam) + fix bug mset has invalid pair (charsyam) + temp fix a core on kqueue (idning) + support "touch" command for memcached (panmiaocai) + fix redis parse rsp bug (charsyam) + SORT command can take multiple arguments. So it should be part of redis_argn() and not redis_arg0() + remove incorrect assert because client could send data after sending a quit request which must be discarded + allow file permissions to be set for UNIX domain listening socket (ori liveneh) + return error if formatted is greater than mbuf size by using nc_vsnprintf() in msg_prepend_format() + fix req_make_reply on msg_get, mark it as response (idning) + redis database select upon connect (arne claus) + redis_auth (charsyam) + allow null key(empty key) (idning) + fix core on invalid mset like "mset a a a" (idning) + +2014-18-10 idning + * twemproxy: version 0.4.0 release + mget improve (idning) + many new commands supported: LEX, PFADD, PFMERGE, SORT, PING, QUIT, SCAN... (mattrobenolt, areina, idning) + handle max open file limit(allenlz) + add notice-log and use ms time in log(idning) + fix bug in string_compare (andyqzb) + fix deadlock in sighandler (idning) + 2013-20-12 Manju Rajashekhar * twemproxy: version 0.3.0 release SRANDMEMBER support for the optional count argument (mkhq) @@ -28,12 +61,10 @@ add support for hash tags 2012-18-10 Manju Rajashekhar - * twemproxy: version 0.2.2 release fix the off-by-one error when calculating redis key length 2012-12-10 Manju Rajashekhar - * twemproxy: version 0.2.1 release don't use buf in conf_add_server allow an optional instance name for consistent hashing (charsyam) @@ -41,7 +72,6 @@ add stats-bind-any -a option (charsyam) 2012-12-03 Manju Rajashekhar - * twemproxy: version 0.2.0 release add -D or --describe-stats command-line argument to print stats description redis support in twemproxy @@ -54,7 +84,6 @@ delete stats tracking memcache requests and responses; stats module no longer tracks protocol related stats 2012-10-27 Manju Rajashekhar - * twemproxy: version 0.1.20 release on msg_repair, msg->pos should point to nbuf->pos and not nbuf->last refactor memcache parsing code into proto directory @@ -63,23 +92,19 @@ fix log_hexdump and loga_hexdump 2012-07-31 Manju Rajashekhar - * twemproxy: version 0.1.19 release close server connection on a stray response (yashh, bmatheny) 2012-06-19 Manju Rajashekhar - * twemproxy: version 0.1.18 release command line option to set mbuf chunk size 2012-05-09 Manju Rajashekhar - * twemproxy: version 0.1.17 release use _exit(0) instead of exit(0) when daemonizing use loga instead of log_stderr in nc_stacktrace 2012-02-09 Manju Rajashekhar - * twemproxy: version 0.1.16 release twemproxy (aka nutcracker) is a fast and lightweight proxy for memcached protocol. diff -Nru nutcracker-0.4.0+dfsg/configure.ac nutcracker-0.4.1+dfsg/configure.ac --- nutcracker-0.4.0+dfsg/configure.ac 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/configure.ac 2015-06-23 06:06:05.000000000 +0000 @@ -1,7 +1,7 @@ # Define the package version numbers and the bug reporting address m4_define([NC_MAJOR], 0) -m4_define([NC_MINOR], 3) -m4_define([NC_PATCH], 0) +m4_define([NC_MINOR], 4) +m4_define([NC_PATCH], 1) m4_define([NC_BUGS], [manj@cs.stanford.edu]) # Initialize autoconf @@ -158,6 +158,7 @@ AM_CONDITIONAL([OS_LINUX], [test "x$ac_cv_epoll_works" = "xyes"]) AM_CONDITIONAL([OS_BSD], [test "x$ac_cv_kqueue_works" = "xyes"]) AM_CONDITIONAL([OS_SOLARIS], [test "x$ac_cv_evports_works" = "xyes"]) +AM_CONDITIONAL([OS_FREEBSD], [test "$(uname -v | cut -c 1-10)" == "FreeBSD 10"]) # Package options AC_MSG_CHECKING([whether to enable debug logs and asserts]) diff -Nru nutcracker-0.4.0+dfsg/debian/changelog nutcracker-0.4.1+dfsg/debian/changelog --- nutcracker-0.4.0+dfsg/debian/changelog 2014-10-21 00:26:54.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/changelog 2016-12-17 02:48:28.000000000 +0000 @@ -1,3 +1,20 @@ +nutcracker (0.4.1+dfsg-1) unstable; urgency=medium + + * New upstream release. + * Bump debhelper compat level to 10; remove dh-autoreconf B-D. + * Update Standards-Version to 3.9.8, no changes needed. + * Fix a few d/copyright structural errors. + * Update Vcs-Git/Vcs-Browser to point to https URLs and cgit. + * Add dependency on lsb-base for /lib/lsb/init-functions. + * Add hardening build flags (hardening=+all). + * Switch pidfile location from /var/run to /run. + * Update debian/patches/sysconfdir to only write a pidfile if --daemonize + was passed. + * Add systemd service file. + * Remove upstart job, dead and unmaintained code. + + -- Faidon Liambotis Sat, 17 Dec 2016 04:48:28 +0200 + nutcracker (0.4.0+dfsg-1) unstable; urgency=medium * New upstream release. diff -Nru nutcracker-0.4.0+dfsg/debian/compat nutcracker-0.4.1+dfsg/debian/compat --- nutcracker-0.4.0+dfsg/debian/compat 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/compat 2016-12-17 02:48:28.000000000 +0000 @@ -1 +1 @@ -9 +10 diff -Nru nutcracker-0.4.0+dfsg/debian/control nutcracker-0.4.1+dfsg/debian/control --- nutcracker-0.4.0+dfsg/debian/control 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/control 2016-12-17 02:48:28.000000000 +0000 @@ -2,15 +2,15 @@ Section: net Priority: optional Maintainer: Faidon Liambotis -Build-Depends: debhelper (>= 9), dh-autoreconf, libyaml-dev -Standards-Version: 3.9.6 +Build-Depends: debhelper (>= 10), libyaml-dev +Standards-Version: 3.9.8 Homepage: https://github.com/twitter/twemproxy -Vcs-Git: git://anonscm.debian.org/collab-maint/nutcracker.git -Vcs-Browser: http://anonscm.debian.org/gitweb/?p=collab-maint/nutcracker.git;a=summary +Vcs-Git: https://anonscm.debian.org/git/collab-maint/nutcracker.git +Vcs-Browser: https://anonscm.debian.org/cgit/collab-maint/nutcracker.git Package: nutcracker Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, adduser +Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, lsb-base (>= 3.0-6) Description: Fast, light-weight proxy for memcached and Redis nutcracker, also known as twemproxy (pronounced "two-em-proxy"), is a fast and lightweight proxy for the memcached and Redis protocols. It was diff -Nru nutcracker-0.4.0+dfsg/debian/copyright nutcracker-0.4.1+dfsg/debian/copyright --- nutcracker-0.4.0+dfsg/debian/copyright 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/copyright 2016-12-17 02:48:28.000000000 +0000 @@ -10,20 +10,15 @@ Copyright: 2014 Faidon Liambotis License: Apache-2.0 -Files: src/hashkit/nc_one_at_a_time.c -Copyright: 2009 Brian Aker - 2011 Twitter, Inc. -License: Apache-2.0 - Files: src/nc_array.c src/nc_array.h src/nc_rbtree.c src/nc_rbtree.h Copyright: 2002-2010 Igor Sysoev 2011 Twitter, Inc. -License: Apache-2.0 and BSD-2-clause +License: BSD-2-clause Files: src/nc_queue.h Copyright: 1991, 1993 The Regents of the University of California 2011 Twitter, Inc. -License: Apache-2.0 and BSD-4-clause +License: BSD-4-clause Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -56,7 +51,7 @@ Copyright: 2007-2010 TangentOrg (Brian Aker) 2011 Data Differential (http://datadifferential.com/) 2011 Twitter, Inc. -License: Apache-2.0 and BSD-4-clause +License: BSD-4-clause-TangentOrg Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -85,10 +80,15 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +Files: src/hashkit/nc_one_at_a_time.c +Copyright: 2009 Brian Aker + 2011 Twitter, Inc. +License: Apache-2.0 + Files: src/hashkit/nc_hsieh.c Copyright: 2004, 2005, Paul Hsieh 2011 Twitter, Inc. -License: Apache-2.0 and PaulHsieh-license +License: PaulHsieh-license The derivative content includes raw computer source code, ideas, opinions, and excerpts whose original source is covered under another license and transformations of such derivatives. Note that mere excerpts by themselves diff -Nru nutcracker-0.4.0+dfsg/debian/init.d nutcracker-0.4.1+dfsg/debian/init.d --- nutcracker-0.4.0+dfsg/debian/init.d 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/init.d 2016-12-17 02:48:28.000000000 +0000 @@ -16,7 +16,7 @@ DAEMON=/usr/sbin/nutcracker NAME="nutcracker" DESC="memcached and redis proxy" -PIDFILE=/var/run/nutcracker/nutcracker.pid +PIDFILE=/run/nutcracker/nutcracker.pid CONFFILE=/etc/nutcracker/nutcracker.yml . /lib/lsb/init-functions @@ -44,9 +44,9 @@ exit 1 fi log_daemon_msg "Starting $NAME" "$DESC" - mkdir -p /var/run/nutcracker - chown nutcracker:nutcracker /var/run/nutcracker - chmod 0755 /var/run/nutcracker + mkdir -p /run/nutcracker + chown nutcracker:nutcracker /run/nutcracker + chmod 0755 /run/nutcracker start-stop-daemon --quiet --start --chuid nutcracker \ --pidfile $PIDFILE --exec $DAEMON \ -- $DAEMON_OPTS diff -Nru nutcracker-0.4.0+dfsg/debian/patches/sysconfdir nutcracker-0.4.1+dfsg/debian/patches/sysconfdir --- nutcracker-0.4.0+dfsg/debian/patches/sysconfdir 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/patches/sysconfdir 2016-12-17 02:48:28.000000000 +0000 @@ -1,7 +1,7 @@ Description: Use sysconfdir & localstatedir for conf/pid/logs Author: Faidon Liambotis Forwarded: https://github.com/twitter/twemproxy/pull/123 -Last-Update: 2014-06-23 +Last-Update: 2016-12-17 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -13,11 +13,10 @@ +cfg-options.h: Makefile + $(AM_V_GEN)echo '#define NC_CONF_PATH "$(sysconfdir)/nutcracker/nutcracker.yml"' >$@ + @echo '#define NC_LOG_PATH "$(localstatedir)/log/nutcracker/nutcracker.log"' >>$@ -+ @echo '#define NC_PID_FILE "$(localstatedir)/run/nutcracker/nutcracker.pid"' >>$@ ++ @echo '#define NC_PID_FILE "/run/nutcracker/nutcracker.pid"' >>$@ AM_CPPFLAGS = if !OS_SOLARIS - --- a/src/nc.c +++ b/src/nc.c @@ -28,19 +28,16 @@ @@ -50,3 +49,12 @@ nci->pidfile = 0; } +@@ -488,7 +485,7 @@ nc_pre_run(struct instance *nci) + return status; + } + +- if (nci->pid_filename) { ++ if (daemonize && nci->pid_filename) { + status = nc_create_pidfile(nci); + if (status != NC_OK) { + return status; diff -Nru nutcracker-0.4.0+dfsg/debian/patches/use_system_libyaml nutcracker-0.4.1+dfsg/debian/patches/use_system_libyaml --- nutcracker-0.4.0+dfsg/debian/patches/use_system_libyaml 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/patches/use_system_libyaml 2016-12-17 02:48:28.000000000 +0000 @@ -5,7 +5,7 @@ --- a/configure.ac +++ b/configure.ac -@@ -195,15 +195,8 @@ AS_IF([test "x$disable_stats" = xyes], +@@ -196,15 +196,8 @@ AS_IF([test "x$disable_stats" = xyes], [AC_DEFINE([HAVE_STATS], [1], [Define to 1 if stats is not disabled])]) AC_MSG_RESULT($disable_stats) @@ -21,7 +21,6 @@ src/Makefile src/hashkit/Makefile src/proto/Makefile - --- a/Makefile.am +++ b/Makefile.am @@ -2,7 +2,7 @@ MAINTAINERCLEANFILES = Makefile.in acloc @@ -33,7 +32,6 @@ dist_man_MANS = man/nutcracker.8 - --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,6 @@ endif @@ -42,9 +40,9 @@ AM_CPPFLAGS += -I $(top_srcdir)/src/event -AM_CPPFLAGS += -I $(top_srcdir)/contrib/yaml-0.1.4/include - AM_CFLAGS = - AM_CFLAGS += -Wall -Wshadow -@@ -19,7 +18,7 @@ AM_CFLAGS += -Wconversion -Wsign-compare + AM_CFLAGS = + # about -fno-strict-aliasing: https://github.com/twitter/twemproxy/issues/276 +@@ -21,7 +20,7 @@ AM_CFLAGS += -Wconversion -Wsign-compare AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations AM_LDFLAGS = @@ -53,7 +51,7 @@ if OS_SOLARIS AM_LDFLAGS += -lnsl -lsocket endif -@@ -52,4 +51,3 @@ nutcracker_SOURCES = \ +@@ -57,4 +56,3 @@ nutcracker_SOURCES = \ nutcracker_LDADD = $(top_builddir)/src/hashkit/libhashkit.a nutcracker_LDADD += $(top_builddir)/src/proto/libproto.a nutcracker_LDADD += $(top_builddir)/src/event/libevent.a diff -Nru nutcracker-0.4.0+dfsg/debian/rules nutcracker-0.4.1+dfsg/debian/rules --- nutcracker-0.4.0+dfsg/debian/rules 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/rules 2016-12-17 02:48:28.000000000 +0000 @@ -1,7 +1,10 @@ #!/usr/bin/make -f -# Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 +export DEB_BUILD_MAINT_OPTIONS = hardening=+all +DPKG_EXPORT_BUILDFLAGS = 1 +include /usr/share/dpkg/buildflags.mk + %: - dh $@ --with autoreconf + dh $@ diff -Nru nutcracker-0.4.0+dfsg/debian/service nutcracker-0.4.1+dfsg/debian/service --- nutcracker-0.4.0+dfsg/debian/service 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/service 2016-12-17 02:48:28.000000000 +0000 @@ -0,0 +1,23 @@ +[Unit] +Description=nutcracker proxy for memcached and Redis +Documentation=man:nutcracker +After=local-fs.target +After=network.target +ConditionPathExists=/etc/nutcracker/nutcracker.yml + +[Service] +Type=simple +EnvironmentFile=-/etc/default/nutcracker +ExecStart=/usr/sbin/nutcracker $DAEMON_OPTS +User=nutcracker +Group=nutcracker + +MountFlags=slave +DevicePolicy=closed +PrivateDevices=true +PrivateTmp=true +ProtectSystem=full +ProtectHome=read-only + +[Install] +WantedBy=multi-user.target diff -Nru nutcracker-0.4.0+dfsg/debian/upstart nutcracker-0.4.1+dfsg/debian/upstart --- nutcracker-0.4.0+dfsg/debian/upstart 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/debian/upstart 1970-01-01 00:00:00.000000000 +0000 @@ -1,22 +0,0 @@ -description "memcached and redis proxy" - -start on (runlevel [2345] and net-device-up IFACE!=lo) -stop on runlevel [!2345] - -respawn - -pre-start script - [ ! -e /etc/nutcracker/nutcracker.yml ] && { stop; exit 0; } - - mkdir -p -m0755 /var/run/nutcracker - chown nutcracker:nutcracker /var/run/nutcracker - - . /etc/default/nutcracker - /usr/sbin/nutcracker $DAEMON_OPTS --test-conf || { stop; exit 1; } -end script - -script - . /etc/default/nutcracker - exec start-stop-daemon --quiet --start --chuid nutcracker \ - --exec /usr/sbin/nutcracker -- $DAEMON_OPTS -end script diff -Nru nutcracker-0.4.0+dfsg/man/nutcracker.8 nutcracker-0.4.1+dfsg/man/nutcracker.8 --- nutcracker-0.4.0+dfsg/man/nutcracker.8 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/man/nutcracker.8 2015-06-23 06:06:05.000000000 +0000 @@ -39,7 +39,7 @@ .BR \-D ", " \-\-describe-stats Print stats description and exit. .TP -.BR \-v ", " \-\-verbosity=\fIN\fP +.BR \-v ", " \-\-verbose=\fIN\fP Set logging level to \fIN\fP. (default: 5, min: 0, max: 11) .TP .BR \-o ", " \-\-output=\fIfilename\fP diff -Nru nutcracker-0.4.0+dfsg/notes/memcache.md nutcracker-0.4.1+dfsg/notes/memcache.md --- nutcracker-0.4.0+dfsg/notes/memcache.md 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/notes/memcache.md 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,162 @@ +## Memcache Command Support + +### Request + +- Twemproxy implements only the memached ASCII commands +- Binary commands are currently unsupported + +#### Ascii Storage Command + + +-------------------+------------+--------------------------------------------------------------------------+ + | Command | Supported? | Format | + +-------------------+------------+--------------------------------------------------------------------------+ + | set | Yes | set [noreply]\r\n\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | add | Yes | add [noreply]\r\n\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | replace | Yes | replace [noreply]\r\n\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | append | Yes | append [noreply]\r\n\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | prepend | Yes | prepend [noreply]\r\n\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | cas | Yes | cas [noreply]\r\n\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + +* Where, + * - uint32_t : data specific client side flags + * - uint32_t : expiration time (in seconds) + * - uint32_t : size of the data (in bytes) + * - uint8_t[]: data block + * - uint64_t + +#### Ascii Retrival Command + + +-------------------+------------+--------------------------------------------------------------------------+ + | Command | Supported? | Format | + +-------------------+------------+--------------------------------------------------------------------------+ + | get | Yes | get []+\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | gets | Yes | gets []+\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + +#### Ascii Delete + + +-------------------+------------+--------------------------------------------------------------------------+ + | Command | Supported? | Format | + +-------------------+------------+--------------------------------------------------------------------------+ + | delete | Yes | delete [noreply]\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + +#### Ascii Arithmetic Command + + +-------------------+------------+--------------------------------------------------------------------------+ + | Command | Supported? | Format | + +-------------------+------------+--------------------------------------------------------------------------+ + | incr | Yes | incr [noreply]\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | decr | Yes | decr [noreply]\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + +* Where, + * - uint64_t + +#### Ascii Misc Command + + +-------------------+------------+--------------------------------------------------------------------------+ + | Command | Supported? | Format | + +-------------------+------------+--------------------------------------------------------------------------+ + | set | Yes | touch [noreply]\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | quit | Yes | quit\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | flush_all | No | flush_all [] [noreply]\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | version | No | version\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | verbosity | No | verbosity [noreply]\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | stats | No | stats\r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + | stats | No | stats \r\n | + +-------------------+------------+--------------------------------------------------------------------------+ + +### Response + +#### Error Responses + + ERROR\r\n + CLIENT_ERROR [error]\r\n + SERVER_ERROR [error]\r\n + + Where, + - ERROR means client sent a non-existent command name + - CLIENT_ERROR means that command sent by the client does not conform to the protocol + - SERVER_ERROR means that there was an error on the server side that made processing of the command impossible + +#### Storage Command Responses + + STORED\r\n + NOT_STORED\r\n + EXISTS\r\n + NOT_FOUND\r\n + + Where, + - STORED indicates success. + - NOT_STORED indicates the data was not stored because condition for an add or replace wasn't met. + - EXISTS indicates that the item you are trying to store with a cas has been modified since you last fetched it. + - NOT_FOUND indicates that the item you are trying to store with a cas does not exist. + +#### Delete Command Responses + + NOT_FOUND\r\n + DELETED\r\n + +#### Retrival Responses + + END\r\n + VALUE []\r\n\r\nEND\r\n + VALUE []\r\n\r\n[VALUE []\r\n]+\r\nEND\r\n + +#### Arithmetic Responses + + NOT_FOUND\r\n + \r\n + + Where, + - - uint64_t : new key value after incr or decr operation + +#### Touch Command Responses + + NOT_FOUND\r\n + TOUCHED\r\n + +#### Statistics Response + + [STAT \r\n]+END\r\n + +#### Misc Responses + + OK\r\n + VERSION \r\n + +### Notes + +- set always creates mapping irrespective of whether it is present on not. +- add, adds only if the mapping is not present +- replace, only replaces if the mapping is present +- append and prepend command ignore flags and expiry values +- noreply instructs the server to not send the reply even if there is an error. +- decr of 0 is 0, while incr of UINT64_MAX is 0 +- maximum length of the key is 250 characters +- expiry of 0 means that item never expires, though it could be evicted from the cache +- non-zero expiry is either unix time (# seconds since 01/01/1970) or, + offset in seconds from the current time (< 60 x 60 x 24 x 30 seconds = 30 days) +- expiry time is with respect to the server (not client) +- can be zero and when it is, the block is empty. +- Thoughts: + - ascii protocol is easier to debug - think using strace or tcpdump to see + protocol on the wire, Or using telnet or netcat or socat to build memcache + requests and responses + http://stackoverflow.com/questions/2525188/are-binary-protocols-dead + - http://news.ycombinator.com/item?id=1712788 diff -Nru nutcracker-0.4.0+dfsg/notes/memcache.txt nutcracker-0.4.1+dfsg/notes/memcache.txt --- nutcracker-0.4.0+dfsg/notes/memcache.txt 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/notes/memcache.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,123 +0,0 @@ -- ascii: - -- Storage Commands (set, add, replace, append, prepend, cas): - - set [noreply]\r\n\r\n - add [noreply]\r\n\r\n - replace [noreply]\r\n\r\n - append [noreply]\r\n\r\n - prepend [noreply]\r\n\r\n - - cas [noreply]\r\n\r\n - - where, - - uint32_t : data specific client side flags - - uint32_t : expiration time (in seconds) - - uint32_t : size of the data (in bytes) - - uint8_t[]: data block - - uint64_t - -- Retrival Commands (get, gets): - - get \r\n - get []+\r\n - - gets \r\n - gets []+\r\n - -- Delete Command (delete): - - delete [noreply]\r\n - -- Arithmetic Commands (incr, decr): - - incr [noreply]\r\n - decr [noreply]\r\n - - where, - - uint64_t - -- Misc Commands (quit) - - quit\r\n - flush_all [] [noreply]\r\n - version\r\n - verbosity [noreply]\r\n - -- Statistics Commands - - stats\r\n - stats \r\n - -- Error Responses: - - ERROR\r\n - CLIENT_ERROR [error]\r\n - SERVER_ERROR [error]\r\n - - where, - ERROR means client sent a non-existent command name - CLIENT_ERROR means that command sent by the client does not conform to the protocol - SERVER_ERROR means that there was an error on the server side that made processing of the command impossible - -- Storage Command Responses: - - STORED\r\n - NOT_STORED\r\n - EXISTS\r\n - NOT_FOUND\r\n - - where, - STORED indicates success. - NOT_STORED indicates the data was not stored because condition for an add or replace wasn't met. - EXISTS indicates that the item you are trying to store with a cas has been modified since you last fetched it. - NOT_FOUND indicates that the item you are trying to store with a cas does not exist. - -- Delete Command Response: - - NOT_FOUND\r\n - DELETED\r\n - -- Retrival Responses: - - END\r\n - VALUE []\r\n\r\nEND\r\n - VALUE []\r\n\r\n[VALUE []\r\n]+\r\nEND\r\n - -- Arithmetic Responses: - - NOT_FOUND\r\n - \r\n - - where, - - uint64_t : new key value after incr or decr operation - -- Statistics Response - [STAT \r\n]+END\r\n - -- Misc Response - - OK\r\n - VERSION \r\n - -- Notes: - - set always creates mapping irrespective of whether it is present on not. - - add, adds only if the mapping is not present - - replace, only replaces if the mapping is present - - append and prepend command ignore flags and expiry values - - noreply instructs the server to not send the reply even if there is an error. - - decr of 0 is 0, while incr of UINT64_MAX is 0 - - maximum length of the key is 250 characters - - expiry of 0 means that item never expires, though it could be evicted from the cache - - non-zero expiry is either unix time (# seconds since 01/01/1970) or, - offset in seconds from the current time (< 60 x 60 x 24 x 30 seconds = 30 days) - - expiry time is with respect to the server (not client) - - can be zero and when it is, the block is empty. - -- Thoughts: - - ascii protocol is easier to debug - think using strace or tcpdump to see - protocol on the wire, Or using telnet or netcat or socat to build memcache - requests and responses - http://stackoverflow.com/questions/2525188/are-binary-protocols-dead - - - http://news.ycombinator.com/item?id=1712788 diff -Nru nutcracker-0.4.0+dfsg/notes/recommendation.md nutcracker-0.4.1+dfsg/notes/recommendation.md --- nutcracker-0.4.0+dfsg/notes/recommendation.md 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/notes/recommendation.md 2015-06-23 06:06:05.000000000 +0000 @@ -2,7 +2,7 @@ ## Log Level -By default debug logging is disabled in nutcracker. However, it is worthwhile running nutcracker with debug logging enabled and verbosity level set to LOG_INFO (-v 6 or --verbosity=6). This in reality does not add much overhead as you only pay the cost of checking an if condition for every log line encountered during the run time. +By default debug logging is disabled in nutcracker. However, it is worthwhile running nutcracker with debug logging enabled and verbosity level set to LOG_INFO (-v 6 or --verbose=6). This in reality does not add much overhead as you only pay the cost of checking an if condition for every log line encountered during the run time. At LOG_INFO level, nutcracker logs the life cycle of every client and server connection and important events like the server being ejected from the hash ring and so on. Eg. @@ -50,7 +50,7 @@ Relying only on client-side timeouts has the adverse effect of the original request having timedout on the client to proxy connection, but still pending and outstanding on the proxy to server connection. This further gets exacerbated when client retries the original request. -By default, nutcracker waits indefinitely for any request sent to the server. However, when `timeout:` key is configured, a requests for which no response is received from the server in `timeout:` msec is timedout and an error response `SERVER_ERROR Connection timed out\r\n` is sent back to the client. +By default, nutcracker waits indefinitely for any request sent to the server. However, when `timeout:` key is configured, a requests for which no response is received from the server in `timeout:` msec is timedout and an error response `SERVER_ERROR Connection timed out\r\n` (memcached) or `-ERR Connection timed out\r\n` (redis) is sent back to the client. ## Error Response @@ -115,7 +115,7 @@ [Hash Tags](http://antirez.com/post/redis-presharding.html) enables you to use part of the key for calculating the hash. When the hash tag is present, we use part of the key within the tag as the key to be used for consistent hashing. Otherwise, we use the full key as is. Hash tags enable you to map different keys to the same server as long as the part of the key within the tag is the same. -For example, the configuration of server pool _beta_, aslo shown below, specifies a two character hash_tag string - "{}". This means that keys "user:{user1}:ids" and "user:{user1}:tweets" map to the same server because we compute the hash on "user1". For a key like "user:user1:ids", we use the entire string "user:user1:ids" to compute the hash and it may map to a different server. +For example, the configuration of server pool _beta_, also shown below, specifies a two character hash_tag string - "{}". This means that keys "user:{user1}:ids" and "user:{user1}:tweets" map to the same server because we compute the hash on "user1". For a key like "user:user1:ids", we use the entire string "user:user1:ids" to compute the hash and it may map to a different server. beta: listen: 127.0.0.1:22122 @@ -130,8 +130,7 @@ - 127.0.0.1:6381:1 server2 - 127.0.0.1:6382:1 server3 - 127.0.0.1:6383:1 server4 - - + ## Graphing Cache-pool State When running nutcracker in production, you often would like to know the list of live and ejected servers at any given time. You can easily answer this question, by generating a time series graph of live and/or dead servers that are part of any cache pool. To do this your graphing client must collect the following stats exposed by nutcracker: @@ -155,3 +154,9 @@ By design, twemproxy multiplexes several client connections over few server connections. It is important to note that **"read my last write"** constraint doesn't necessarily hold true when twemproxy is configured with `server_connections: > 1`. To illustrate this, consider a scenario where twemproxy is configured with `server_connections: 2`. If a client makes pipelined requests with the first request in pipeline being `set foo 0 0 3\r\nbar\r\n` (write) and the second request being `get foo\r\n` (read), the expectation is that the read of key `foo` would return the value `bar`. However, with configuration of two server connections it is possible that write and read request are sent on different server connections which would mean that their completion could race with one another. In summary, if the client expects "read my last write" constraint, you either configure twemproxy to use `server_connections:1` or use clients that only make synchronous requests to twemproxy. + +## twemproxy and python-memcached + +The implementation of delete command in [python-memcached](https://github.com/linsomniac/python-memcached) conflicts with the one in twemproxy. See [issue 283](https://github.com/twitter/twemproxy/pull/283) for details. The workaround for this issue is to call `delete_multi` in python-memcached as follows: + + mc.delete_multi([key1, key2, ... keyN], time=None) diff -Nru nutcracker-0.4.0+dfsg/notes/redis.md nutcracker-0.4.1+dfsg/notes/redis.md --- nutcracker-0.4.0+dfsg/notes/redis.md 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/notes/redis.md 2015-06-23 06:06:05.000000000 +0000 @@ -39,7 +39,7 @@ +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | RESTORE | Yes | RESTORE key ttl serialized-value | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - | SORT | Yes* | SORT key [BY pattern] [LIMIT offset count] [GET pattern [GET pattern ...]] [ASC|DESC] [ALPHA] [STORE destination] | + | SORT | Yes | SORT key [BY pattern] [LIMIT offset count] [GET pattern [GET pattern ...]] [ASC|DESC] [ALPHA] [STORE destination] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ | TTL | Yes | TTL key | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ @@ -48,8 +48,6 @@ | SCAN | No | SCAN cursor [MATCH pattern] [COUNT count] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ -* SORT support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](notes/recommendation.md#hash-tags) for all keys in the command. - ### Strings Command +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ @@ -176,7 +174,7 @@ | RPUSHX | Yes | RPUSHX key value | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ -* RPOPLPUSH support requires that source and destination keys hash to the same server. You can ensure this by using the same [hashtag](notes/recommendation.md#hash-tags) for source and destination key. Twemproxy does no checking on its end to verify that source and destination key hash to the same server, and the RPOPLPUSH command is forwarded to the server that the source key hashes to +* RPOPLPUSH support requires that source and destination keys hash to the same server. You can ensure this by using the same [hashtag](recommendation.md#hash-tags) for source and destination key. Twemproxy does no checking on its end to verify that source and destination key hash to the same server, and the RPOPLPUSH command is forwarded to the server that the source key hashes to ### Sets @@ -214,7 +212,7 @@ | SSCAN | Yes | SSCAN key cursor [MATCH pattern] [COUNT count] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ -* SIDFF, SDIFFSTORE, SINTER, SINTERSTORE, SMOVE, SUNION and SUNIONSTORE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](notes/recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to. +* SIDFF, SDIFFSTORE, SINTER, SINTERSTORE, SMOVE, SUNION and SUNIONSTORE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to. ### Sorted Sets @@ -263,7 +261,7 @@ | ZSCAN | Yes | ZSCAN key cursor [MATCH pattern] [COUNT count] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ -* ZINTERSTORE and ZUNIONSTORE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](notes/recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to. +* ZINTERSTORE and ZUNIONSTORE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to. ### HyperLogLog @@ -277,7 +275,7 @@ | PFMERGE | Yes* | PFMERGE destkey sourcekey [sourcekey ...] | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ -* PFMERGE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](notes/recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to. +* PFMERGE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to. ### Pub/Sub @@ -330,7 +328,7 @@ | SCRIPT LOAD | No | SCRIPT LOAD script | +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+ - * EVAL and EVALSHA support is limited to scripts that take at least 1 key. If multiple keys are used, all keys must hash to the same server. You can ensure this by using the same [hashtag](notes/recommendation.md#hash-tags) for all keys. If you use more than 1 key, the proxy does no checking to verify that all keys hash to the same server, and the entire command is forwarded to the server that the first key hashes to + * EVAL and EVALSHA support is limited to scripts that take at least 1 key. If multiple keys are used, all keys must hash to the same server. You can ensure this by using the same [hashtag](recommendation.md#hash-tags) for all keys. If you use more than 1 key, the proxy does no checking to verify that all keys hash to the same server, and the entire command is forwarded to the server that the first key hashes to ### Connection @@ -399,7 +397,7 @@ ## Note - redis commands are not case sensitive -- only vectored commands 'MGET key [key ...]' and 'DEL key [key ...]' needs to be fragmented +- only vectored commands 'MGET key [key ...]', 'MSET key value [key value ...]', 'DEL key [key ...]' needs to be fragmented ## Performance @@ -445,3 +443,19 @@ LRANGE_500 (first 450 elements): 8605.85 requests per second LRANGE_600 (first 600 elements): 6587.62 requests per second +## redis-auth feature + ++ you can enable redis-auth for a pool with 'redis_auth': + + alpha: + listen: 127.0.0.1:22121 + hash: fnv1a_64 + distribution: ketama + redis: true + redis_auth: testpass + ++ notice: + + *MUST* set all redis with a same passwd, and all twemproxy with the same passwd + + Length of password should less than 256 + + diff -Nru nutcracker-0.4.0+dfsg/README.md nutcracker-0.4.1+dfsg/README.md --- nutcracker-0.4.0+dfsg/README.md 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/README.md 2015-06-23 06:06:05.000000000 +0000 @@ -1,30 +1,37 @@ # twemproxy (nutcracker) [![Build Status](https://secure.travis-ci.org/twitter/twemproxy.png)](http://travis-ci.org/twitter/twemproxy) -**twemproxy** (pronounced "two-em-proxy"), aka **nutcracker** is a fast and lightweight proxy for [memcached](http://www.memcached.org/) and [redis](http://redis.io/) protocol. It was primarily built to reduce the connection count on the backend caching servers. +**twemproxy** (pronounced "two-em-proxy"), aka **nutcracker** is a fast and lightweight proxy for [memcached](http://www.memcached.org/) and [redis](http://redis.io/) protocol. It was built primarily to reduce the number of connections to the caching servers on the backend. This, together with protocol pipelining and sharding enables you to horizontally scale your distributed caching architecture. ## Build -To build nutcracker from [distribution tarball](http://code.google.com/p/twemproxy/downloads/list): +To build twemproxy from [distribution tarball](https://drive.google.com/open?id=0B6pVMMV5F5dfMUdJV25abllhUWM&authuser=0): $ ./configure $ make $ sudo make install -To build nutcracker from [distribution tarball](http://code.google.com/p/twemproxy/downloads/list) in _debug mode_: +To build twemproxy from [distribution tarball](https://drive.google.com/open?id=0B6pVMMV5F5dfMUdJV25abllhUWM&authuser=0) in _debug mode_: $ CFLAGS="-ggdb3 -O0" ./configure --enable-debug=full $ make $ sudo make install -To build nutcracker from source with _debug logs enabled_ and _assertions disabled_: +To build twemproxy from source with _debug logs enabled_ and _assertions enabled_: $ git clone git@github.com:twitter/twemproxy.git $ cd twemproxy $ autoreconf -fvi - $ ./configure --enable-debug=log + $ ./configure --enable-debug=full $ make $ src/nutcracker -h +A quick checklist: + ++ Use newer version of gcc (older version of gcc has problems) ++ Use CFLAGS="-O1" ./configure && make ++ Use CFLAGS="-O3 -fno-strict-aliasing" ./configure && make ++ `autoreconf -fvi && ./configure` needs `automake` and `libtool` to be installed + ## Features + Fast. @@ -35,12 +42,12 @@ + Supports proxying to multiple servers. + Supports multiple server pools simultaneously. + Shard data automatically across multiple servers. -+ Implements the complete [memcached ascii](notes/memcache.txt) and [redis](notes/redis.md) protocol. ++ Implements the complete [memcached ascii](notes/memcache.md) and [redis](notes/redis.md) protocol. + Easy configuration of server pools through a YAML file. + Supports multiple hashing modes including consistent hashing and distribution. + Can be configured to disable nodes on failures. -+ Observability through stats exposed on stats monitoring port. -+ Works with Linux, *BSD, OS X and Solaris (SmartOS) ++ Observability via stats exposed on the stats monitoring port. ++ Works with Linux, *BSD, OS X and SmartOS (Solaris) ## Help @@ -54,7 +61,7 @@ -t, --test-conf : test configuration for syntax errors and exit -d, --daemonize : run as a daemon -D, --describe-stats : print stats description and exit - -v, --verbosity=N : set logging level (default: 5, min: 0, max: 11) + -v, --verbose=N : set logging level (default: 5, min: 0, max: 11) -o, --output=S : set logging file (default: stderr) -c, --conf-file=S : set configuration file (default: conf/nutcracker.yml) -s, --stats-port=N : set stats monitoring port (default: 22222) @@ -65,15 +72,15 @@ ## Zero Copy -In nutcracker, all the memory for incoming requests and outgoing responses is allocated in mbuf. Mbuf enables zero-copy because the same buffer on which a request was received from the client is used for forwarding it to the server. Similarly the same mbuf on which a response was received from the server is used for forwarding it to the client. +In twemproxy, all the memory for incoming requests and outgoing responses is allocated in mbuf. Mbuf enables zero-copy because the same buffer on which a request was received from the client is used for forwarding it to the server. Similarly the same mbuf on which a response was received from the server is used for forwarding it to the client. -Furthermore, memory for mbufs is managed using a reuse pool. This means that once mbuf is allocated, it is not deallocated, but just put back into the reuse pool. By default each mbuf chunk is set to 16K bytes in size. There is a trade-off between the mbuf size and number of concurrent connections nutcracker can support. A large mbuf size reduces the number of read syscalls made by nutcracker when reading requests or responses. However, with large mbuf size, every active connection would use up 16K bytes of buffer which might be an issue when nutcracker is handling large number of concurrent connections from clients. When nutcracker is meant to handle a large number of concurrent client connections, you should set chunk size to a small value like 512 bytes using the -m or --mbuf-size=N argument. +Furthermore, memory for mbufs is managed using a reuse pool. This means that once mbuf is allocated, it is not deallocated, but just put back into the reuse pool. By default each mbuf chunk is set to 16K bytes in size. There is a trade-off between the mbuf size and number of concurrent connections twemproxy can support. A large mbuf size reduces the number of read syscalls made by twemproxy when reading requests or responses. However, with a large mbuf size, every active connection would use up 16K bytes of buffer which might be an issue when twemproxy is handling large number of concurrent connections from clients. When twemproxy is meant to handle a large number of concurrent client connections, you should set chunk size to a small value like 512 bytes using the -m or --mbuf-size=N argument. ## Configuration -nutcracker can be configured through a YAML file specified by the -c or --conf-file command-line argument on process start. The configuration file is used to specify the server pools and the servers within each pool that nutcracker manages. The configuration files parses and understands the following keys: +Twemproxy can be configured through a YAML file specified by the -c or --conf-file command-line argument on process start. The configuration file is used to specify the server pools and the servers within each pool that twemproxy manages. The configuration files parses and understands the following keys: -+ **listen**: The listening address and port (name:port or ip:port) for this server pool. ++ **listen**: The listening address and port (name:port or ip:port) or an absolute path to sock file (e.g. /var/run/nutcracker.sock) for this server pool. + **hash**: The name of the hash function. Possible values are: + one_at_a_time + md5 @@ -87,15 +94,17 @@ + hsieh + murmur + jenkins -+ **hash_tag**: A two character string that specifies the part of the key used for hashing. Eg "{}" or "$$". [Hash tag](notes/recommendation.md#hash-tags) enable mapping different keys to the same server as long as the part of the key within the tag is the same. ++ **hash_tag**: A two character string that specifies the part of the key used for hashing. Eg "{}" or "$$". [Hash tag](notes/recommendation.md#hash-tags) enable mapping different keys to the same server as long as the part of the key within the tag is the same. + **distribution**: The key distribution mode. Possible values are: + ketama + modula + random + **timeout**: The timeout value in msec that we wait for to establish a connection to the server or receive a response from a server. By default, we wait indefinitely. + **backlog**: The TCP backlog argument. Defaults to 512. -+ **preconnect**: A boolean value that controls if nutcracker should preconnect to all the servers in this pool on process start. Defaults to false. ++ **preconnect**: A boolean value that controls if twemproxy should preconnect to all the servers in this pool on process start. Defaults to false. + **redis**: A boolean value that controls if a server pool speaks redis or memcached protocol. Defaults to false. ++ **redis_auth**: Authenticate to the Redis server on connect. ++ **redis_db**: The DB number to use on the pool servers. Defaults to 0. Note: Twemproxy will always present itself to clients as DB 0. + **server_connections**: The maximum number of connections that can be opened to each server. By default, we open at most 1 server connection. + **auto_eject_hosts**: A boolean value that controls if server should be ejected temporarily when it fails consecutively server_failure_limit times. See [liveness recommendations](notes/recommendation.md#liveness) for information. Defaults to false. + **server_retry_timeout**: The timeout value in msec to wait for before retrying on a temporarily ejected server, when auto_eject_host is set to true. Defaults to 30000 msec. @@ -103,7 +112,7 @@ + **servers**: A list of server address, port and weight (name:port:weight or ip:port:weight) for this server pool. -For example, the configuration file in [conf/nutcracker.yml](conf/nutcracker.yml), also shown below, configures 5 server pools with names - _alpha_, _beta_, _gamma_, _delta_ and omega. Clients that intend to send requests to one of the 10 servers in pool delta connect to port 22124 on 127.0.0.1. Clients that intend to send request to one of 2 servers in pool omega connect to unix path /tmp/gamma. Requests sent to pool alpha and omega have no timeout and might require timeout functionality to be implemented on the client side. On the other hand, requests sent to pool beta, gamma and delta timeout after 400 msec, 400 msec and 100 msec respectively when no response is received from the server. Of the 5 server pools, only pools alpha, gamma and delta are configured to use server ejection and hence are resilient to server failures. All the 5 server pools use ketama consistent hashing for key distribution with the key hasher for pools alpha, beta, gamma and delta set to fnv1a_64 while that for pool omega set to hsieh. Also only pool beta uses [nodes names](notes/recommendation.md#node-names-for-consistent-hashing) for consistent hashing, while pool alpha, gamma, delta and omega use 'host:port:weight' for consistent hashing. Finally, only pool alpha and beta can speak redis protocol, while pool gamma, deta and omega speak memcached protocol. +For example, the configuration file in [conf/nutcracker.yml](conf/nutcracker.yml), also shown below, configures 5 server pools with names - _alpha_, _beta_, _gamma_, _delta_ and omega. Clients that intend to send requests to one of the 10 servers in pool delta connect to port 22124 on 127.0.0.1. Clients that intend to send request to one of 2 servers in pool omega connect to unix path /tmp/gamma. Requests sent to pool alpha and omega have no timeout and might require timeout functionality to be implemented on the client side. On the other hand, requests sent to pool beta, gamma and delta timeout after 400 msec, 400 msec and 100 msec respectively when no response is received from the server. Of the 5 server pools, only pools alpha, gamma and delta are configured to use server ejection and hence are resilient to server failures. All the 5 server pools use ketama consistent hashing for key distribution with the key hasher for pools alpha, beta, gamma and delta set to fnv1a_64 while that for pool omega set to hsieh. Also only pool beta uses [nodes names](notes/recommendation.md#node-names-for-consistent-hashing) for consistent hashing, while pool alpha, gamma, delta and omega use 'host:port:weight' for consistent hashing. Finally, only pool alpha and beta can speak the redis protocol, while pool gamma, deta and omega speak memcached protocol. alpha: listen: 127.0.0.1:22121 @@ -173,13 +182,13 @@ - 127.0.0.1:11214:100000 - 127.0.0.1:11215:1 -Finally, to make writing syntactically correct configuration file easier, nutcracker provides a command-line argument -t or --test-conf that can be used to test the YAML configuration file for any syntax error. +Finally, to make writing a syntactically correct configuration file easier, twemproxy provides a command-line argument -t or --test-conf that can be used to test the YAML configuration file for any syntax error. ## Observability -Observability in nutcracker is through logs and stats. +Observability in twemproxy is through logs and stats. -Nutcracker exposes stats at the granularity of server pool and servers per pool through the stats monitoring port. The stats are essentially JSON formatted key-value pairs, with the keys corresponding to counter names. By default stats are exposed on port 22222 and aggregated every 30 seconds. Both these values can be configured on program start using the -c or --conf-file and -i or --stats-interval command-line arguments respectively. You can print the description of all stats exported by nutcracker using the -D or --describe-stats command-line argument. +Twemproxy exposes stats at the granularity of server pool and servers per pool through the stats monitoring port. The stats are essentially JSON formatted key-value pairs, with the keys corresponding to counter names. By default stats are exposed on port 22222 and aggregated every 30 seconds. Both these values can be configured on program start using the -c or --conf-file and -i or --stats-interval command-line arguments respectively. You can print the description of all stats exported by using the -D or --describe-stats command-line argument. $ nutcracker --describe-stats @@ -205,20 +214,31 @@ out_queue "# requests in outgoing queue" out_queue_bytes "current request bytes in outgoing queue" -Logging in nutcracker is only available when nutcracker is built with logging enabled. By default logs are written to stderr. Nutcracker can also be configured to write logs to a specific file through the -o or --output command-line argument. On a running nutcracker, we can turn log levels up and down by sending it SIGTTIN and SIGTTOU signals respectively and reopen log files by sending it SIGHUP signal. +Logging in twemproxy is only available when twemproxy is built with logging enabled. By default logs are written to stderr. Twemproxy can also be configured to write logs to a specific file through the -o or --output command-line argument. On a running twemproxy, we can turn log levels up and down by sending it SIGTTIN and SIGTTOU signals respectively and reopen log files by sending it SIGHUP signal. ## Pipelining +Twemproxy enables proxying multiple client connections onto one or few server connections. This architectural setup makes it ideal for pipelining requests and responses and hence saving on the round trip time. -Nutcracker enables proxying multiple client connections onto one or few server connections. This architectural setup makes it ideal for pipelining requests and responses and hence saving on the round trip time. - -For example, if nutcracker is proxying three client connections onto a single server and we get requests - 'get key\r\n', 'set key 0 0 3\r\nval\r\n' and 'delete key\r\n' on these three connections respectively, nutcracker would try to batch these requests and send them as a single message onto the server connection as 'get key\r\nset key 0 0 3\r\nval\r\ndelete key\r\n'. +For example, if twemproxy is proxying three client connections onto a single server and we get requests - 'get key\r\n', 'set key 0 0 3\r\nval\r\n' and 'delete key\r\n' on these three connections respectively, twemproxy would try to batch these requests and send them as a single message onto the server connection as 'get key\r\nset key 0 0 3\r\nval\r\ndelete key\r\n'. -Pipelining is the reason why nutcracker ends up doing better in terms of throughput even though it introduces an extra hop between the client and server. +Pipelining is the reason why twemproxy ends up doing better in terms of throughput even though it introduces an extra hop between the client and server. ## Deployment -If you are deploying nutcracker in production, you might consider reading through the [recommendation document](notes/recommendation.md) to understand the parameters you could tune in nutcracker to run it efficiently in the production environment. +If you are deploying twemproxy in production, you might consider reading through the [recommendation document](notes/recommendation.md) to understand the parameters you could tune in twemproxy to run it efficiently in the production environment. + +## Packages + +### Ubuntu + +#### PPA Stable + +https://launchpad.net/~twemproxy/+archive/ubuntu/stable + +#### PPA Daily + +https://launchpad.net/~twemproxy/+archive/ubuntu/daily ## Utils + [nagios checks](https://github.com/wanelo/nagios-checks/blob/master/check_twemproxy) @@ -231,8 +251,11 @@ + [sensu-metrics](https://github.com/sensu/sensu-community-plugins/blob/master/plugins/twemproxy/twemproxy-metrics.rb) + [redis-mgr](https://github.com/idning/redis-mgr) + [smitty for twemproxy failover](https://github.com/areina/smitty) ++ [Beholder, a Python agent for twemproxy failover](https://github.com/Serekh/beholder) ++ [chef cookbook](https://supermarket.getchef.com/cookbooks/twemproxy) ++ [twemsentinel] (https://github.com/yak0/twemsentinel) -## Users +## Companies using Twemproxy in Production + [Pinterest](http://pinterest.com/) + [Tumblr](https://www.tumblr.com/) + [Twitter](https://twitter.com/) @@ -250,6 +273,18 @@ + [3scale.net](http://3scale.net) + [Ooyala](http://www.ooyala.com) + [Twitch](http://twitch.tv) ++ [Socrata](http://www.socrata.com/) ++ [Hootsuite](http://hootsuite.com/) ++ [Trivago](http://www.trivago.com/) ++ [Machinezone](http://www.machinezone.com) ++ [Flickr](https://www.flickr.com) ++ [Yahoo!](https://www.yahoo.com) ++ [Path](https://path.com) ++ [AOL](http://engineering.aol.com/) ++ [Soysuper](https://soysuper.com/) ++ [Vinted](http://vinted.com/) ++ [Poshmark](https://poshmark.com/) ++ [FanDuel](https://www.fanduel.com/) ## Issues and Support diff -Nru nutcracker-0.4.0+dfsg/scripts/nutcracker.init nutcracker-0.4.1+dfsg/scripts/nutcracker.init --- nutcracker-0.4.0+dfsg/scripts/nutcracker.init 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/scripts/nutcracker.init 2015-06-23 06:06:05.000000000 +0000 @@ -26,6 +26,16 @@ start () { echo -n $"Starting $prog: " + #Test the config before start. + daemon --user ${USER} ${prog} $OPTIONS -t >/dev/null 2>&1 + RETVAL=$? + if [ $RETVAL -ne 0 ] ; then + echo "Config check fail! Please use 'nutcracker -c /etc/nutcracker/nutcracker.yml' for detail." + echo_failure; + echo; + exit 1 + fi + daemon --user ${USER} ${prog} $OPTIONS RETVAL=$? echo diff -Nru nutcracker-0.4.0+dfsg/scripts/nutcracker.init.debian nutcracker-0.4.1+dfsg/scripts/nutcracker.init.debian --- nutcracker-0.4.0+dfsg/scripts/nutcracker.init.debian 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/scripts/nutcracker.init.debian 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,83 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: nutcracker +# Required-Start: $network $remote_fs $local_fs +# Required-Stop: $network $remote_fs $local_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Stop/start nutcracker +### END INIT INFO + +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC=nutcracker +NAME=nutcracker +USER=nobody +CONFFILE=/opt/nutcracker/etc/$NAME.yml +LOGFILE=/opt/nutcracker/log/nutcracker.log +DAEMON=/opt/nutcracker/sbin/nutcracker +PIDFILE=/var/run/nutcracker/$NAME.pid +STATSPORT=22222 +DAEMON_ARGS="-c $CONFFILE -o $LOGFILE -p $PIDFILE -s $STATSPORT -v 11 -m 2048 -d" +#DAEMON_ARGS="-c $CONFFILE -p $PIDFILE -s $STATSPORT -d" +SCRIPTNAME=/etc/init.d/$NAME + +ulimit -Hn 100000 +ulimit -Sn 100000 + +[ -x $DAEMON ] || exit 0 + +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +. /lib/init/vars.sh + +. /lib/lsb/init-functions + +do_start() +{ + mkdir -p /var/run/nutcracker + touch $PIDFILE + chown $USER:$USER -R /var/run/nutcracker + chmod 755 /var/run/nutcracker + + echo -n "Starting ${NAME}: " + start-stop-daemon --start --quiet -m --pidfile $PIDFILE --chuid $USER:$USER --exec $DAEMON -- \ + $DAEMON_ARGS + case "$?" in + 0|1) echo "STARTED." ;; + 2) echo "FAILED." ;; + esac +} + +do_stop() +{ + echo -n "Stopping ${NAME}: " + start-stop-daemon --stop --quiet --pidfile $PIDFILE --exec $DAEMON || true + + case "$?" in + 0|1) echo "STOPPED.";; + 2) echo "FAILED." ;; + esac +} + +case "$1" in + start) + do_start + ;; + stop) + do_stop + ;; + status) + status_of_proc -p $PIDFILE "$DAEMON" nutcracker && exit 0 || exit $? + ;; + restart) + do_stop + do_start + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart}" >&2 + exit 3 + ;; +esac + +exit +$RETVAL diff -Nru nutcracker-0.4.0+dfsg/scripts/nutcracker.spec nutcracker-0.4.1+dfsg/scripts/nutcracker.spec --- nutcracker-0.4.0+dfsg/scripts/nutcracker.spec 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/scripts/nutcracker.spec 2015-06-23 06:06:05.000000000 +0000 @@ -1,6 +1,6 @@ Summary: Twitter's nutcracker redis and memcached proxy Name: nutcracker -Version: 0.3.0 +Version: 0.4.1 Release: 1 URL: https://github.com/twitter/twemproxy/ @@ -16,7 +16,8 @@ %description twemproxy (pronounced "two-em-proxy"), aka nutcracker is a fast and lightweight proxy for memcached and redis protocol. -It was primarily built to reduce the connection count on the backend caching servers. +It was primarily built to reduce the connection count on the backend caching servers. This, together with protocol +pipelining and sharding enables you to horizontally scale your distributed caching architecture. %prep %setup -q @@ -65,6 +66,39 @@ %config(noreplace)%{_sysconfdir}/%{name}/%{name}.yml %changelog +* Mon Jun 22 2015 Manju Rajashekhar +- twemproxy: version 0.4.1 release +- redis_auth is only valid for a redis pool +- getaddrinfo returns non-zero +ve value on error +- fix-hang-when-command-only (charsyam) +- fix bug crash when get command without key and whitespace (charsyam) +- mark server as failed on protocol level transiet failures like -OOM, -LOADING, etc +- implemented support for parsing fine grained redis error response +- remove redundant conditional judgement in rbtree deletion (leo ma) +- fix bug mset has invalid pair (charsyam) +- fix bug mset has invalid pair (charsyam) +- temp fix a core on kqueue (idning) +- support "touch" command for memcached (panmiaocai) +- fix redis parse rsp bug (charsyam) +- SORT command can take multiple arguments. So it should be part of redis_argn() and not redis_arg0() +- remove incorrect assert because client could send data after sending a quit request which must be discarded +- allow file permissions to be set for UNIX domain listening socket (ori liveneh) +- return error if formatted is greater than mbuf size by using nc_vsnprintf() in msg_prepend_format() +- fix req_make_reply on msg_get, mark it as response (idning) +- redis database select upon connect (arne claus) +- redis_auth (charsyam) +- allow null key(empty key) (idning) +- fix core on invalid mset like "mset a a a" (idning) + +* Tue Oct 18 2014 idning +- twemproxy: version 0.4.0 release +- mget improve (idning) +- many new commands supported: LEX, PFADD, PFMERGE, SORT, PING, QUIT, SCAN... (mattrobenolt, areina, idning) +- handle max open file limit(allenlz) +- add notice-log and use ms time in log(idning) +- fix bug in string_compare (andyqzb) +- fix deadlock in sighandler (idning) + * Fri Dec 20 2013 Manju Rajashekhar - twemproxy: version 0.3.0 release - SRANDMEMBER support for the optional count argument (mkhq) diff -Nru nutcracker-0.4.0+dfsg/scripts/redis-check.sh nutcracker-0.4.1+dfsg/scripts/redis-check.sh --- nutcracker-0.4.0+dfsg/scripts/redis-check.sh 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/scripts/redis-check.sh 2015-06-23 06:06:05.000000000 +0000 @@ -616,3 +616,7 @@ printf '*7\r\n$4\r\neval\r\n$40\r\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}\r\n$1\r\n2\r\n$9\r\nkey1{tag}\r\n$4\r\narg1\r\n$9\r\nkey2{tag}\r\n$4\r\narg2\r\n' | socat ${debug} ${timeout} - TCP:localhost:${port},shut-close printf '*9\r\n$4\r\neval\r\n$56\r\nreturn {KEYS[1],KEYS[2],KEYS[3],ARGV[1],ARGV[2],ARGV[3]}\r\n$1\r\n3\r\n$9\r\nkey1{tag}\r\n$4\r\narg1\r\n$9\r\nkey2{tag}\r\n$4\r\narg2\r\n$9\r\nkey3{tag}\r\n$4\r\narg3\r\n' | socat ${debug} ${timeout} - TCP:localhost:${port},shut-close printf '*4\r\n$4\r\neval\r\n$11\r\nreturn {10}\r\n$1\r\n1\r\n$4\r\nTEMP\r\n' | socat ${debug} ${timeout} - TCP:localhost:${port},shut-close + +# quit +printf '\nquit\n' +printf '*1\r\n$4\r\nquit\r\n' | socat ${debug} ${timeout} - TCP:localhost:${port},shut-close diff -Nru nutcracker-0.4.0+dfsg/src/hashkit/nc_jenkins.c nutcracker-0.4.1+dfsg/src/hashkit/nc_jenkins.c --- nutcracker-0.4.0+dfsg/src/hashkit/nc_jenkins.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/hashkit/nc_jenkins.c 2015-06-23 06:06:05.000000000 +0000 @@ -106,7 +106,7 @@ * rest of the string. Every machine with memory protection I've seen * does it on word boundaries, so is OK with this. But VALGRIND will * still catch it and complain. The masking trick does make the hash - * noticably faster for short strings (like English words). + * noticeably faster for short strings (like English words). */ switch(length) { diff -Nru nutcracker-0.4.0+dfsg/src/hashkit/nc_ketama.c nutcracker-0.4.1+dfsg/src/hashkit/nc_ketama.c --- nutcracker-0.4.0+dfsg/src/hashkit/nc_ketama.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/hashkit/nc_ketama.c 2015-06-23 06:06:05.000000000 +0000 @@ -162,10 +162,10 @@ pointer_per_server = (uint32_t) ((floorf((float) (pct * KETAMA_POINTS_PER_SERVER / 4 * (float)nlive_server + 0.0000000001))) * 4); pointer_per_hash = 4; - log_debug(LOG_VERB, "%.*s:%"PRIu16" weight %"PRIu32" of %"PRIu32" " + log_debug(LOG_VERB, "%.*s weight %"PRIu32" of %"PRIu32" " "pct %0.5f points per server %"PRIu32"", - server->name.len, server->name.data, server->port, - server->weight, total_weight, pct, pointer_per_server); + server->name.len, server->name.data, server->weight, + total_weight, pct, pointer_per_server); for (pointer_index = 1; pointer_index <= pointer_per_server / pointer_per_hash; diff -Nru nutcracker-0.4.0+dfsg/src/Makefile.am nutcracker-0.4.1+dfsg/src/Makefile.am --- nutcracker-0.4.0+dfsg/src/Makefile.am 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/Makefile.am 2015-06-23 06:06:05.000000000 +0000 @@ -9,7 +9,9 @@ AM_CPPFLAGS += -I $(top_srcdir)/src/event AM_CPPFLAGS += -I $(top_srcdir)/contrib/yaml-0.1.4/include -AM_CFLAGS = +AM_CFLAGS = +# about -fno-strict-aliasing: https://github.com/twitter/twemproxy/issues/276 +AM_CFLAGS += -fno-strict-aliasing AM_CFLAGS += -Wall -Wshadow AM_CFLAGS += -Wpointer-arith AM_CFLAGS += -Winline @@ -23,6 +25,9 @@ if OS_SOLARIS AM_LDFLAGS += -lnsl -lsocket endif +if OS_FREEBSD +AM_LDFLAGS += -lexecinfo +endif SUBDIRS = hashkit proto event diff -Nru nutcracker-0.4.0+dfsg/src/nc.c nutcracker-0.4.1+dfsg/src/nc.c --- nutcracker-0.4.0+dfsg/src/nc.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc.c 2015-06-23 06:06:05.000000000 +0000 @@ -214,7 +214,7 @@ " -d, --daemonize : run as a daemon" CRLF " -D, --describe-stats : print stats description and exit"); log_stderr( - " -v, --verbosity=N : set logging level (default: %d, min: %d, max: %d)" CRLF + " -v, --verbose=N : set logging level (default: %d, min: %d, max: %d)" CRLF " -o, --output=S : set logging file (default: %s)" CRLF " -c, --conf-file=S : set configuration file (default: %s)" CRLF " -s, --stats-port=N : set stats monitoring port (default: %d)" CRLF diff -Nru nutcracker-0.4.0+dfsg/src/nc_conf.c nutcracker-0.4.1+dfsg/src/nc_conf.c --- nutcracker-0.4.0+dfsg/src/nc_conf.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_conf.c 2015-06-23 06:06:05.000000000 +0000 @@ -74,6 +74,14 @@ conf_set_bool, offsetof(struct conf_pool, redis) }, + { string("redis_auth"), + conf_set_string, + offsetof(struct conf_pool, redis_auth) }, + + { string("redis_db"), + conf_set_num, + offsetof(struct conf_pool, redis_db) }, + { string("preconnect"), conf_set_bool, offsetof(struct conf_pool, preconnect) }, @@ -106,6 +114,7 @@ { string_init(&cs->pname); string_init(&cs->name); + string_init(&cs->addrstr); cs->port = 0; cs->weight = 0; @@ -121,6 +130,7 @@ { string_deinit(&cs->pname); string_deinit(&cs->name); + string_deinit(&cs->addrstr); cs->valid = 0; log_debug(LOG_VVERB, "deinit conf server %p", cs); } @@ -142,12 +152,11 @@ s->pname = cs->pname; s->name = cs->name; + s->addrstr = cs->addrstr; s->port = (uint16_t)cs->port; s->weight = (uint32_t)cs->weight; - s->family = cs->info.family; - s->addrlen = cs->info.addrlen; - s->addr = (struct sockaddr *)&cs->info.addr; + nc_memcpy(&s->info, &cs->info, sizeof(cs->info)); s->ns_conn_q = 0; TAILQ_INIT(&s->s_conn_q); @@ -170,6 +179,7 @@ string_init(&cp->listen.pname); string_init(&cp->listen.name); + string_init(&cp->redis_auth); cp->listen.port = 0; memset(&cp->listen.info, 0, sizeof(cp->listen.info)); cp->listen.valid = 0; @@ -184,6 +194,7 @@ cp->client_connections = CONF_UNSET_NUM; cp->redis = CONF_UNSET_NUM; + cp->redis_db = CONF_UNSET_NUM; cp->preconnect = CONF_UNSET_NUM; cp->auto_eject_hosts = CONF_UNSET_NUM; cp->server_connections = CONF_UNSET_NUM; @@ -219,6 +230,10 @@ string_deinit(&cp->listen.pname); string_deinit(&cp->listen.name); + if (cp->redis_auth.len > 0) { + string_deinit(&cp->redis_auth); + } + while (array_n(&cp->server) != 0) { conf_server_deinit(array_pop(&cp->server)); } @@ -258,9 +273,8 @@ sp->addrstr = cp->listen.pname; sp->port = (uint16_t)cp->listen.port; - sp->family = cp->listen.info.family; - sp->addrlen = cp->listen.info.addrlen; - sp->addr = (struct sockaddr *)&cp->listen.info.addr; + nc_memcpy(&sp->info, &cp->listen.info, sizeof(cp->listen.info)); + sp->perm = cp->listen.perm; sp->key_hash_type = cp->hash; sp->key_hash = hash_algos[cp->hash]; @@ -270,9 +284,12 @@ sp->redis = cp->redis ? 1 : 0; sp->timeout = cp->timeout; sp->backlog = cp->backlog; + sp->redis_db = cp->redis_db; - sp->client_connections = (uint32_t)cp->client_connections; + sp->redis_auth = cp->redis_auth; + sp->require_auth = cp->redis_auth.len > 0 ? 1 : 0; + sp->client_connections = (uint32_t)cp->client_connections; sp->server_connections = (uint32_t)cp->server_connections; sp->server_retry_timeout = (int64_t)cp->server_retry_timeout * 1000LL; sp->server_failure_limit = (uint32_t)cp->server_failure_limit; @@ -1161,7 +1178,7 @@ if (string_compare(&cs1->name, &cs2->name) == 0) { log_error("conf: pool '%.*s' has servers with same name '%.*s'", - cp->name.len, cp->name.data, cs1->name.len, + cp->name.len, cp->name.data, cs1->name.len, cs1->name.data); valid = false; break; @@ -1211,6 +1228,10 @@ cp->redis = CONF_DEFAULT_REDIS; } + if (cp->redis_db == CONF_UNSET_NUM) { + cp->redis_db = CONF_DEFAULT_REDIS_DB; + } + if (cp->preconnect == CONF_UNSET_NUM) { cp->preconnect = CONF_DEFAULT_PRECONNECT; } @@ -1234,6 +1255,11 @@ cp->server_failure_limit = CONF_DEFAULT_SERVER_FAILURE_LIMIT; } + if (!cp->redis && cp->redis_auth.len > 0) { + log_error("conf: directive \"redis_auth:\" is only valid for a redis pool"); + return NC_ERROR; + } + status = conf_validate_server(cf, cp); if (status != NC_OK) { return status; @@ -1420,8 +1446,32 @@ } if (value->data[0] == '/') { - name = value->data; - namelen = value->len; + uint8_t *q, *start, *perm; + uint32_t permlen; + + + /* parse "socket_path permissions" from the end */ + p = value->data + value->len -1; + start = value->data; + q = nc_strrchr(p, start, ' '); + if (q == NULL) { + /* no permissions field, so use defaults */ + name = value->data; + namelen = value->len; + } else { + perm = q + 1; + permlen = (uint32_t)(p - perm + 1); + + p = q - 1; + name = start; + namelen = (uint32_t)(p - start + 1); + + errno = 0; + field->perm = (mode_t)strtol((char *)perm, NULL, 8); + if (errno || field->perm > 0777) { + return "has an invalid file permission in \"socket_path permission\" format string"; + } + } } else { uint8_t *q, *start, *port; uint32_t portlen; @@ -1473,10 +1523,8 @@ uint8_t *p, *q, *start; uint8_t *pname, *addr, *port, *weight, *name; uint32_t k, delimlen, pnamelen, addrlen, portlen, weightlen, namelen; - struct string address; char delim[] = " ::"; - string_init(&address); p = conf; a = (struct array *)(p + cmd->offset); @@ -1588,18 +1636,18 @@ return CONF_ERROR; } - status = string_copy(&address, addr, addrlen); + status = string_copy(&field->addrstr, addr, addrlen); if (status != NC_OK) { return CONF_ERROR; } - status = nc_resolve(&address, field->port, &field->info); - if (status != NC_OK) { - string_deinit(&address); - return CONF_ERROR; - } + /* + * The address resolution of the backend server hostname is lazy. + * The resolution occurs when a new connection to the server is + * created, which could either be the first time or every time + * the server gets re-added to the pool after an auto ejection + */ - string_deinit(&address); field->valid = 1; return CONF_OK; diff -Nru nutcracker-0.4.0+dfsg/src/nc_conf.h nutcracker-0.4.1+dfsg/src/nc_conf.h --- nutcracker-0.4.0+dfsg/src/nc_conf.h 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_conf.h 2015-06-23 06:06:05.000000000 +0000 @@ -47,6 +47,7 @@ #define CONF_DEFAULT_LISTEN_BACKLOG 512 #define CONF_DEFAULT_CLIENT_CONNECTIONS 0 #define CONF_DEFAULT_REDIS false +#define CONF_DEFAULT_REDIS_DB 0 #define CONF_DEFAULT_PRECONNECT false #define CONF_DEFAULT_AUTO_EJECT_HOSTS false #define CONF_DEFAULT_SERVER_RETRY_TIMEOUT 30 * 1000 /* in msec */ @@ -55,16 +56,18 @@ #define CONF_DEFAULT_KETAMA_PORT 11211 struct conf_listen { - struct string pname; /* listen: as "name:port" */ - struct string name; /* name */ + struct string pname; /* listen: as "hostname:port" */ + struct string name; /* hostname:port */ int port; /* port */ + mode_t perm; /* socket permissions */ struct sockinfo info; /* listen socket info */ unsigned valid:1; /* valid? */ }; struct conf_server { - struct string pname; /* server: as "name:port:weight" */ - struct string name; /* name */ + struct string pname; /* server: as "hostname:port:weight" */ + struct string name; /* hostname:port or [name] */ + struct string addrstr; /* hostname */ int port; /* port */ int weight; /* weight */ struct sockinfo info; /* connect socket info */ @@ -81,6 +84,8 @@ int backlog; /* backlog: */ int client_connections; /* client_connections: */ int redis; /* redis: */ + struct string redis_auth; /* redis_auth: redis auth password (matches requirepass on redis) */ + int redis_db; /* redis_db: redis db */ int preconnect; /* preconnect: */ int auto_eject_hosts; /* auto_eject_hosts: */ int server_connections; /* server_connections: */ diff -Nru nutcracker-0.4.0+dfsg/src/nc_connection.c nutcracker-0.4.1+dfsg/src/nc_connection.c --- nutcracker-0.4.0+dfsg/src/nc_connection.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_connection.c 2015-06-23 06:06:05.000000000 +0000 @@ -156,6 +156,7 @@ conn->eof = 0; conn->done = 0; conn->redis = 0; + conn->authenticated = 0; ntotal_conn++; ncurr_conn++; @@ -201,7 +202,9 @@ conn->dequeue_inq = NULL; conn->enqueue_outq = req_client_enqueue_omsgq; conn->dequeue_outq = req_client_dequeue_omsgq; - + conn->post_connect = NULL; + conn->swallow_msg = NULL; + ncurr_cconn++; } else { /* @@ -226,10 +229,16 @@ conn->dequeue_inq = req_server_dequeue_imsgq; conn->enqueue_outq = req_server_enqueue_omsgq; conn->dequeue_outq = req_server_dequeue_omsgq; + if (redis) { + conn->post_connect = redis_post_connect; + conn->swallow_msg = redis_swallow_msg; + } else { + conn->post_connect = memcache_post_connect; + conn->swallow_msg = memcache_swallow_msg; + } } conn->ref(conn, owner); - log_debug(LOG_VVERB, "get conn %p client %d", conn, conn->client); return conn; @@ -438,3 +447,27 @@ { return ncurr_cconn; } + +/* + * Returns true if the connection is authenticated or doesn't require + * authentication, otherwise return false + */ +bool +conn_authenticated(struct conn *conn) +{ + struct server_pool *pool; + + ASSERT(!conn->proxy); + + pool = conn->client ? conn->owner : ((struct server *)conn->owner)->owner; + + if (!pool->require_auth) { + return true; + } + + if (!conn->authenticated) { + return false; + } + + return true; +} diff -Nru nutcracker-0.4.0+dfsg/src/nc_connection.h nutcracker-0.4.1+dfsg/src/nc_connection.h --- nutcracker-0.4.0+dfsg/src/nc_connection.h 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_connection.h 2015-06-23 06:06:05.000000000 +0000 @@ -35,55 +35,60 @@ typedef void (*conn_unref_t)(struct conn *); typedef void (*conn_msgq_t)(struct context *, struct conn *, struct msg *); +typedef void (*conn_post_connect_t)(struct context *ctx, struct conn *, struct server *server); +typedef void (*conn_swallow_msg_t)(struct conn *, struct msg *, struct msg *); struct conn { - TAILQ_ENTRY(conn) conn_tqe; /* link in server_pool / server / free q */ - void *owner; /* connection owner - server_pool / server */ + TAILQ_ENTRY(conn) conn_tqe; /* link in server_pool / server / free q */ + void *owner; /* connection owner - server_pool / server */ - int sd; /* socket descriptor */ - int family; /* socket address family */ - socklen_t addrlen; /* socket length */ - struct sockaddr *addr; /* socket address (ref in server or server_pool) */ - - struct msg_tqh imsg_q; /* incoming request Q */ - struct msg_tqh omsg_q; /* outstanding request Q */ - struct msg *rmsg; /* current message being rcvd */ - struct msg *smsg; /* current message being sent */ - - conn_recv_t recv; /* recv (read) handler */ - conn_recv_next_t recv_next; /* recv next message handler */ - conn_recv_done_t recv_done; /* read done handler */ - conn_send_t send; /* send (write) handler */ - conn_send_next_t send_next; /* write next message handler */ - conn_send_done_t send_done; /* write done handler */ - conn_close_t close; /* close handler */ - conn_active_t active; /* active? handler */ - - conn_ref_t ref; /* connection reference handler */ - conn_unref_t unref; /* connection unreference handler */ - - conn_msgq_t enqueue_inq; /* connection inq msg enqueue handler */ - conn_msgq_t dequeue_inq; /* connection inq msg dequeue handler */ - conn_msgq_t enqueue_outq; /* connection outq msg enqueue handler */ - conn_msgq_t dequeue_outq; /* connection outq msg dequeue handler */ - - size_t recv_bytes; /* received (read) bytes */ - size_t send_bytes; /* sent (written) bytes */ - - uint32_t events; /* connection io events */ - err_t err; /* connection errno */ - unsigned recv_active:1; /* recv active? */ - unsigned recv_ready:1; /* recv ready? */ - unsigned send_active:1; /* send active? */ - unsigned send_ready:1; /* send ready? */ - - unsigned client:1; /* client? or server? */ - unsigned proxy:1; /* proxy? */ - unsigned connecting:1; /* connecting? */ - unsigned connected:1; /* connected? */ - unsigned eof:1; /* eof? aka passive close? */ - unsigned done:1; /* done? aka close? */ - unsigned redis:1; /* redis? */ + int sd; /* socket descriptor */ + int family; /* socket address family */ + socklen_t addrlen; /* socket length */ + struct sockaddr *addr; /* socket address (ref in server or server_pool) */ + + struct msg_tqh imsg_q; /* incoming request Q */ + struct msg_tqh omsg_q; /* outstanding request Q */ + struct msg *rmsg; /* current message being rcvd */ + struct msg *smsg; /* current message being sent */ + + conn_recv_t recv; /* recv (read) handler */ + conn_recv_next_t recv_next; /* recv next message handler */ + conn_recv_done_t recv_done; /* read done handler */ + conn_send_t send; /* send (write) handler */ + conn_send_next_t send_next; /* write next message handler */ + conn_send_done_t send_done; /* write done handler */ + conn_close_t close; /* close handler */ + conn_active_t active; /* active? handler */ + conn_post_connect_t post_connect; /* post connect handler */ + conn_swallow_msg_t swallow_msg; /* react on messages to be swallowed */ + + conn_ref_t ref; /* connection reference handler */ + conn_unref_t unref; /* connection unreference handler */ + + conn_msgq_t enqueue_inq; /* connection inq msg enqueue handler */ + conn_msgq_t dequeue_inq; /* connection inq msg dequeue handler */ + conn_msgq_t enqueue_outq; /* connection outq msg enqueue handler */ + conn_msgq_t dequeue_outq; /* connection outq msg dequeue handler */ + + size_t recv_bytes; /* received (read) bytes */ + size_t send_bytes; /* sent (written) bytes */ + + uint32_t events; /* connection io events */ + err_t err; /* connection errno */ + unsigned recv_active:1; /* recv active? */ + unsigned recv_ready:1; /* recv ready? */ + unsigned send_active:1; /* send active? */ + unsigned send_ready:1; /* send ready? */ + + unsigned client:1; /* client? or server? */ + unsigned proxy:1; /* proxy? */ + unsigned connecting:1; /* connecting? */ + unsigned connected:1; /* connected? */ + unsigned eof:1; /* eof? aka passive close? */ + unsigned done:1; /* done? aka close? */ + unsigned redis:1; /* redis? */ + unsigned authenticated:1; /* authenticated? */ }; TAILQ_HEAD(conn_tqh, conn); @@ -99,5 +104,6 @@ uint32_t conn_ncurr_conn(void); uint64_t conn_ntotal_conn(void); uint32_t conn_ncurr_cconn(void); +bool conn_authenticated(struct conn *conn); #endif diff -Nru nutcracker-0.4.0+dfsg/src/nc_core.c nutcracker-0.4.1+dfsg/src/nc_core.c --- nutcracker-0.4.0+dfsg/src/nc_core.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_core.c 2015-06-23 06:06:05.000000000 +0000 @@ -311,7 +311,14 @@ { rstatus_t status; struct conn *conn = arg; - struct context *ctx = conn_to_ctx(conn); + struct context *ctx; + + if (conn->owner == NULL) { + log_warn("conn is already unrefed!"); + return NC_OK; + } + + ctx = conn_to_ctx(conn); log_debug(LOG_VVERB, "event %04"PRIX32" on %c %d", events, conn->client ? 'c' : (conn->proxy ? 'p' : 's'), conn->sd); diff -Nru nutcracker-0.4.0+dfsg/src/nc_message.c nutcracker-0.4.1+dfsg/src/nc_message.c --- nutcracker-0.4.0+dfsg/src/nc_message.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_message.c 2015-06-23 06:06:05.000000000 +0000 @@ -226,6 +226,7 @@ msg->token = NULL; msg->parser = NULL; + msg->add_auth = NULL; msg->result = MSG_PARSE_OK; msg->fragment = NULL; @@ -292,8 +293,10 @@ } else { msg->parser = redis_parse_rsp; } + msg->add_auth = redis_add_auth; msg->fragment = redis_fragment; msg->reply = redis_reply; + msg->failure = redis_failure; msg->pre_coalesce = redis_pre_coalesce; msg->post_coalesce = redis_post_coalesce; } else { @@ -302,7 +305,9 @@ } else { msg->parser = memcache_parse_rsp; } + msg->add_auth = memcache_add_auth; msg->fragment = memcache_fragment; + msg->failure = memcache_failure; msg->pre_coalesce = memcache_pre_coalesce; msg->post_coalesce = memcache_post_coalesce; } @@ -472,11 +477,13 @@ } else { mbuf = STAILQ_LAST(&msg->mhdr, mbuf, next); } + return mbuf; } /* - * append small(small than a mbuf) content into msg + * Append n bytes of data, with n <= mbuf_size(mbuf) + * into mbuf */ rstatus_t msg_append(struct msg *msg, uint8_t *pos, size_t n) @@ -494,11 +501,13 @@ mbuf_copy(mbuf, pos, n); msg->mlen += (uint32_t)n; + return NC_OK; } /* - * prepend small(small than a mbuf) content into msg + * Prepend n bytes of data, with n <= mbuf_size(mbuf) + * into mbuf */ rstatus_t msg_prepend(struct msg *msg, uint8_t *pos, size_t n) @@ -516,17 +525,20 @@ msg->mlen += (uint32_t)n; STAILQ_INSERT_HEAD(&msg->mhdr, mbuf, next); + return NC_OK; } /* - * prepend small(small than a mbuf) content into msg + * Prepend a formatted string into msg. Returns an error if the formatted + * string does not fit in a single mbuf. */ rstatus_t msg_prepend_format(struct msg *msg, const char *fmt, ...) { struct mbuf *mbuf; - int32_t n; + int n; + uint32_t size; va_list args; mbuf = mbuf_get(); @@ -534,15 +546,19 @@ return NC_ENOMEM; } + size = mbuf_size(mbuf); + va_start(args, fmt); - n = nc_vscnprintf(mbuf->last, mbuf_size(mbuf), fmt, args); + n = nc_vsnprintf(mbuf->last, size, fmt, args); va_end(args); + if (n <= 0 || n >= (int)size) { + return NC_ERROR; + } mbuf->last += n; msg->mlen += (uint32_t)n; - - ASSERT(mbuf_size(mbuf) >= 0); STAILQ_INSERT_HEAD(&msg->mhdr, mbuf, next); + return NC_OK; } diff -Nru nutcracker-0.4.0+dfsg/src/nc_message.h nutcracker-0.4.1+dfsg/src/nc_message.h --- nutcracker-0.4.0+dfsg/src/nc_message.h 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_message.h 2015-06-23 06:06:05.000000000 +0000 @@ -21,9 +21,11 @@ #include typedef void (*msg_parse_t)(struct msg *); +typedef rstatus_t (*msg_add_auth_t)(struct context *ctx, struct conn *c_conn, struct conn *s_conn); typedef rstatus_t (*msg_fragment_t)(struct msg *, uint32_t, struct msg_tqh *); typedef void (*msg_coalesce_t)(struct msg *r); typedef rstatus_t (*msg_reply_t)(struct msg *r); +typedef bool (*msg_failure_t)(struct msg *r); typedef enum msg_parse_result { MSG_PARSE_OK, /* parsing ok */ @@ -45,6 +47,7 @@ ACTION( REQ_MC_PREPEND ) \ ACTION( REQ_MC_INCR ) /* memcache arithmetic request */ \ ACTION( REQ_MC_DECR ) \ + ACTION( REQ_MC_TOUCH ) /* memcache touch request */ \ ACTION( REQ_MC_QUIT ) /* memcache quit request */ \ ACTION( RSP_MC_NUM ) /* memcache arithmetic response */ \ ACTION( RSP_MC_STORED ) /* memcache cas and storage response */ \ @@ -54,6 +57,7 @@ ACTION( RSP_MC_END ) \ ACTION( RSP_MC_VALUE ) \ ACTION( RSP_MC_DELETED ) /* memcache delete response */ \ + ACTION( RSP_MC_TOUCHED ) /* memcache touch response */ \ ACTION( RSP_MC_ERROR ) /* memcache error responses */ \ ACTION( RSP_MC_CLIENT_ERROR ) \ ACTION( RSP_MC_SERVER_ERROR ) \ @@ -160,8 +164,23 @@ ACTION( REQ_REDIS_EVALSHA ) \ ACTION( REQ_REDIS_PING ) /* redis requests - ping/quit */ \ ACTION( REQ_REDIS_QUIT) \ + ACTION( REQ_REDIS_AUTH) \ + ACTION( REQ_REDIS_SELECT) /* only during init */ \ ACTION( RSP_REDIS_STATUS ) /* redis response */ \ ACTION( RSP_REDIS_ERROR ) \ + ACTION( RSP_REDIS_ERROR_ERR ) \ + ACTION( RSP_REDIS_ERROR_OOM ) \ + ACTION( RSP_REDIS_ERROR_BUSY ) \ + ACTION( RSP_REDIS_ERROR_NOAUTH ) \ + ACTION( RSP_REDIS_ERROR_LOADING ) \ + ACTION( RSP_REDIS_ERROR_BUSYKEY ) \ + ACTION( RSP_REDIS_ERROR_MISCONF ) \ + ACTION( RSP_REDIS_ERROR_NOSCRIPT ) \ + ACTION( RSP_REDIS_ERROR_READONLY ) \ + ACTION( RSP_REDIS_ERROR_WRONGTYPE ) \ + ACTION( RSP_REDIS_ERROR_EXECABORT ) \ + ACTION( RSP_REDIS_ERROR_MASTERDOWN ) \ + ACTION( RSP_REDIS_ERROR_NOREPLICAS ) \ ACTION( RSP_REDIS_INTEGER ) \ ACTION( RSP_REDIS_BULK ) \ ACTION( RSP_REDIS_MULTIBULK ) \ @@ -202,7 +221,10 @@ msg_parse_result_t result; /* message parsing result */ msg_fragment_t fragment; /* message fragment */ - msg_reply_t reply; /* gen message reply (example: ping) */ + msg_reply_t reply; /* generate message reply (example: ping) */ + msg_add_auth_t add_auth; /* add auth message when we forward msg */ + msg_failure_t failure; /* transient failure response? */ + msg_coalesce_t pre_coalesce; /* message pre-coalesce */ msg_coalesce_t post_coalesce; /* message post-coalesce */ @@ -267,6 +289,7 @@ bool req_done(struct conn *conn, struct msg *msg); bool req_error(struct conn *conn, struct msg *msg); void req_server_enqueue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg); +void req_server_enqueue_imsgq_head(struct context *ctx, struct conn *conn, struct msg *msg); void req_server_dequeue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg); void req_client_enqueue_omsgq(struct context *ctx, struct conn *conn, struct msg *msg); void req_server_enqueue_omsgq(struct context *ctx, struct conn *conn, struct msg *msg); diff -Nru nutcracker-0.4.0+dfsg/src/nc_proxy.c nutcracker-0.4.1+dfsg/src/nc_proxy.c --- nutcracker-0.4.0+dfsg/src/nc_proxy.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_proxy.c 2015-06-23 06:06:05.000000000 +0000 @@ -15,6 +15,7 @@ * limitations under the License. */ +#include #include #include @@ -29,9 +30,9 @@ ASSERT(!conn->client && conn->proxy); ASSERT(conn->owner == NULL); - conn->family = pool->family; - conn->addrlen = pool->addrlen; - conn->addr = pool->addr; + conn->family = pool->info.family; + conn->addrlen = pool->info.addrlen; + conn->addr = (struct sockaddr *)&pool->info.addr; pool->p_conn = conn; @@ -148,6 +149,16 @@ return NC_ERROR; } + if (p->family == AF_UNIX && pool->perm) { + struct sockaddr_un *un = (struct sockaddr_un *)p->addr; + status = chmod(un->sun_path, pool->perm); + if (status < 0) { + log_error("chmod on p %d on addr '%.*s' failed: %s", p->sd, + pool->addrstr.len, pool->addrstr.data, strerror(errno)); + return NC_ERROR; + } + } + status = listen(p->sd, pool->backlog); if (status < 0) { log_error("listen on p %d on addr '%.*s' failed: %s", p->sd, diff -Nru nutcracker-0.4.0+dfsg/src/nc_rbtree.c nutcracker-0.4.1+dfsg/src/nc_rbtree.c --- nutcracker-0.4.0+dfsg/src/nc_rbtree.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_rbtree.c 2015-06-23 06:06:05.000000000 +0000 @@ -218,11 +218,7 @@ subst = node; } else { subst = rbtree_node_min(node->right, sentinel); - if (subst->left != sentinel) { - temp = subst->left; - } else { - temp = subst->right; - } + temp = subst->right; } if (subst == *root) { diff -Nru nutcracker-0.4.0+dfsg/src/nc_request.c nutcracker-0.4.1+dfsg/src/nc_request.c --- nutcracker-0.4.0+dfsg/src/nc_request.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_request.c 2015-06-23 06:06:05.000000000 +0000 @@ -303,7 +303,7 @@ * or the message is dequeued from the server out_q * * noreply request are free from timeouts because client is not intrested - * in the reponse anyway! + * in the response anyway! */ if (!msg->noreply) { msg_tmo_insert(msg, conn); @@ -316,6 +316,30 @@ } void +req_server_enqueue_imsgq_head(struct context *ctx, struct conn *conn, struct msg *msg) +{ + ASSERT(msg->request); + ASSERT(!conn->client && !conn->proxy); + + /* + * timeout clock starts ticking the instant the message is enqueued into + * the server in_q; the clock continues to tick until it either expires + * or the message is dequeued from the server out_q + * + * noreply request are free from timeouts because client is not intrested + * in the reponse anyway! + */ + if (!msg->noreply) { + msg_tmo_insert(msg, conn); + } + + TAILQ_INSERT_HEAD(&conn->imsg_q, msg, s_tqe); + + stats_server_incr(ctx, conn->owner, in_queue); + stats_server_incr_by(ctx, conn->owner, in_queue_bytes, msg->mlen); +} + +void req_server_dequeue_imsgq(struct context *ctx, struct conn *conn, struct msg *msg) { ASSERT(msg->request); @@ -429,20 +453,21 @@ static rstatus_t req_make_reply(struct context *ctx, struct conn *conn, struct msg *req) { - struct msg *msg; + struct msg *rsp; - msg = msg_get(conn, true, conn->redis); /* replay */ - if (msg == NULL) { + rsp = msg_get(conn, false, conn->redis); /* replay */ + if (rsp == NULL) { conn->err = errno; return NC_ENOMEM; } - req->peer = msg; - msg->peer = req; - msg->request = 0; + req->peer = rsp; + rsp->peer = req; + rsp->request = 0; req->done = 1; conn->enqueue_outq(ctx, conn, req); + return NC_OK; } @@ -460,19 +485,32 @@ } /* - * Handle "quit\r\n", which is the protocol way of doing a - * passive close + * Handle "quit\r\n" (memcache) or "*1\r\n$4\r\nquit\r\n" (redis), which + * is the protocol way of doing a passive close. The connection is closed + * as soon as all pending replies have been written to the client. */ if (msg->quit) { - ASSERT(conn->rmsg == NULL); log_debug(LOG_INFO, "filter quit req %"PRIu64" from c %d", msg->id, conn->sd); + if (conn->rmsg != NULL) { + log_debug(LOG_INFO, "discard invalid req %"PRIu64" len %"PRIu32" " + "from c %d sent after quit req", conn->rmsg->id, + conn->rmsg->mlen, conn->sd); + } conn->eof = 1; conn->recv_ready = 0; req_put(msg); return true; } + /* + * If this conn is not authenticated, we will mark it as noforward, + * and handle it in the redis_reply handler. + */ + if (!conn_authenticated(conn)) { + msg->noforward = 1; + } + return false; } @@ -554,6 +592,16 @@ return; } } + + if (!conn_authenticated(s_conn)) { + status = msg->add_auth(ctx, c_conn, s_conn); + if (status != NC_OK) { + req_forward_error(ctx, c_conn, msg); + s_conn->err = errno; + return; + } + } + s_conn->enqueue_inq(ctx, s_conn, msg); req_forward_stats(ctx, s_conn->owner, msg); diff -Nru nutcracker-0.4.0+dfsg/src/nc_response.c nutcracker-0.4.1+dfsg/src/nc_response.c --- nutcracker-0.4.0+dfsg/src/nc_response.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_response.c 2015-06-23 06:06:05.000000000 +0000 @@ -171,7 +171,7 @@ * We handle this stray packet scenario in nutcracker by closing the * server connection which would end up sending SERVER_ERROR to all * clients that have requests pending on this server connection. The - * fix is aggresive, but not doing so would lead to clients getting + * fix is aggressive, but not doing so would lead to clients getting * out of sync with the server and as a result clients end up getting * responses that don't correspond to the right request. * @@ -184,7 +184,28 @@ ASSERT(pmsg->peer == NULL); ASSERT(pmsg->request && !pmsg->done); + /* + * If the response from a server suggests a protocol level transient + * failure, close the server connection and send back a generic error + * response to the client. + * + * If auto_eject_host is enabled, this will also update the failure_count + * and eject the server if it exceeds the failure_limit + */ + if (msg->failure(msg)) { + log_debug(LOG_INFO, "server failure rsp %"PRIu64" len %"PRIu32" " + "type %d on s %d", msg->id, msg->mlen, msg->type, conn->sd); + rsp_put(msg); + + conn->err = EINVAL; + conn->done = 1; + + return true; + } + if (pmsg->swallow) { + conn->swallow_msg(conn, pmsg, msg); + conn->dequeue_outq(ctx, conn, pmsg); pmsg->done = 1; @@ -201,12 +222,12 @@ } static void -rsp_forward_stats(struct context *ctx, struct server *server, struct msg *msg) +rsp_forward_stats(struct context *ctx, struct server *server, struct msg *msg, uint32_t msgsize) { ASSERT(!msg->request); stats_server_incr(ctx, server, responses); - stats_server_incr_by(ctx, server, response_bytes, msg->mlen); + stats_server_incr_by(ctx, server, response_bytes, msgsize); } static void @@ -215,8 +236,10 @@ rstatus_t status; struct msg *pmsg; struct conn *c_conn; + uint32_t msgsize; ASSERT(!s_conn->client && !s_conn->proxy); + msgsize = msg->mlen; /* response from server implies that server is ok and heartbeating */ server_ok(ctx, s_conn); @@ -245,7 +268,7 @@ } } - rsp_forward_stats(ctx, s_conn->owner, msg); + rsp_forward_stats(ctx, s_conn->owner, msg, msgsize); } void diff -Nru nutcracker-0.4.0+dfsg/src/nc_server.c nutcracker-0.4.1+dfsg/src/nc_server.c --- nutcracker-0.4.0+dfsg/src/nc_server.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_server.c 2015-06-23 06:06:05.000000000 +0000 @@ -22,6 +22,23 @@ #include #include +static void +server_resolve(struct server *server, struct conn *conn) +{ + rstatus_t status; + + status = nc_resolve(&server->addrstr, server->port, &server->info); + if (status != NC_OK) { + conn->err = EHOSTDOWN; + conn->done = 1; + return; + } + + conn->family = server->info.family; + conn->addrlen = server->info.addrlen; + conn->addr = (struct sockaddr *)&server->info.addr; +} + void server_ref(struct conn *conn, void *owner) { @@ -30,9 +47,7 @@ ASSERT(!conn->client && !conn->proxy); ASSERT(conn->owner == NULL); - conn->family = server->family; - conn->addrlen = server->addrlen; - conn->addr = server->addr; + server_resolve(server, conn); server->ns_conn_q++; TAILQ_INSERT_TAIL(&server->s_conn_q, conn, conn_tqe); @@ -338,6 +353,8 @@ server_close_stats(ctx, conn->owner, conn->err, conn->eof, conn->connected); + conn->connected = false; + if (conn->sd < 0) { server_failure(ctx, conn->owner); conn->unref(conn); @@ -452,6 +469,12 @@ ASSERT(!conn->client && !conn->proxy); + if (conn->err) { + ASSERT(conn->done && conn->sd < 0); + errno = conn->err; + return NC_ERROR; + } + if (conn->sd > 0) { /* already connected on server connection */ return NC_OK; @@ -535,6 +558,8 @@ conn->connecting = 0; conn->connected = 1; + conn->post_connect(ctx, conn, server); + log_debug(LOG_INFO, "connected on s %d to server '%.*s'", conn->sd, server->pname.len, server->pname.data); } @@ -605,12 +630,15 @@ server_pool_hash(struct server_pool *pool, uint8_t *key, uint32_t keylen) { ASSERT(array_n(&pool->server) != 0); + ASSERT(key != NULL); if (array_n(&pool->server) == 1) { return 0; } - ASSERT(key != NULL && keylen != 0); + if (keylen == 0) { + return 0; + } return pool->key_hash((char *)key, keylen); } @@ -621,7 +649,7 @@ uint32_t hash, idx; ASSERT(array_n(&pool->server) != 0); - ASSERT(key != NULL && keylen != 0); + ASSERT(key != NULL); /* * If hash_tag: is configured for this server pool, we use the part of diff -Nru nutcracker-0.4.0+dfsg/src/nc_server.h nutcracker-0.4.1+dfsg/src/nc_server.h --- nutcracker-0.4.0+dfsg/src/nc_server.h 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_server.h 2015-06-23 06:06:05.000000000 +0000 @@ -70,13 +70,12 @@ uint32_t idx; /* server index */ struct server_pool *owner; /* owner pool */ - struct string pname; /* name:port:weight (ref in conf_server) */ - struct string name; /* name (ref in conf_server) */ + struct string pname; /* hostname:port:weight (ref in conf_server) */ + struct string name; /* hostname:port or [name] (ref in conf_server) */ + struct string addrstr; /* hostname (ref in conf_server) */ uint16_t port; /* port */ uint32_t weight; /* weight */ - int family; /* socket family */ - socklen_t addrlen; /* socket length */ - struct sockaddr *addr; /* socket address (ref in conf_server) */ + struct sockinfo info; /* server socket info */ uint32_t ns_conn_q; /* # server connection */ struct conn_tqh s_conn_q; /* server connection q */ @@ -101,21 +100,23 @@ int64_t next_rebuild; /* next distribution rebuild time in usec */ struct string name; /* pool name (ref in conf_pool) */ - struct string addrstr; /* pool address (ref in conf_pool) */ + struct string addrstr; /* pool address - hostname:port (ref in conf_pool) */ uint16_t port; /* port */ - int family; /* socket family */ - socklen_t addrlen; /* socket length */ - struct sockaddr *addr; /* socket address (ref in conf_pool) */ + struct sockinfo info; /* listen socket info */ + mode_t perm; /* socket permission */ int dist_type; /* distribution type (dist_type_t) */ int key_hash_type; /* key hash type (hash_type_t) */ hash_t key_hash; /* key hasher */ struct string hash_tag; /* key hash tag (ref in conf_pool) */ int timeout; /* timeout in msec */ int backlog; /* listen backlog */ + int redis_db; /* redis database to connect to */ uint32_t client_connections; /* maximum # client connection */ uint32_t server_connections; /* maximum # server connection */ int64_t server_retry_timeout; /* server retry timeout in usec */ uint32_t server_failure_limit; /* server failure limit */ + struct string redis_auth; /* redis_auth password (matches requirepass on redis) */ + unsigned require_auth; /* require_auth? */ unsigned auto_eject_hosts:1; /* auto_eject_hosts? */ unsigned preconnect:1; /* preconnect? */ unsigned redis:1; /* redis? */ diff -Nru nutcracker-0.4.0+dfsg/src/nc_string.h nutcracker-0.4.1+dfsg/src/nc_string.h --- nutcracker-0.4.0+dfsg/src/nc_string.h 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_string.h 2015-06-23 06:06:05.000000000 +0000 @@ -77,12 +77,29 @@ #define nc_strndup(_s, _n) \ (uint8_t *)strndup((char *)(_s), (size_t)(_n)); +/* + * snprintf(s, n, ...) will write at most n - 1 of the characters printed into + * the output string; the nth character then gets the terminating `\0'; if + * the return value is greater than or equal to the n argument, the string + * was too short and some of the printed characters were discarded; the output + * is always null-terminated. + * + * Note that, the return value of snprintf() is always the number of characters + * that would be printed into the output string, assuming n were limited not + * including the trailing `\0' used to end output. + * + * scnprintf(s, n, ...) is same as snprintf() except, it returns the number + * of characters printed into the output string not including the trailing '\0' + */ #define nc_snprintf(_s, _n, ...) \ snprintf((char *)(_s), (size_t)(_n), __VA_ARGS__) #define nc_scnprintf(_s, _n, ...) \ _scnprintf((char *)(_s), (size_t)(_n), __VA_ARGS__) +#define nc_vsnprintf(_s, _n, _f, _a) \ + vsnprintf((char *)(_s), (size_t)(_n), _f, _a) + #define nc_vscnprintf(_s, _n, _f, _a) \ _vscnprintf((char *)(_s), (size_t)(_n), _f, _a) diff -Nru nutcracker-0.4.0+dfsg/src/nc_util.c nutcracker-0.4.1+dfsg/src/nc_util.c --- nutcracker-0.4.0+dfsg/src/nc_util.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/nc_util.c 2015-06-23 06:06:05.000000000 +0000 @@ -489,8 +489,12 @@ nc_snprintf(service, NC_UINTMAX_MAXLEN, "%d", port); + /* + * getaddrinfo() returns zero on success or one of the error codes listed + * in gai_strerror(3) if an error occurs + */ status = getaddrinfo(node, service, &hints, &ai); - if (status < 0) { + if (status != 0) { log_error("address resolution of node '%s' service '%s' failed: %s", node, service, gai_strerror(status)); return -1; diff -Nru nutcracker-0.4.0+dfsg/src/proto/nc_memcache.c nutcracker-0.4.1+dfsg/src/proto/nc_memcache.c --- nutcracker-0.4.0+dfsg/src/proto/nc_memcache.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/proto/nc_memcache.c 2015-06-23 06:06:05.000000000 +0000 @@ -121,6 +121,20 @@ return false; } +/* + * Return true, if the memcache command is a touch command, otherwise + * return false + */ +static bool +memcache_touch(struct msg *r) +{ + if (r->type == MSG_REQ_MC_TOUCH) { + return true; + } + + return false; +} + void memcache_parse_req(struct msg *r) { @@ -243,6 +257,14 @@ break; + case 5: + if (str5cmp(m, 't', 'o', 'u', 'c', 'h')) { + r->type = MSG_REQ_MC_TOUCH; + break; + } + + break; + case 6: if (str6cmp(m, 'a', 'p', 'p', 'e', 'n', 'd')) { r->type = MSG_REQ_MC_APPEND; @@ -282,6 +304,7 @@ case MSG_REQ_MC_PREPEND: case MSG_REQ_MC_INCR: case MSG_REQ_MC_DECR: + case MSG_REQ_MC_TOUCH: if (ch == CR) { goto error; } @@ -320,13 +343,17 @@ } if (ch == ' ' || ch == CR) { struct keypos *kpos; - - if ((p - r->token) > MEMCACHE_MAX_KEY_LENGTH) { + int keylen = p - r->token; + if (keylen > MEMCACHE_MAX_KEY_LENGTH) { log_error("parsed bad req %"PRIu64" of type %d with key " "prefix '%.*s...' and length %d that exceeds " "maximum key length", r->id, r->type, 16, r->token, p - r->token); goto error; + } else if (keylen == 0) { + log_error("parsed bad req %"PRIu64" of type %d with an " + "empty key", r->id, r->type); + goto error; } kpos = array_push(r->keys); @@ -342,7 +369,7 @@ /* get next state */ if (memcache_storage(r)) { state = SW_SPACES_BEFORE_FLAGS; - } else if (memcache_arithmetic(r)) { + } else if (memcache_arithmetic(r) || memcache_touch(r) ) { state = SW_SPACES_BEFORE_NUM; } else if (memcache_delete(r)) { state = SW_RUNTO_CRLF; @@ -531,7 +558,7 @@ case SW_SPACES_BEFORE_NUM: if (ch != ' ') { - if (!isdigit(ch)) { + if (!(isdigit(ch) || ch == '-')) { goto error; } /* num_start <- p; num <- ch - '0' */ @@ -562,7 +589,7 @@ break; case 'n': - if (memcache_storage(r) || memcache_arithmetic(r) || memcache_delete(r)) { + if (memcache_storage(r) || memcache_arithmetic(r) || memcache_delete(r) || memcache_touch(r)) { /* noreply_start <- p */ r->token = p; state = SW_NOREPLY; @@ -593,7 +620,7 @@ case CR: m = r->token; if (((p - m) == 7) && str7cmp(m, 'n', 'o', 'r', 'e', 'p', 'l', 'y')) { - ASSERT(memcache_storage(r) || memcache_arithmetic(r) || memcache_delete(r)); + ASSERT(memcache_storage(r) || memcache_arithmetic(r) || memcache_delete(r) || memcache_touch(r)); r->token = NULL; /* noreply_end <- p - 1 */ r->noreply = 1; @@ -854,6 +881,11 @@ break; } + if (str7cmp(m, 'T', 'O', 'U', 'C', 'H', 'E', 'D')) { + r->type = MSG_RSP_MC_TOUCHED; + break; + } + break; case 9: @@ -895,6 +927,7 @@ case MSG_RSP_MC_EXISTS: case MSG_RSP_MC_NOT_FOUND: case MSG_RSP_MC_DELETED: + case MSG_RSP_MC_TOUCHED: state = SW_CRLF; break; @@ -1167,6 +1200,12 @@ r->state); } +bool +memcache_failure(struct msg *r) +{ + return false; +} + static rstatus_t memcache_append_key(struct msg *r, uint8_t *key, uint32_t keylen) { @@ -1377,9 +1416,8 @@ } /* - * copy one response from src to dst - * return bytes copied - * */ + * Copy one response from src to dst and return bytes copied + */ static rstatus_t memcache_copy_bulk(struct msg *dst, struct msg *src) { @@ -1403,9 +1441,10 @@ } p = mbuf->pos; - /* get : VALUE key 0 len\r\nval\r\n */ - /* gets: VALUE key 0 len cas\r\nval\r\n */ - + /* + * get : VALUE key 0 len\r\nval\r\n + * gets: VALUE key 0 len cas\r\nval\r\n + */ ASSERT(*p == 'V'); for (i = 0; i < 3; i++) { /* eat 'VALUE key 0 ' */ for (; *p != ' ';) { @@ -1495,3 +1534,28 @@ return; } } + +void +memcache_post_connect(struct context *ctx, struct conn *conn, struct server *server) +{ +} + +void +memcache_swallow_msg(struct conn *conn, struct msg *pmsg, struct msg *msg) +{ +} + +rstatus_t +memcache_add_auth(struct context *ctx, struct conn *c_conn, struct conn *s_conn) +{ + NOT_REACHED(); + return NC_OK; +} + +rstatus_t +memcache_reply(struct msg *r) +{ + NOT_REACHED(); + return NC_OK; +} + diff -Nru nutcracker-0.4.0+dfsg/src/proto/nc_proto.h nutcracker-0.4.1+dfsg/src/proto/nc_proto.h --- nutcracker-0.4.0+dfsg/src/proto/nc_proto.h 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/proto/nc_proto.h 2015-06-23 06:06:05.000000000 +0000 @@ -140,15 +140,24 @@ void memcache_parse_req(struct msg *r); void memcache_parse_rsp(struct msg *r); +bool memcache_failure(struct msg *r); void memcache_pre_coalesce(struct msg *r); void memcache_post_coalesce(struct msg *r); +rstatus_t memcache_add_auth(struct context *ctx, struct conn *c_conn, struct conn *s_conn); rstatus_t memcache_fragment(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq); +rstatus_t memcache_reply(struct msg *r); +void memcache_post_connect(struct context *ctx, struct conn *conn, struct server *server); +void memcache_swallow_msg(struct conn *conn, struct msg *pmsg, struct msg *msg); void redis_parse_req(struct msg *r); void redis_parse_rsp(struct msg *r); +bool redis_failure(struct msg *r); void redis_pre_coalesce(struct msg *r); void redis_post_coalesce(struct msg *r); +rstatus_t redis_add_auth(struct context *ctx, struct conn *c_conn, struct conn *s_conn); rstatus_t redis_fragment(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq); rstatus_t redis_reply(struct msg *r); +void redis_post_connect(struct context *ctx, struct conn *conn, struct server *server); +void redis_swallow_msg(struct conn *conn, struct msg *pmsg, struct msg *msg); #endif diff -Nru nutcracker-0.4.0+dfsg/src/proto/nc_redis.c nutcracker-0.4.1+dfsg/src/proto/nc_redis.c --- nutcracker-0.4.0+dfsg/src/proto/nc_redis.c 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/src/proto/nc_redis.c 2015-06-23 06:06:05.000000000 +0000 @@ -17,10 +17,24 @@ #include #include +#include #include #include +#define RSP_STRING(ACTION) \ + ACTION( ok, "+OK\r\n" ) \ + ACTION( pong, "+PONG\r\n" ) \ + ACTION( invalid_password, "-ERR invalid password\r\n" ) \ + ACTION( auth_required, "-NOAUTH Authentication required\r\n" ) \ + ACTION( no_password, "-ERR Client sent AUTH, but no password is set\r\n" ) \ + +#define DEFINE_ACTION(_var, _str) static struct string rsp_##_var = string(_str); + RSP_STRING( DEFINE_ACTION ) +#undef DEFINE_ACTION + +static rstatus_t redis_handle_auth_req(struct msg *request, struct msg *response); + /* * Return true, if the redis command take no key, otherwise * return false @@ -51,7 +65,6 @@ case MSG_REQ_REDIS_EXISTS: case MSG_REQ_REDIS_PERSIST: case MSG_REQ_REDIS_PTTL: - case MSG_REQ_REDIS_SORT: case MSG_REQ_REDIS_TTL: case MSG_REQ_REDIS_TYPE: case MSG_REQ_REDIS_DUMP: @@ -75,8 +88,8 @@ case MSG_REQ_REDIS_SPOP: case MSG_REQ_REDIS_ZCARD: - case MSG_REQ_REDIS_PFCOUNT: + case MSG_REQ_REDIS_AUTH: return true; default: @@ -198,6 +211,8 @@ redis_argn(struct msg *r) { switch (r->type) { + case MSG_REQ_REDIS_SORT: + case MSG_REQ_REDIS_BITCOUNT: case MSG_REQ_REDIS_SET: @@ -301,6 +316,37 @@ } /* + * Return true, if the redis response is an error response i.e. a simple + * string whose first character is '-', otherwise return false. + */ +static bool +redis_error(struct msg *r) +{ + switch (r->type) { + case MSG_RSP_REDIS_ERROR: + case MSG_RSP_REDIS_ERROR_ERR: + case MSG_RSP_REDIS_ERROR_OOM: + case MSG_RSP_REDIS_ERROR_BUSY: + case MSG_RSP_REDIS_ERROR_NOAUTH: + case MSG_RSP_REDIS_ERROR_LOADING: + case MSG_RSP_REDIS_ERROR_BUSYKEY: + case MSG_RSP_REDIS_ERROR_MISCONF: + case MSG_RSP_REDIS_ERROR_NOSCRIPT: + case MSG_RSP_REDIS_ERROR_READONLY: + case MSG_RSP_REDIS_ERROR_WRONGTYPE: + case MSG_RSP_REDIS_ERROR_EXECABORT: + case MSG_RSP_REDIS_ERROR_MASTERDOWN: + case MSG_RSP_REDIS_ERROR_NOREPLICAS: + return true; + + default: + break; + } + + return false; +} + +/* * Reference: http://redis.io/topics/protocol * * Redis >= 1.2 uses the unified protocol to send requests to the Redis @@ -625,6 +671,12 @@ break; } + if (str4icmp(m, 'a', 'u', 't', 'h')) { + r->type = MSG_REQ_REDIS_AUTH; + r->noforward = 1; + break; + } + break; case 5: @@ -1051,6 +1103,8 @@ case LF: if (redis_argz(r)) { goto done; + } else if (r->narg == 1) { + goto error; } else if (redis_argeval(r)) { state = SW_ARG1_LEN; } else { @@ -1074,11 +1128,6 @@ } else if (isdigit(ch)) { r->rlen = r->rlen * 10 + (uint32_t)(ch - '0'); } else if (ch == CR) { - if (r->rlen == 0) { - log_error("parsed bad req %"PRIu64" of type %d with empty " - "key", r->id, r->type); - goto error; - } if (r->rlen >= mbuf_data_size()) { log_error("parsed bad req %"PRIu64" of type %d with key " "length %d that greater than or equal to maximum" @@ -1178,8 +1227,8 @@ } state = SW_KEY_LEN; } else if (redis_argkvx(r)) { - if (r->rnarg == 0) { - goto done; + if (r->narg % 2 == 0) { + goto error; } state = SW_ARG1_LEN; } else if (redis_argeval(r)) { @@ -1672,6 +1721,7 @@ SW_ERROR, SW_INTEGER, SW_INTEGER_START, + SW_SIMPLE, SW_BULK, SW_BULK_LF, SW_BULK_ARG, @@ -1748,8 +1798,125 @@ break; case SW_ERROR: - /* rsp_start <- p */ - state = SW_RUNTO_CRLF; + if (r->token == NULL) { + if (ch != '-') { + goto error; + } + /* rsp_start <- p */ + r->token = p; + } + if (ch == ' ' || ch == CR) { + m = r->token; + r->token = NULL; + switch (p - m) { + + case 4: + /* + * -ERR no such key\r\n + * -ERR syntax error\r\n + * -ERR source and destination objects are the same\r\n + * -ERR index out of range\r\n + */ + if (str4cmp(m, '-', 'E', 'R', 'R')) { + r->type = MSG_RSP_REDIS_ERROR_ERR; + break; + } + + /* -OOM command not allowed when used memory > 'maxmemory'.\r\n */ + if (str4cmp(m, '-', 'O', 'O', 'M')) { + r->type = MSG_RSP_REDIS_ERROR_OOM; + break; + } + + break; + + case 5: + /* -BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE.\r\n" */ + if (str5cmp(m, '-', 'B', 'U', 'S', 'Y')) { + r->type = MSG_RSP_REDIS_ERROR_BUSY; + break; + } + + break; + + case 7: + /* -NOAUTH Authentication required.\r\n */ + if (str7cmp(m, '-', 'N', 'O', 'A', 'U', 'T', 'H')) { + r->type = MSG_RSP_REDIS_ERROR_NOAUTH; + break; + } + + break; + + case 8: + /* rsp: "-LOADING Redis is loading the dataset in memory\r\n" */ + if (str8cmp(m, '-', 'L', 'O', 'A', 'D', 'I', 'N', 'G')) { + r->type = MSG_RSP_REDIS_ERROR_LOADING; + break; + } + + /* -BUSYKEY Target key name already exists.\r\n */ + if (str8cmp(m, '-', 'B', 'U', 'S', 'Y', 'K', 'E', 'Y')) { + r->type = MSG_RSP_REDIS_ERROR_BUSYKEY; + break; + } + + /* "-MISCONF Redis is configured to save RDB snapshots, but is currently not able to persist on disk. Commands that may modify the data set are disabled. Please check Redis logs for details about the error.\r\n" */ + if (str8cmp(m, '-', 'M', 'I', 'S', 'C', 'O', 'N', 'F')) { + r->type = MSG_RSP_REDIS_ERROR_MISCONF; + break; + } + + break; + + case 9: + /* -NOSCRIPT No matching script. Please use EVAL.\r\n */ + if (str9cmp(m, '-', 'N', 'O', 'S', 'C', 'R', 'I', 'P', 'T')) { + r->type = MSG_RSP_REDIS_ERROR_NOSCRIPT; + break; + } + + /* -READONLY You can't write against a read only slave.\r\n */ + if (str9cmp(m, '-', 'R', 'E', 'A', 'D', 'O', 'N', 'L', 'Y')) { + r->type = MSG_RSP_REDIS_ERROR_READONLY; + break; + } + + break; + + case 10: + /* -WRONGTYPE Operation against a key holding the wrong kind of value\r\n */ + if (str10cmp(m, '-', 'W', 'R', 'O', 'N', 'G', 'T', 'Y', 'P', 'E')) { + r->type = MSG_RSP_REDIS_ERROR_WRONGTYPE; + break; + } + + /* -EXECABORT Transaction discarded because of previous errors.\r\n" */ + if (str10cmp(m, '-', 'E', 'X', 'E', 'C', 'A', 'B', 'O', 'R', 'T')) { + r->type = MSG_RSP_REDIS_ERROR_EXECABORT; + break; + } + + break; + + case 11: + /* -MASTERDOWN Link with MASTER is down and slave-serve-stale-data is set to 'no'.\r\n */ + if (str11cmp(m, '-', 'M', 'A', 'S', 'T', 'E', 'R', 'D', 'O', 'W', 'N')) { + r->type = MSG_RSP_REDIS_ERROR_MASTERDOWN; + break; + } + + /* -NOREPLICAS Not enough good slaves to write.\r\n */ + if (str11cmp(m, '-', 'N', 'O', 'R', 'E', 'P', 'L', 'I', 'C', 'A', 'S')) { + r->type = MSG_RSP_REDIS_ERROR_NOREPLICAS; + break; + } + + break; + } + state = SW_RUNTO_CRLF; + } + break; case SW_INTEGER: @@ -1758,6 +1925,13 @@ r->integer = 0; break; + case SW_SIMPLE: + if (ch == CR) { + state = SW_MULTIBULK_ARGN_LF; + r->rnarg--; + } + break; + case SW_INTEGER_START: if (ch == CR) { state = SW_ALMOST_DONE; @@ -1897,6 +2071,7 @@ /* response is '*0\r\n' */ goto done; } + state = SW_MULTIBULK_ARGN_LEN; break; @@ -1938,9 +2113,17 @@ break; } - if (ch != '$' && ch != ':') { + if (ch == ':' || ch == '+' || ch == '-') { + /* handles not-found reply = '$-1' or integer reply = ':' */ + /* and *2\r\n$2\r\nr0\r\n+OK\r\n or *1\r\n+OK\r\n */ + state = SW_SIMPLE; + break; + } + + if (ch != '$') { goto error; } + r->token = p; r->rlen = 0; } else if (isdigit(ch)) { @@ -1952,8 +2135,7 @@ goto error; } - if ((r->rlen == 1 && (p - r->token) == 3) || *r->token == ':') { - /* handles not-found reply = '$-1' or integer reply = ':' */ + if ((r->rlen == 1 && (p - r->token) == 3)) { r->rlen = 0; state = SW_MULTIBULK_ARGN_LF; } else { @@ -2063,6 +2245,37 @@ } /* + * Return true, if redis replies with a transient server failure response, + * otherwise return false + * + * Transient failures on redis are scenarios when it is temporarily + * unresponsive and responds with the following protocol specific error + * reply: + * -OOM, when redis is out-of-memory + * -BUSY, when redis is busy + * -LOADING when redis is loading dataset into memory + * + * See issue: https://github.com/twitter/twemproxy/issues/369 + */ +bool +redis_failure(struct msg *r) +{ + ASSERT(!r->request); + + switch (r->type) { + case MSG_RSP_REDIS_ERROR_OOM: + case MSG_RSP_REDIS_ERROR_BUSY: + case MSG_RSP_REDIS_ERROR_LOADING: + return true; + + default: + break; + } + + return false; +} + +/* * copy one bulk from src to dst * * if dst == NULL, we just eat the bulk @@ -2443,8 +2656,10 @@ case MSG_REQ_REDIS_MGET: case MSG_REQ_REDIS_DEL: return redis_fragment_argx(r, ncontinuum, frag_msgq, 1); + case MSG_REQ_REDIS_MSET: return redis_fragment_argx(r, ncontinuum, frag_msgq, 2); + default: return NC_OK; } @@ -2453,13 +2668,23 @@ rstatus_t redis_reply(struct msg *r) { + struct conn *c_conn; struct msg *response = r->peer; - ASSERT(response != NULL); + ASSERT(response != NULL && response->owner != NULL); + + c_conn = response->owner; + if (r->type == MSG_REQ_REDIS_AUTH) { + return redis_handle_auth_req(r, response); + } + + if (!conn_authenticated(c_conn)) { + return msg_append(response, rsp_auth_required.data, rsp_auth_required.len); + } switch (r->type) { case MSG_REQ_REDIS_PING: - return msg_append(response, (uint8_t *)"+PONG\r\n", 7); + return msg_append(response, rsp_pong.data, rsp_pong.len); default: NOT_REACHED(); @@ -2470,10 +2695,10 @@ void redis_post_coalesce_mset(struct msg *request) { - struct msg *response = request->peer; rstatus_t status; + struct msg *response = request->peer; - status = msg_append(response, (uint8_t *)"+OK\r\n", 5); + status = msg_append(response, rsp_ok.data, rsp_ok.len); if (status != NC_OK) { response->error = 1; /* mark this msg as err */ response->err = errno; @@ -2546,11 +2771,168 @@ switch (r->type) { case MSG_REQ_REDIS_MGET: return redis_post_coalesce_mget(r); + case MSG_REQ_REDIS_DEL: return redis_post_coalesce_del(r); + case MSG_REQ_REDIS_MSET: return redis_post_coalesce_mset(r); + default: NOT_REACHED(); } } + +static rstatus_t +redis_handle_auth_req(struct msg *req, struct msg *rsp) +{ + struct conn *conn = (struct conn *)rsp->owner; + struct server_pool *pool; + struct keypos *kpos; + uint8_t *key; + uint32_t keylen; + bool valid; + + ASSERT(conn->client && !conn->proxy); + + pool = (struct server_pool *)conn->owner; + + if (!pool->require_auth) { + /* + * AUTH command from the client in absence of a redis_auth: + * directive should be treated as an error + */ + return msg_append(rsp, rsp_no_password.data, rsp_no_password.len); + } + + kpos = array_get(req->keys, 0); + key = kpos->start; + keylen = (uint32_t)(kpos->end - kpos->start); + valid = (keylen == pool->redis_auth.len) && + (memcmp(pool->redis_auth.data, key, keylen) == 0) ? true : false; + if (valid) { + conn->authenticated = 1; + return msg_append(rsp, rsp_ok.data, rsp_ok.len); + } + + /* + * Password in the AUTH command doesn't match the one configured in + * redis_auth: directive + * + * We mark the connection has unauthenticated until the client + * reauthenticates with the correct password + */ + conn->authenticated = 0; + return msg_append(rsp, rsp_invalid_password.data, rsp_invalid_password.len); +} + +rstatus_t +redis_add_auth(struct context *ctx, struct conn *c_conn, struct conn *s_conn) +{ + rstatus_t status; + struct msg *msg; + struct server_pool *pool; + + ASSERT(!s_conn->client && !s_conn->proxy); + ASSERT(!conn_authenticated(s_conn)); + + pool = c_conn->owner; + + msg = msg_get(c_conn, true, c_conn->redis); + if (msg == NULL) { + c_conn->err = errno; + return NC_ENOMEM; + } + + status = msg_prepend_format(msg, "*2\r\n$4\r\nAUTH\r\n$%d\r\n%s\r\n", + pool->redis_auth.len, pool->redis_auth.data); + if (status != NC_OK) { + msg_put(msg); + return status; + } + + msg->swallow = 1; + s_conn->enqueue_inq(ctx, s_conn, msg); + s_conn->authenticated = 1; + + return NC_OK; +} + +void +redis_post_connect(struct context *ctx, struct conn *conn, struct server *server) +{ + rstatus_t status; + struct server_pool *pool = server->owner; + struct msg *msg; + int digits; + + ASSERT(!conn->client && conn->connected); + ASSERT(conn->redis); + + /* + * By default, every connection to redis uses the database DB 0. You + * can select a different one on a per-connection basis by sending + * a request 'SELECT ', where is the configured + * on a per pool basis in the configuration + */ + if (pool->redis_db <= 0) { + return; + } + + /* + * Create a fake client message and add it to the pipeline. We force this + * message to be head of queue as it might already contain a command + * that triggered the connect. + */ + msg = msg_get(conn, true, conn->redis); + if (msg == NULL) { + return; + } + + digits = (pool->redis_db >= 10) ? (int)log10(pool->redis_db) + 1 : 1; + status = msg_prepend_format(msg, "*2\r\n$6\r\nSELECT\r\n$%d\r\n%d\r\n", digits, pool->redis_db); + if (status != NC_OK) { + msg_put(msg); + return; + } + msg->type = MSG_REQ_REDIS_SELECT; + msg->result = MSG_PARSE_OK; + msg->swallow = 1; + msg->owner = NULL; + + /* enqueue as head and send */ + req_server_enqueue_imsgq_head(ctx, conn, msg); + msg_send(ctx, conn); + + log_debug(LOG_NOTICE, "sent 'SELECT %d' to %s | %s", pool->redis_db, + pool->name.data, server->name.data); +} + +void +redis_swallow_msg(struct conn *conn, struct msg *pmsg, struct msg *msg) +{ + if (pmsg != NULL && pmsg->type == MSG_REQ_REDIS_SELECT && + msg != NULL && redis_error(msg)) { + struct server* conn_server; + struct server_pool* conn_pool; + struct mbuf* rsp_buffer; + uint8_t message[128]; + size_t copy_len; + + /* + * Get a substring from the message so that the inital - and the trailing + * \r\n is removed. + */ + conn_server = (struct server*)conn->owner; + conn_pool = conn_server->owner; + rsp_buffer = STAILQ_LAST(&msg->mhdr, mbuf, next); + copy_len = MIN(mbuf_length(rsp_buffer) - 3, sizeof(message) - 1); + + nc_memcpy(message, &rsp_buffer->start[1], copy_len); + message[copy_len] = 0; + + log_warn("SELECT %d failed on %s | %s: %s", + conn_pool->redis_db, conn_pool->name.data, + conn_server->name.data, message); + } +} diff -Nru nutcracker-0.4.0+dfsg/tests/_binaries/.gitignore nutcracker-0.4.1+dfsg/tests/_binaries/.gitignore --- nutcracker-0.4.0+dfsg/tests/_binaries/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/_binaries/.gitignore 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1 @@ +* diff -Nru nutcracker-0.4.0+dfsg/tests/conf/conf.py nutcracker-0.4.1+dfsg/tests/conf/conf.py --- nutcracker-0.4.0+dfsg/tests/conf/conf.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/conf/conf.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,15 @@ +#coding: utf-8 + +import os +import sys + +PWD = os.path.dirname(os.path.realpath(__file__)) +WORKDIR = os.path.join(PWD, '../') + +BINARYS = { + 'REDIS_SERVER_BINS' : os.path.join(WORKDIR, '_binaries/redis-*'), + 'REDIS_CLI' : os.path.join(WORKDIR, '_binaries/redis-cli'), + 'MEMCACHED_BINS' : os.path.join(WORKDIR, '_binaries/memcached'), + 'NUTCRACKER_BINS' : os.path.join(WORKDIR, '_binaries/nutcracker'), +} + diff -Nru nutcracker-0.4.0+dfsg/tests/conf/control.sh nutcracker-0.4.1+dfsg/tests/conf/control.sh --- nutcracker-0.4.0+dfsg/tests/conf/control.sh 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/conf/control.sh 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,38 @@ +#!/bin/bash + +start() +{ + stop + ulimit -c unlimited + + pushd . > /dev/null + + cd `dirname $$0` + ${startcmd} + popd +} + +stop() +{ + pkill -9 -f '${runcmd}' +} + +case C"$$1" in + C) + echo "Usage: $$0 {start|stop}" + ;; + Cstart) + start + echo "Done!" + ;; + Cstop) + stop + echo "Done!" + ;; + C*) + echo "Usage: $$0 {start|stop}" + ;; +esac + + + diff -Nru nutcracker-0.4.0+dfsg/tests/conf/redis.conf nutcracker-0.4.1+dfsg/tests/conf/redis.conf --- nutcracker-0.4.0+dfsg/tests/conf/redis.conf 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/conf/redis.conf 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,716 @@ + +# Redis configuration file example +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize yes + +#whitelist configure +#whitelist yes +#whitelist-file ./whitelist + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile ${pidfile} + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port ${port} + +# If you want you can bind a single interface, if the bind option is not +# specified all the interfaces will listen for incoming connections. +# +# bind 127.0.0.1 + +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +# unixsocket /tmp/redis.sock +# unixsocketperm 755 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Take the connection alive from the point of view of network +# equipment in the middle. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 60 seconds. +tcp-keepalive 60 + +# Specify the server verbosity level. +# This can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also 'stdout' can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile ${logfile} + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +################################ SNAPSHOTTING ################################# +# +# Save the DB on disk: +# +# save +# +# Will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# In the example below the behaviour will be to save: +# after 900 sec (15 min) if at least 1 key changed +# after 300 sec (5 min) if at least 10 keys changed +# after 60 sec if at least 10000 keys changed +# +# Note: you can disable saving at all commenting all the "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# + +#save 900 1 +#save 300 10 +#save 60 10000 +save "" + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in an hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# distater will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usually even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + +# Compress string objects using LZF when dump .rdb databases? +# For default that's set to 'yes' as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# The filename where to dump the DB +dbfilename dump.rdb + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# The Append Only File will also be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir ${dir} + +################################# REPLICATION ################################# + +# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# another Redis server. Note that the configuration is local to the slave +# so for example it is possible to configure the slave to save the DB with a +# different interval, or to listen to another port, and so on. +# +# slaveof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth + +# When a slave loses its connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extend you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets a timeout for both Bulk transfer I/O timeout and +# master data or ping response timeout. The default value is 60 seconds. +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +repl-timeout 120 + +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# slave data when slaves are disconnected for some time, so that when a slave +# wants to reconnect again, often a full resync is not needed, but a partial +# resync is enough, just passing the portion of data the slave missed while +# disconnected. +# +# The biggest the replication backlog, the longer the time the slave can be +# disconnected and later be able to perform a partial resynchronization. +# +# The backlog is only allocated once there is at least a slave connected. +# +repl-backlog-size 64mb + +# After a master has no longer connected slaves for some time, the backlog +# will be freed. The following option configures the amount of seconds that +# need to elapse, starting from the time the last slave disconnected, for +# the backlog buffer to be freed. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The slave priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# pick the one wtih priority 10, that is the lowest. +# +# However a special priority of 0 marks the slave as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# It is possible for a master to stop accepting writes if there are less than +# N slaves connected, having a lag less or equal than M seconds. +# +# The N slaves need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the slave, that is usually sent every second. +# +# This option does not GUARANTEES that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough slaves +# are available, to the specified number of seconds. +# +# For example to require at least 3 slaves with a lag <= 10 seconds use: +# +# min-slaves-to-write 3 +# min-slaves-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-slaves-to-write is set to 0 (feature disabled) and +# min-slaves-max-lag is set to 10. + +################################## SECURITY ################################### + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running redis-server. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to slaves may cause problems. + +################################### LIMITS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# maxclients 10000 + +# Don't use more memory than the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# accordingly to the eviction policy selected (see maxmemmory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# an hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). +# +maxmemory 5368709120 + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select among five behaviors: +# +# volatile-lru -> remove the key with an expire set using an LRU algorithm +# allkeys-lru -> remove any key accordingly to the LRU algorithm +# volatile-random -> remove a random key with an expire set +# allkeys-random -> remove a random key, any key +# volatile-ttl -> remove the key with the nearest expire time (minor TTL) +# noeviction -> don't expire at all, just return an error on write operations +# +# Note: with any of the above policies, Redis will return an error on write +# operations, when there are not suitable keys for eviction. +# +# At the date of writing this commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +maxmemory-policy volatile-lru + +# LRU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can select as well the sample +# size to check. For instance for default Redis will check three keys and +# pick the one that was used less recently, you can change the sample size +# using the following configuration directive. +# +maxmemory-samples 3 + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. + +appendonly yes + +# The name of the append only file (default: "appendonly.aof") +# appendfilename appendonly.aof + +# The fsync() call tells the Operating System to actually write data on disk +# instead to wait for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log . Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec", as that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (if no rewrite has happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 0 +auto-aof-rewrite-min-size 64mb + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceed the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write commands was +# already issue by the script but the user don't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################ REDIS CLUSTER ############################### +# +# Normal Redis instances can't be part of a Redis Cluster; only nodes that are +# started as cluster nodes can. In order to start a Redis instance as a +# cluster node enable the cluster support uncommenting the following: +# +# cluster-enabled yes + +# Every cluster node has a cluster configuration file. This file is not +# intended to be edited by hand. It is created and updated by Redis nodes. +# Every Redis Cluster node requires a different cluster configuration file. +# Make sure that instances running in the same system does not have +# overlapping cluster configuration file names. +# +# cluster-config-file nodes-6379.conf + +# Cluster node timeout is the amount of milliseconds a node must be unreachable +# for it to be considered in failure state. +# Most other internal time limits are multiple of the node timeout. +# +# cluster-node-timeout 15000 + +# In order to setup your cluster make sure to read the documentation +# available at http://redis.io web site. + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +############################# Event notification ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at http://redis.io/topics/keyspace-events +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# \ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# A Alias for g\lshzxe, so that the "AKE" string means all the events. +# +# The "notify-keyspace-events" takes as argument a string that is composed +# by zero or multiple characters. The empty string means that notifications +# are disabled at all. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Similarly to hashes, small lists are also encoded in a special way in order +# to save a lot of space. The special representation is only used when +# you are under the following limits: +list-max-ziplist-entries 512 +list-max-ziplist-value 64 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happens to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into an hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# active rehashing the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply form time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients +# slave -> slave clients and MONITOR clients +# pubsub -> clients subcribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled by setting them to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeot, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are perforemd with the same frequency, but Redis checks for +# tasks to perform accordingly to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis server but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# include /path/to/local.conf +# include /path/to/other.conf diff -Nru nutcracker-0.4.0+dfsg/tests/.gitignore nutcracker-0.4.1+dfsg/tests/.gitignore --- nutcracker-0.4.0+dfsg/tests/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/.gitignore 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,3 @@ +*.pyc +*.out +*.log diff -Nru nutcracker-0.4.0+dfsg/tests/lib/server_modules.py nutcracker-0.4.1+dfsg/tests/lib/server_modules.py --- nutcracker-0.4.0+dfsg/tests/lib/server_modules.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/lib/server_modules.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,321 @@ +#!/usr/bin/env python +#coding: utf-8 +#file : server_modules.py +#author : ning +#date : 2014-02-24 13:00:28 + +import os +import sys + +from utils import * +import conf + +class Base: + ''' + Sub class should implement: + _alive, _pre_deploy, status, and init self.args + ''' + def __init__(self, name, host, port, path): + self.args = { + 'name' : name, + 'host' : host, + 'port' : port, + 'path' : path, + + #startcmd and runcmd will used to generate the control script + #used for the start cmd + 'startcmd' : '', + #process name you see in `ps -aux`, used this to generate stop cmd + 'runcmd' : '', + 'logfile' : '', + } + + def __str__(self): + return TT('[$name:$host:$port]', self.args) + + def deploy(self): + logging.info('deploy %s' % self) + self._run(TTCMD('mkdir -p $path/bin && \ + mkdir -p $path/conf && \ + mkdir -p $path/log && \ + mkdir -p $path/data', + self.args)) + + self._pre_deploy() + self._gen_control_script() + + def _gen_control_script(self): + content = file(os.path.join(WORKDIR, 'conf/control.sh')).read() + content = TT(content, self.args) + + control_filename = TT('${path}/${name}_control', self.args) + + fout = open(control_filename, 'w+') + fout.write(content) + fout.close() + os.chmod(control_filename, 0755) + + def start(self): + if self._alive(): + logging.warn('%s already running' %(self) ) + return + + logging.debug('starting %s' % self) + t1 = time.time() + sleeptime = .1 + + cmd = TT("cd $path && ./${name}_control start", self.args) + self._run(cmd) + + while not self._alive(): + lets_sleep(sleeptime) + if sleeptime < 5: + sleeptime *= 2 + else: + sleeptime = 5 + logging.warn('%s still not alive' % self) + + t2 = time.time() + logging.info('%s start ok in %.2f seconds' %(self, t2-t1) ) + + def stop(self): + if not self._alive(): + logging.warn('%s already stop' %(self) ) + return + + cmd = TT("cd $path && ./${name}_control stop", self.args) + self._run(cmd) + + t1 = time.time() + while self._alive(): + lets_sleep() + t2 = time.time() + logging.info('%s stop ok in %.2f seconds' %(self, t2-t1) ) + + def pid(self): + cmd = TT("pgrep -f '^$runcmd'", self.args) + return self._run(cmd) + + def status(self): + logging.warn("status: not implement") + + def _alive(self): + logging.warn("_alive: not implement") + + def _run(self, raw_cmd): + ret = system(raw_cmd, logging.debug) + logging.debug('return : [%d] [%s] ' % (len(ret), shorten(ret)) ) + return ret + + def clean(self): + cmd = TT("rm -rf $path", self.args) + self._run(cmd) + + def host(self): + return self.args['host'] + + def port(self): + return self.args['port'] + +class RedisServer(Base): + def __init__(self, host, port, path, cluster_name, server_name, auth = None): + Base.__init__(self, 'redis', host, port, path) + + self.args['startcmd'] = TT('bin/redis-server conf/redis.conf', self.args) + self.args['runcmd'] = TT('redis-server \*:$port', self.args) + self.args['conf'] = TT('$path/conf/redis.conf', self.args) + self.args['pidfile'] = TT('$path/log/redis.pid', self.args) + self.args['logfile'] = TT('$path/log/redis.log', self.args) + self.args['dir'] = TT('$path/data', self.args) + self.args['REDIS_CLI'] = conf.BINARYS['REDIS_CLI'] + + self.args['cluster_name'] = cluster_name + self.args['server_name'] = server_name + self.args['auth'] = auth + + def _info_dict(self): + cmd = TT('$REDIS_CLI -h $host -p $port INFO', self.args) + if self.args['auth']: + cmd = TT('$REDIS_CLI -h $host -p $port -a $auth INFO', self.args) + info = self._run(cmd) + + info = [line.split(':', 1) for line in info.split('\r\n') + if not line.startswith('#')] + info = [i for i in info if len(i) > 1] + return defaultdict(str, info) #this is a defaultdict, be Notice + + def _ping(self): + cmd = TT('$REDIS_CLI -h $host -p $port PING', self.args) + if self.args['auth']: + cmd = TT('$REDIS_CLI -h $host -p $port -a $auth PING', self.args) + return self._run(cmd) + + def _alive(self): + return strstr(self._ping(), 'PONG') + + def _gen_conf(self): + content = file(os.path.join(WORKDIR, 'conf/redis.conf')).read() + content = TT(content, self.args) + if self.args['auth']: + content += '\r\nrequirepass %s' % self.args['auth'] + return content + + def _pre_deploy(self): + self.args['BINS'] = conf.BINARYS['REDIS_SERVER_BINS'] + self._run(TT('cp $BINS $path/bin/', self.args)) + + fout = open(TT('$path/conf/redis.conf', self.args), 'w+') + fout.write(self._gen_conf()) + fout.close() + + def status(self): + uptime = self._info_dict()['uptime_in_seconds'] + if uptime: + logging.info('%s uptime %s seconds' % (self, uptime)) + else: + logging.error('%s is down' % self) + + def isslaveof(self, master_host, master_port): + info = self._info_dict() + if info['master_host'] == master_host and \ + int(info['master_port']) == master_port: + logging.debug('already slave of %s:%s' % (master_host, master_port)) + return True + + def slaveof(self, master_host, master_port): + cmd = 'SLAVEOF %s %s' % (master_host, master_port) + return self.rediscmd(cmd) + + def rediscmd(self, cmd): + args = copy.deepcopy(self.args) + args['cmd'] = cmd + cmd = TT('$REDIS_CLI -h $host -p $port $cmd', args) + logging.info('%s %s' % (self, cmd)) + return self._run(cmd) + +class Memcached(Base): + def __init__(self, host, port, path, cluster_name, server_name): + Base.__init__(self, 'memcached', host, port, path) + + self.args['startcmd'] = TT('bin/memcached -d -p $port', self.args) + self.args['runcmd'] = self.args['startcmd'] + + self.args['cluster_name'] = cluster_name + self.args['server_name'] = server_name + + def _alive(self): + cmd = TT('echo "stats" | socat - TCP:$host:$port', self.args) + ret = self._run(cmd) + return strstr(ret, 'END') + + def _pre_deploy(self): + self.args['BINS'] = conf.BINARYS['MEMCACHED_BINS'] + self._run(TT('cp $BINS $path/bin/', self.args)) + +class NutCracker(Base): + def __init__(self, host, port, path, cluster_name, masters, mbuf=512, + verbose=5, is_redis=True, redis_auth=None): + Base.__init__(self, 'nutcracker', host, port, path) + + self.masters = masters + + self.args['mbuf'] = mbuf + self.args['verbose'] = verbose + self.args['redis_auth'] = redis_auth + self.args['conf'] = TT('$path/conf/nutcracker.conf', self.args) + self.args['pidfile'] = TT('$path/log/nutcracker.pid', self.args) + self.args['logfile'] = TT('$path/log/nutcracker.log', self.args) + self.args['status_port'] = self.args['port'] + 1000 + + self.args['startcmd'] = TTCMD('bin/nutcracker -d -c $conf -o $logfile \ + -p $pidfile -s $status_port \ + -v $verbose -m $mbuf -i 1', self.args) + self.args['runcmd'] = TTCMD('bin/nutcracker -d -c $conf -o $logfile \ + -p $pidfile -s $status_port', self.args) + + self.args['cluster_name']= cluster_name + self.args['is_redis']= str(is_redis).lower() + + def _alive(self): + return self._info_dict() + + def _gen_conf_section(self): + template = ' - $host:$port:1 $server_name' + cfg = '\n'.join([TT(template, master.args) for master in self.masters]) + return cfg + + def _gen_conf(self): + content = ''' +$cluster_name: + listen: 0.0.0.0:$port + hash: fnv1a_64 + distribution: modula + preconnect: true + auto_eject_hosts: false + redis: $is_redis + backlog: 512 + timeout: 400 + client_connections: 0 + server_connections: 1 + server_retry_timeout: 2000 + server_failure_limit: 2 + servers: +''' + if self.args['redis_auth']: + content = content.replace('redis: $is_redis', + 'redis: $is_redis\r\n redis_auth: $redis_auth') + content = TT(content, self.args) + return content + self._gen_conf_section() + + def _pre_deploy(self): + self.args['BINS'] = conf.BINARYS['NUTCRACKER_BINS'] + self._run(TT('cp $BINS $path/bin/', self.args)) + + fout = open(TT('$path/conf/nutcracker.conf', self.args), 'w+') + fout.write(self._gen_conf()) + fout.close() + + def version(self): + #This is nutcracker-0.4.0 + s = self._run(TT('$BINS --version', self.args)) + return s.strip().replace('This is nutcracker-', '') + + def _info_dict(self): + try: + c = telnetlib.Telnet(self.args['host'], self.args['status_port']) + ret = c.read_all() + return json_decode(ret) + except Exception, e: + logging.debug('can not get _info_dict of nutcracker, \ + [Exception: %s]' % (e, )) + return None + + def reconfig(self, masters): + self.masters = masters + self.stop() + self.deploy() + self.start() + logging.info('proxy %s:%s is updated' % (self.args['host'], self.args['port'])) + + def logfile(self): + return self.args['logfile'] + + def cleanlog(self): + cmd = TT("rm '$logfile'", self.args) + self._run(cmd) + + def signal(self, signo): + self.args['signo'] = signo + cmd = TT("pkill -$signo -f '^$runcmd'", self.args) + self._run(cmd) + + def reload(self): + self.signal('USR1') + + def set_config(self, content): + fout = open(TT('$path/conf/nutcracker.conf', self.args), 'w+') + fout.write(content) + fout.close() + + self.reload() + diff -Nru nutcracker-0.4.0+dfsg/tests/lib/utils.py nutcracker-0.4.1+dfsg/tests/lib/utils.py --- nutcracker-0.4.0+dfsg/tests/lib/utils.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/lib/utils.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,119 @@ +import os +import re +import sys +import time +import copy +import thread +import socket +import threading +import logging +import inspect +import argparse +import telnetlib +import redis +import random +import redis +import json +import glob +import commands + +from collections import defaultdict +from argparse import RawTextHelpFormatter + +from string import Template + +PWD = os.path.dirname(os.path.realpath(__file__)) +WORKDIR = os.path.join(PWD, '../') + +def getenv(key, default): + if key in os.environ: + return os.environ[key] + return default + +logfile = getenv('T_LOGFILE', 'log/t.log') +if logfile == '-': + logging.basicConfig(level=logging.DEBUG, + format="%(asctime)-15s [%(threadName)s] [%(levelname)s] %(message)s") +else: + logging.basicConfig(filename=logfile, level=logging.DEBUG, + format="%(asctime)-15s [%(threadName)s] [%(levelname)s] %(message)s") + +logging.info("test running") + +def strstr(s1, s2): + return s1.find(s2) != -1 + +def lets_sleep(SLEEP_TIME = 0.1): + time.sleep(SLEEP_TIME) + +def TT(template, args): #todo: modify all + return Template(template).substitute(args) + +def TTCMD(template, args): #todo: modify all + ''' + Template for cmd (we will replace all spaces) + ''' + ret = TT(template, args) + return re.sub(' +', ' ', ret) + +def nothrow(ExceptionToCheck=Exception, logger=None): + def deco_retry(f): + def f_retry(*args, **kwargs): + try: + return f(*args, **kwargs) + except ExceptionToCheck, e: + if logger: + logger.info(e) + else: + print str(e) + return f_retry # true decorator + return deco_retry + +@nothrow(Exception) +def test_nothrow(): + raise Exception('exception: xx') + +def json_encode(j): + return json.dumps(j, indent=4, cls=MyEncoder) + +def json_decode(j): + return json.loads(j) + +#commands dose not work on windows.. +def system(cmd, log_fun=logging.info): + if log_fun: log_fun(cmd) + r = commands.getoutput(cmd) + return r + +def shorten(s, l=80): + if len(s)<=l: + return s + return s[:l-3] + '...' + +def assert_true(a): + assert a, 'assert fail: except true, got %s' % a + +def assert_equal(a, b): + assert a == b, 'assert fail: %s vs %s' % (shorten(str(a)), shorten(str(b))) + +def assert_raises(exception_cls, callable, *args, **kwargs): + try: + callable(*args, **kwargs) + except exception_cls as e: + return e + except Exception as e: + assert False, 'assert_raises %s but raised: %s' % (exception_cls, e) + assert False, 'assert_raises %s but nothing raise' % (exception_cls) + +def assert_fail(err_response, callable, *args, **kwargs): + try: + callable(*args, **kwargs) + except Exception as e: + assert re.search(err_response, str(e)), \ + 'assert "%s" but got "%s"' % (err_response, e) + return + + assert False, 'assert_fail %s but nothing raise' % (err_response) + +if __name__ == "__main__": + test_nothrow() diff -Nru nutcracker-0.4.0+dfsg/tests/log/.gitignore nutcracker-0.4.1+dfsg/tests/log/.gitignore --- nutcracker-0.4.0+dfsg/tests/log/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/log/.gitignore 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1 @@ +*.log diff -Nru nutcracker-0.4.0+dfsg/tests/README.rst nutcracker-0.4.1+dfsg/tests/README.rst --- nutcracker-0.4.0+dfsg/tests/README.rst 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/README.rst 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,74 @@ +Python testing facilities for twemproxy, this test suite is based on https://github.com/idning/redis-mgr + +already add to https://travis-ci.org/idning/twemproxy as travis-ci + +see https://github.com/idning/twemproxy/blob/travis-ci/travis.sh + +usage +===== + +1. install dependency:: + + pip install nose + pip install git+https://github.com/andymccurdy/redis-py.git@2.9.0 + pip install git+https://github.com/idning/python-memcached.git#egg=memcache + +2. copy binarys to _binaries/:: + + _binaries/ + |-- nutcracker + |-- redis-benchmark + |-- redis-check-aof + |-- redis-check-dump + |-- redis-cli + |-- redis-sentinel + |-- redis-server + |-- memcached + +3. run:: + + $ nosetests -v + test_del.test_multi_delete_on_readonly ... ok + test_mget.test_mget ... ok + + ---------------------------------------------------------------------- + Ran 2 tests in 4.483s + + OK + +4. add A case:: + + cp tests/test_del.py tests/test_xxx.py + vim tests/test_xxx.py + + + +variables +========= +:: + + export T_VERBOSE=9 will start nutcracker with '-v 9' (default:4) + export T_MBUF=512 will start nutcracker whit '-m 512' (default:521) + export T_LARGE=10000 will test 10000 keys for mget/mset (default:1000) + +T_LOGFILE: + +- to put test log on stderr:: + + export T_LOGFILE=- + +- to put test log on t.log:: + + export T_LOGFILE=t.log + + or:: + + unset T_LOGFILE + + +notes +===== + +- After all the tests. you may got a core because we have a case in test_signal which will send SEGV to nutcracker + + diff -Nru nutcracker-0.4.0+dfsg/tests/test_memcache/test_gets.py nutcracker-0.4.1+dfsg/tests/test_memcache/test_gets.py --- nutcracker-0.4.0+dfsg/tests/test_memcache/test_gets.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_memcache/test_gets.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,108 @@ +#!/usr/bin/env python +#coding: utf-8 + +import os +import sys +import redis +import memcache + +PWD = os.path.dirname(os.path.realpath(__file__)) +WORKDIR = os.path.join(PWD, '../') +sys.path.append(os.path.join(WORKDIR, 'lib/')) +sys.path.append(os.path.join(WORKDIR, 'conf/')) +import conf + +from server_modules import * +from utils import * + +CLUSTER_NAME = 'ntest' +all_mc= [ + Memcached('127.0.0.1', 2200, '/tmp/r/memcached-2200/', CLUSTER_NAME, 'mc-2200'), + Memcached('127.0.0.1', 2201, '/tmp/r/memcached-2201/', CLUSTER_NAME, 'mc-2201'), + ] + +nc_verbose = int(getenv('T_VERBOSE', 4)) +mbuf = int(getenv('T_MBUF', 512)) +large = int(getenv('T_LARGE', 1000)) + +nc = NutCracker('127.0.0.1', 4100, '/tmp/r/nutcracker-4100', CLUSTER_NAME, + all_mc, mbuf=mbuf, verbose=nc_verbose, is_redis=False) + +def setup(): + for r in all_mc: + r.deploy() + r.stop() + r.start() + + nc.deploy() + nc.stop() + nc.start() + +def teardown(): + for r in all_mc: + r.stop() + assert(nc._alive()) + nc.stop() + +def getconn(): + host_port = '%s:%s' % (nc.host(), nc.port()) + return memcache.Client([host_port]) + +def test_basic(): + conn = getconn() + conn.set('k', 'v') + assert('v' == conn.get('k')) + + conn.set("key", "1") + for i in range(10): + conn.incr("key") + assert(str(i+2) == conn.get('key')) + + conn.delete("key") + assert(None == conn.get('key')) + +default_kv = {'kkk-%s' % i :'vvv-%s' % i for i in range(10)} +def test_mget_mset(kv=default_kv): + conn = getconn() + conn.set_multi(kv) + keys = sorted(kv.keys()) + + assert(conn.get_multi(keys) == kv) + assert(conn.gets_multi(keys) == kv) + + #del + conn.delete_multi(keys) + #mget again + vals = conn.get_multi(keys) + assert({} == vals) + +def test_mget_mset_large(): + for cnt in range(179, large, 179): + #print 'test', cnt + kv = {'kkk-%s' % i :'vvv-%s' % i for i in range(cnt)} + test_mget_mset(kv) + +def test_mget_mset_key_not_exists(kv=default_kv): + conn = getconn() + conn.set_multi(kv) + + keys = kv.keys() + keys2 = ['x-'+k for k in keys] + keys = keys + keys2 + random.shuffle(keys) + + for i in range(2): + #mget to check + vals = conn.get_multi(keys) + for i, k in enumerate(keys): + if k in kv: + assert(kv[k] == vals[k]) + else: + assert(k not in vals) + + #del + conn.delete_multi(keys) + #mget again + vals = conn.get_multi(keys) + assert({} == vals) + diff -Nru nutcracker-0.4.0+dfsg/tests/test_redis/common.py nutcracker-0.4.1+dfsg/tests/test_redis/common.py --- nutcracker-0.4.0+dfsg/tests/test_redis/common.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_redis/common.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,53 @@ +#!/usr/bin/env python +#coding: utf-8 + +import os +import sys +import redis + +PWD = os.path.dirname(os.path.realpath(__file__)) +WORKDIR = os.path.join(PWD,'../') +sys.path.append(os.path.join(WORKDIR,'lib/')) +sys.path.append(os.path.join(WORKDIR,'conf/')) + +import conf + +from server_modules import * +from utils import * + +CLUSTER_NAME = 'ntest' +nc_verbose = int(getenv('T_VERBOSE', 5)) +mbuf = int(getenv('T_MBUF', 512)) +large = int(getenv('T_LARGE', 1000)) + +all_redis = [ + RedisServer('127.0.0.1', 2100, '/tmp/r/redis-2100/', CLUSTER_NAME, 'redis-2100'), + RedisServer('127.0.0.1', 2101, '/tmp/r/redis-2101/', CLUSTER_NAME, 'redis-2101'), + ] + +nc = NutCracker('127.0.0.1', 4100, '/tmp/r/nutcracker-4100', CLUSTER_NAME, + all_redis, mbuf=mbuf, verbose=nc_verbose) + +def setup(): + print 'setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose) + for r in all_redis + [nc]: + r.clean() + r.deploy() + r.stop() + r.start() + +def teardown(): + for r in all_redis + [nc]: + assert(r._alive()) + r.stop() + +default_kv = {'kkk-%s' % i : 'vvv-%s' % i for i in range(10)} + +def getconn(): + for r in all_redis: + c = redis.Redis(r.host(), r.port()) + c.flushdb() + + r = redis.Redis(nc.host(), nc.port()) + return r + diff -Nru nutcracker-0.4.0+dfsg/tests/test_redis/test_auth.py nutcracker-0.4.1+dfsg/tests/test_redis/test_auth.py --- nutcracker-0.4.0+dfsg/tests/test_redis/test_auth.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_redis/test_auth.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,111 @@ +#!/usr/bin/env python +#coding: utf-8 + +from common import * + +all_redis = [ + RedisServer('127.0.0.1', 2100, '/tmp/r/redis-2100/', + CLUSTER_NAME, 'redis-2100', auth = 'hellopasswd'), + RedisServer('127.0.0.1', 2101, '/tmp/r/redis-2101/', + CLUSTER_NAME, 'redis-2101', auth = 'hellopasswd'), +] + +nc = NutCracker('127.0.0.1', 4100, '/tmp/r/nutcracker-4100', CLUSTER_NAME, + all_redis, mbuf=mbuf, verbose=nc_verbose, + redis_auth = 'hellopasswd') + +nc_badpass = NutCracker('127.0.0.1', 4101, '/tmp/r/nutcracker-4101', CLUSTER_NAME, + all_redis, mbuf=mbuf, verbose=nc_verbose, + redis_auth = 'badpasswd') +nc_nopass = NutCracker('127.0.0.1', 4102, '/tmp/r/nutcracker-4102', CLUSTER_NAME, + all_redis, mbuf=mbuf, verbose=nc_verbose) + +def setup(): + print 'setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose) + for r in all_redis + [nc, nc_badpass, nc_nopass]: + r.clean() + r.deploy() + r.stop() + r.start() + +def teardown(): + for r in all_redis + [nc, nc_badpass, nc_nopass]: + assert(r._alive()) + r.stop() + +default_kv = {'kkk-%s' % i : 'vvv-%s' % i for i in range(10)} + +def getconn(): + r = redis.Redis(nc.host(), nc.port()) + return r + +''' + +cases: + + +redis proxy case +1 1 test_auth_basic +1 bad test_badpass_on_proxy +1 0 test_nopass_on_proxy +0 0 already tested on other case +0 1 + +''' + +def test_auth_basic(): + # we hope to have same behavior when the server is redis or twemproxy + conns = [ + redis.Redis(all_redis[0].host(), all_redis[0].port()), + redis.Redis(nc.host(), nc.port()), + ] + + for r in conns: + assert_fail('NOAUTH|operation not permitted', r.ping) + assert_fail('NOAUTH|operation not permitted', r.set, 'k', 'v') + assert_fail('NOAUTH|operation not permitted', r.get, 'k') + + # bad passwd + assert_fail('invalid password', r.execute_command, 'AUTH', 'badpasswd') + + # everything is ok after auth + r.execute_command('AUTH', 'hellopasswd') + r.set('k', 'v') + assert(r.ping() == True) + assert(r.get('k') == 'v') + + # auth fail here, should we return ok or not => we will mark the conn state as not authed + assert_fail('invalid password', r.execute_command, 'AUTH', 'badpasswd') + + assert_fail('NOAUTH|operation not permitted', r.ping) + assert_fail('NOAUTH|operation not permitted', r.get, 'k') + +def test_nopass_on_proxy(): + r = redis.Redis(nc_nopass.host(), nc_nopass.port()) + + # if you config pass on redis but not on twemproxy, + # twemproxy will reply ok for ping, but once you do get/set, you will get errmsg from redis + assert(r.ping() == True) + assert_fail('NOAUTH|operation not permitted', r.set, 'k', 'v') + assert_fail('NOAUTH|operation not permitted', r.get, 'k') + + # proxy has no pass, when we try to auth + assert_fail('Client sent AUTH, but no password is set', r.execute_command, 'AUTH', 'anypasswd') + pass + +def test_badpass_on_proxy(): + r = redis.Redis(nc_badpass.host(), nc_badpass.port()) + + assert_fail('NOAUTH|operation not permitted', r.ping) + assert_fail('NOAUTH|operation not permitted', r.set, 'k', 'v') + assert_fail('NOAUTH|operation not permitted', r.get, 'k') + + # we can auth with bad pass (twemproxy will say ok for this) + r.execute_command('AUTH', 'badpasswd') + # after that, we still got NOAUTH for get/set (return from redis-server) + assert(r.ping() == True) + assert_fail('NOAUTH|operation not permitted', r.set, 'k', 'v') + assert_fail('NOAUTH|operation not permitted', r.get, 'k') + +def setup_and_wait(): + time.sleep(60*60) diff -Nru nutcracker-0.4.0+dfsg/tests/test_redis/test_basic.py nutcracker-0.4.1+dfsg/tests/test_redis/test_basic.py --- nutcracker-0.4.0+dfsg/tests/test_redis/test_basic.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_redis/test_basic.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,127 @@ +#!/usr/bin/env python +#coding: utf-8 + +from common import * + +def test_setget(): + r = getconn() + + rst = r.set('k', 'v') + assert(r.get('k') == 'v') + +def test_msetnx(): + r = getconn() + + #not supported + keys = default_kv.keys() + assert_fail('Socket closed|Connection closed', r.msetnx,**default_kv) + +def test_null_key(): + r = getconn() + rst = r.set('', 'v') + assert(r.get('') == 'v') + + rst = r.set('', '') + assert(r.get('') == '') + + kv = {'' : 'val', 'k': 'v'} + ret = r.mset(**kv) + assert(r.get('') == 'val') + +def test_ping_quit(): + r = getconn() + assert(r.ping() == True) + + #get set + rst = r.set('k', 'v') + assert(r.get('k') == 'v') + + assert_fail('Socket closed|Connection closed', r.execute_command, 'QUIT') + +def test_slow_req(): + r = getconn() + + kv = {'mkkk-%s' % i : 'mvvv-%s' % i for i in range(500000)} + + pipe = r.pipeline(transaction=False) + pipe.set('key-1', 'v1') + pipe.get('key-1') + pipe.hmset('xxx', kv) + pipe.get('key-2') + pipe.get('key-3') + + assert_fail('timed out', pipe.execute) + +def test_signal(): + #init + nc.cleanlog() + nc.signal('HUP') + + nc.signal('HUP') + nc.signal('TTIN') + nc.signal('TTOU') + nc.signal('SEGV') + + time.sleep(.3) + log = file(nc.logfile()).read() + + assert(strstr(log, 'HUP')) + assert(strstr(log, 'TTIN')) + assert(strstr(log, 'TTOU')) + assert(strstr(log, 'SEGV')) + + #recover + nc.start() + +def test_nc_stats(): + nc.stop() #reset counters + nc.start() + r = getconn() + kv = {'kkk-%s' % i :'vvv-%s' % i for i in range(10)} + for k, v in kv.items(): + r.set(k, v) + r.get(k) + + def get_stat(name): + time.sleep(1) + stat = nc._info_dict() + #pprint(stat) + if name in ['client_connections', 'client_eof', 'client_err', \ + 'forward_error', 'fragments', 'server_ejects']: + return stat[CLUSTER_NAME][name] + + #sum num of each server + ret = 0 + for k, v in stat[CLUSTER_NAME].items(): + if type(v) == dict: + ret += v[name] + return ret + + assert(get_stat('requests') == 20) + assert(get_stat('responses') == 20) + + ##### mget + keys = kv.keys() + r.mget(keys) + + #for version<=0.3.0 + #assert(get_stat('requests') == 30) + #assert(get_stat('responses') == 30) + + #for mget-improve + assert(get_stat('requests') == 22) + assert(get_stat('responses') == 22) + +def test_issue_323(): + # do on redis + r = all_redis[0] + c = redis.Redis(r.host(), r.port()) + assert([1, 'OK'] == c.eval("return {1, redis.call('set', 'x', '1')}", 1, 'tmp')) + + # do on twemproxy + c = getconn() + assert([1, 'OK'] == c.eval("return {1, redis.call('set', 'x', '1')}", 1, 'tmp')) + +def setup_and_wait(): + time.sleep(60*60) + diff -Nru nutcracker-0.4.0+dfsg/tests/test_redis/test_commands.py nutcracker-0.4.1+dfsg/tests/test_redis/test_commands.py --- nutcracker-0.4.0+dfsg/tests/test_redis/test_commands.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_redis/test_commands.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,95 @@ +#!/usr/bin/env python +#coding: utf-8 + +from common import * + +def test_linsert(): + r = getconn() + + r.rpush('mylist', 'Hello') + r.rpush('mylist', 'World') + r.linsert('mylist', 'BEFORE', 'World', 'There') + + rst = r.lrange('mylist', 0, -1) + assert(rst == ['Hello', 'There', 'World']) + +def test_lpush_lrange(): + r = getconn() + + vals = ['vvv-%s' % i for i in range(10) ] + assert([] == r.lrange('mylist', 0, -1)) + + r.lpush('mylist', *vals) + rst = r.lrange('mylist', 0, -1) + + assert(10 == len(rst)) + +def test_hscan(): + r = getconn() + + kv = {'kkk-%s' % i : 'vvv-%s' % i for i in range(10)} + r.hmset('a', kv) + + cursor, dic = r.hscan('a') + assert(str(cursor) == '0') + assert(dic == kv) + + cursor, dic = r.hscan('a', match='kkk-5') + assert(str(cursor) == '0') + assert(dic == {'kkk-5': 'vvv-5'}) + +def test_hscan_large(): + r = getconn() + + kv = {'x'* 100 + 'kkk-%s' % i : 'vvv-%s' % i for i in range(1000)} + r.hmset('a', kv) + + cursor = '0' + dic = {} + while True: + cursor, t = r.hscan('a', cursor, count=10) + for k, v in t.items(): + dic[k] = v + + if '0' == str(cursor): + break + + assert(dic == kv) + + cursor, dic = r.hscan('a', '0', match='*kkk-5*', count=1000) + if str(cursor) == '0': + assert(len(dic) == 111) + else: + assert(len(dic) == 111) + + #again. + cursor, dic = r.hscan('a', cursor, match='*kkk-5*', count=1000) + assert(str(cursor) == '0') + assert(len(dic) == 0) + +def test_zscan(): + r = getconn() + + r.zadd('a', 'a', 1, 'b', 2, 'c', 3) + + cursor, pairs = r.zscan('a') + assert(str(cursor) == '0') + assert(set(pairs) == set([('a', 1), ('b', 2), ('c', 3)])) + + cursor, pairs = r.zscan('a', match='a') + assert(str(cursor) == '0') + assert(set(pairs) == set([('a', 1)])) + +def test_sscan(): + r = getconn() + + r.sadd('a', 1, 2, 3) + + cursor, members = r.sscan('a') + assert(str(cursor) == '0') + assert(set(members) == set(['1', '2', '3'])) + + cursor, members = r.sscan('a', match='1') + assert(str(cursor) == '0') + assert(set(members) == set(['1'])) + diff -Nru nutcracker-0.4.0+dfsg/tests/test_redis/test_mget_large_binary.py nutcracker-0.4.1+dfsg/tests/test_redis/test_mget_large_binary.py --- nutcracker-0.4.0+dfsg/tests/test_redis/test_mget_large_binary.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_redis/test_mget_large_binary.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,34 @@ +#!/usr/bin/env python +#coding: utf-8 + +from common import * +from test_mget_mset import test_mget_mset as _mget_mset + +#force to use large mbuf, we need to copy the setup/teardown here.. + +mbuf = 64*1024 + +nc = NutCracker(nc.host(), nc.port(), '/tmp/r/nutcracker-4100', CLUSTER_NAME, + all_redis, mbuf=mbuf, verbose=nc_verbose) + +def setup(): + print 'special setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose) + for r in all_redis + [nc]: + r.deploy() + r.stop() + r.start() + +def teardown(): + for r in all_redis + [nc]: + assert(r._alive()) + r.stop() + +###################################################### +def test_mget_binary_value(cnt=5): + kv = {} + for i in range(cnt): + kv['kkk-%s' % i] = os.urandom(1024*1024*16+1024) #16M + for i in range(cnt): + kv['kkk2-%s' % i] = '' + _mget_mset(kv) + diff -Nru nutcracker-0.4.0+dfsg/tests/test_redis/test_mget_mset.py nutcracker-0.4.1+dfsg/tests/test_redis/test_mget_mset.py --- nutcracker-0.4.0+dfsg/tests/test_redis/test_mget_mset.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_redis/test_mget_mset.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,242 @@ +#!/usr/bin/env python +#coding: utf-8 + +from common import * + +def test_mget_mset(kv=default_kv): + r = getconn() + + def insert_by_pipeline(): + pipe = r.pipeline(transaction=False) + for k, v in kv.items(): + pipe.set(k, v) + pipe.execute() + + def insert_by_mset(): + ret = r.mset(**kv) + + #insert_by_mset() #only the mget-imporve branch support this + try: + insert_by_mset() #only the mget-imporve branch support this + except: + insert_by_pipeline() + + keys = kv.keys() + + #mget to check + vals = r.mget(keys) + for i, k in enumerate(keys): + assert(kv[k] == vals[i]) + + #del + assert (len(keys) == r.delete(*keys) ) + + #mget again + vals = r.mget(keys) + + for i, k in enumerate(keys): + assert(None == vals[i]) + +def test_mget_mset_on_key_not_exist(kv=default_kv): + r = getconn() + + def insert_by_pipeline(): + pipe = r.pipeline(transaction=False) + for k, v in kv.items(): + pipe.set(k, v) + pipe.execute() + + def insert_by_mset(): + ret = r.mset(**kv) + + try: + insert_by_mset() #only the mget-imporve branch support this + except: + insert_by_pipeline() + + keys = kv.keys() + keys2 = ['x-'+k for k in keys] + keys = keys + keys2 + random.shuffle(keys) + + #mget to check + vals = r.mget(keys) + for i, k in enumerate(keys): + if k in kv: + assert(kv[k] == vals[i]) + else: + assert(vals[i] == None) + + #del + assert (len(kv) == r.delete(*keys) ) + + #mget again + vals = r.mget(keys) + + for i, k in enumerate(keys): + assert(None == vals[i]) + +def test_mget_mset_large(): + for cnt in range(171, large, 171): + kv = {'kkk-%s' % i :'vvv-%s' % i for i in range(cnt)} + test_mget_mset(kv) + +def test_mget_special_key(cnt=5): + #key length = 512-48-1 + kv = {} + for i in range(cnt): + k = 'kkk-%s' % i + k = k + 'x'*(512-48-1-len(k)) + kv[k] = 'vvv' + + test_mget_mset(kv) + +def test_mget_special_key_2(cnt=5): + #key length = 512-48-2 + kv = {} + for i in range(cnt): + k = 'kkk-%s' % i + k = k + 'x'*(512-48-2-len(k)) + kv[k] = 'vvv'*9 + + test_mget_mset(kv) + +def test_mget_on_backend_down(): + #one backend down + + r = redis.Redis(nc.host(), nc.port()) + assert_equal(None, r.get('key-2')) + assert_equal(None, r.get('key-1')) + + all_redis[0].stop() + + assert_fail('Connection refused|reset by peer|Broken pipe', r.mget, 'key-1') + assert_fail('Connection refused|reset by peer|Broken pipe', r.get, 'key-1') + assert_equal(None, r.get('key-2')) + + keys = ['key-1', 'key-2', 'kkk-3'] + assert_fail('Connection refused|reset by peer|Broken pipe', r.mget, *keys) + + #all backend down + all_redis[1].stop() + r = redis.Redis(nc.host(), nc.port()) + + assert_fail('Connection refused|reset by peer|Broken pipe', r.mget, 'key-1') + assert_fail('Connection refused|reset by peer|Broken pipe', r.mget, 'key-2') + + keys = ['key-1', 'key-2', 'kkk-3'] + assert_fail('Connection refused|reset by peer|Broken pipe', r.mget, *keys) + + for r in all_redis: + r.start() + +def test_mset_on_backend_down(): + all_redis[0].stop() + r = redis.Redis(nc.host(),nc.port()) + + assert_fail('Connection refused|Broken pipe',r.mset,default_kv) + + all_redis[1].stop() + assert_fail('Connection refused|Broken pipe',r.mset,default_kv) + + for r in all_redis: + r.start() + +def test_mget_pipeline(): + r = getconn() + + pipe = r.pipeline(transaction=False) + for k,v in default_kv.items(): + pipe.set(k,v) + keys = default_kv.keys() + pipe.mget(keys) + kv = {} + for i in range(large): + kv['kkk-%s' % i] = os.urandom(100) + for k,v in kv.items(): + pipe.set(k,v) + for k in kv.keys(): + pipe.get(k) + rst = pipe.execute() + + #print rst + #check the result + keys = default_kv.keys() + + #mget to check + vals = r.mget(keys) + for i, k in enumerate(keys): + assert(kv[k] == vals[i]) + + #del + assert (len(keys) == r.delete(*keys) ) + + #mget again + vals = r.mget(keys) + + for i, k in enumerate(keys): + assert(None == vals[i]) + +def test_multi_delete_normal(): + r = getconn() + + for i in range(100): + r.set('key-%s'%i, 'val-%s'%i) + for i in range(100): + assert_equal('val-%s'%i, r.get('key-%s'%i) ) + + keys = ['key-%s'%i for i in range(100)] + assert_equal(100, r.delete(*keys)) + + for i in range(100): + assert_equal(None, r.get('key-%s'%i) ) + +def test_multi_delete_on_readonly(): + all_redis[0].slaveof(all_redis[1].args['host'], all_redis[1].args['port']) + + r = redis.Redis(nc.host(), nc.port()) + + # got "READONLY You can't write against a read only slave" + assert_fail('READONLY|Invalid', r.delete, 'key-1') + assert_equal(0, r.delete('key-2')) + assert_fail('READONLY|Invalid', r.delete, 'key-3') + + keys = ['key-1', 'key-2', 'kkk-3'] + assert_fail('Invalid argument', r.delete, *keys) # got "Invalid argument" + +def test_multi_delete_on_backend_down(): + #one backend down + all_redis[0].stop() + r = redis.Redis(nc.host(), nc.port()) + + assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, 'key-1') + assert_equal(None, r.get('key-2')) + + keys = ['key-1', 'key-2', 'kkk-3'] + assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, *keys) + + #all backend down + all_redis[1].stop() + r = redis.Redis(nc.host(), nc.port()) + + assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, 'key-1') + assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, 'key-2') + + keys = ['key-1', 'key-2', 'kkk-3'] + assert_fail('Connection refused|reset by peer|Broken pipe', r.delete, *keys) + + for r in all_redis: + r.start() + + +def test_multi_delete_20140525(): + r = getconn() + + cnt = 126 + keys = ['key-%s'%i for i in range(cnt)] + pipe = r.pipeline(transaction=False) + pipe.mget(keys) + pipe.delete(*keys) + pipe.execute() + + diff -Nru nutcracker-0.4.0+dfsg/tests/test_redis/test_pipeline.py nutcracker-0.4.1+dfsg/tests/test_redis/test_pipeline.py --- nutcracker-0.4.0+dfsg/tests/test_redis/test_pipeline.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_redis/test_pipeline.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,58 @@ +#!/usr/bin/env python +#coding: utf-8 + +from common import * + +def test_pipeline(): + r = getconn() + + pipe = r.pipeline(transaction = False) + + pipe.set('a', 'a1').get('a').zadd('z', z1=1).zadd('z', z2=4) + pipe.zincrby('z', 'z1').zrange('z', 0, 5, withscores=True) + + assert pipe.execute() == \ + [ + True, + 'a1', + True, + True, + 2.0, + [('z1', 2.0), ('z2', 4)], + ] + +def test_invalid_pipeline(): + r = getconn() + + pipe = r.pipeline(transaction = False) + + pipe.set('a', 1).set('b', 2).lpush('a', 3).set('d', 4).get('a') + result = pipe.execute(raise_on_error = False) + + assert result[0] + assert result[1] + + # we can't lpush to a key that's a string value, so this should + # be a ResponseError exception + assert isinstance(result[2], redis.ResponseError) + + # since this isn't a transaction, the other commands after the + # error are still executed + assert result[3] + assert result[4] == '1' + + # make sure the pipe was restored to a working state + assert pipe.set('z', 'zzz').execute() == [True] + +def test_parse_error_raised(): + r = getconn() + + pipe = r.pipeline(transaction = False) + + # the zrem is invalid because we don't pass any keys to it + pipe.set('a', 1).zrem('b').set('b', 2) + result = pipe.execute(raise_on_error = False) + + assert result[0] + assert isinstance(result[1], redis.ResponseError) + assert result[2] diff -Nru nutcracker-0.4.0+dfsg/tests/test_redis/test_protocol.py nutcracker-0.4.1+dfsg/tests/test_redis/test_protocol.py --- nutcracker-0.4.0+dfsg/tests/test_redis/test_protocol.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_redis/test_protocol.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,90 @@ +#!/usr/bin/env python +from common import * +from pprint import pprint + +def get_conn(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((nc.host(), nc.port())) + s.settimeout(.3) + return s + +def _test(req, resp, sleep=0): + s = get_conn() + + for i in req: + s.sendall(i) + time.sleep(sleep) + + s.settimeout(.3) + + data = s.recv(10000) + assert(data == resp) + +def test_slow(): + req = '*1\r\n$4\r\nPING\r\n' + resp = '+PONG\r\n' + + if large > 1000: + sleep = 1 + else: + sleep = .1 + + _test(req, resp, sleep) + +def test_pingpong(): + req = '*1\r\n$4\r\nPING\r\n' + resp = '+PONG\r\n' + _test(req, resp) + +def test_quit(): + if nc.version() < '0.4.2': + return + req = '*1\r\n$4\r\nQUIT\r\n' + resp = '+OK\r\n' + _test(req, resp) + +def test_quit_without_recv(): + if nc.version() < '0.4.2': + return + req = '*1\r\n$4\r\nQUIT\r\n' + resp = '+OK\r\n' + s = get_conn() + + s.sendall(req) + s.close() + info = nc._info_dict() + #pprint(info) + assert(info['ntest']['client_err'] == 1) + +def _test_bad(req): + s = get_conn() + + s.sendall(req) + data = s.recv(10000) + print data + + assert('' == s.recv(1000)) # peer is closed + +def test_badreq(): + reqs = [ + # '*1\r\n$3\r\nPING\r\n', + '\r\n', + # '*3abcdefg\r\n', + '*3\r\n*abcde\r\n', + + '*4\r\n$4\r\nMSET\r\n$1\r\nA\r\n$1\r\nA\r\n$1\r\nA\r\n', + '*2\r\n$4\r\nMSET\r\n$1\r\nA\r\n', + # '*3\r\n$abcde\r\n', + # '*3\r\n$3abcde\r\n', + # '*3\r\n$3\r\nabcde\r\n', + ] + + for req in reqs: + _test_bad(req) + + +def test_wrong_argc(): + s = get_conn() + + s.sendall('*1\r\n$3\r\nGET\r\n') + assert('' == s.recv(1000)) # peer is closed diff -Nru nutcracker-0.4.0+dfsg/tests/test_system/test_reload.py nutcracker-0.4.1+dfsg/tests/test_system/test_reload.py --- nutcracker-0.4.0+dfsg/tests/test_system/test_reload.py 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/tests/test_system/test_reload.py 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,193 @@ +#!/usr/bin/env python +#coding: utf-8 +#file : test_reload.py +#author : ning +#date : 2014-09-03 12:28:16 + +import os +import sys +import redis + +PWD = os.path.dirname(os.path.realpath(__file__)) +WORKDIR = os.path.join(PWD,'../') +sys.path.append(os.path.join(WORKDIR,'lib/')) +sys.path.append(os.path.join(WORKDIR,'conf/')) + +import conf + +from server_modules import * +from utils import * +from nose import with_setup + +CLUSTER_NAME = 'ntest' +nc_verbose = int(getenv('T_VERBOSE', 5)) +mbuf = int(getenv('T_MBUF', 512)) +large = int(getenv('T_LARGE', 1000)) + +T_RELOAD_DELAY = 3 + 1 + +all_redis = [ + RedisServer('127.0.0.1', 2100, '/tmp/r/redis-2100/', CLUSTER_NAME, 'redis-2100'), + RedisServer('127.0.0.1', 2101, '/tmp/r/redis-2101/', CLUSTER_NAME, 'redis-2101'), + ] + +nc = NutCracker('127.0.0.1', 4100, '/tmp/r/nutcracker-4100', CLUSTER_NAME, + all_redis, mbuf=mbuf, verbose=nc_verbose) + +def _setup(): + print 'setup(mbuf=%s, verbose=%s)' %(mbuf, nc_verbose) + for r in all_redis + [nc]: + r.deploy() + r.stop() + r.start() + +def _teardown(): + for r in all_redis + [nc]: + assert(r._alive()) + r.stop() + +def get_tcp_conn(host, port): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((host, port)) + s.settimeout(.3) + return s + +def send_cmd(s, req, resp): + s.sendall(req) + data = s.recv(10000) + assert(data == resp) + +@with_setup(_setup, _teardown) +def test_reload_with_old_conf(): + if nc.version() < '0.4.2': + print 'Ignore test_reload for version %s' % nc.version() + return + pid = nc.pid() + # print 'old pid:', pid + r = redis.Redis(nc.host(), nc.port()) + r.set('k', 'v') + + conn = get_tcp_conn(nc.host(), nc.port()) + send_cmd(conn, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '$1\r\nv\r\n') + + # nc.reload() is same as nc.stop() and nc.start() + nc.reload() + time.sleep(.01) #it need time for the old process fork new process. + + # the old connection is still ok in T_RELOAD_DELAY seconds + send_cmd(conn, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '$1\r\nv\r\n') + + # conn2 should connect to new instance + conn2 = get_tcp_conn(nc.host(), nc.port()) + send_cmd(conn2, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '$1\r\nv\r\n') + + # the old connection is still ok in T_RELOAD_DELAY seconds + send_cmd(conn, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '$1\r\nv\r\n') + + time.sleep(T_RELOAD_DELAY) + assert(pid != nc.pid()) + + # assert the old connection is closed. + send_cmd(conn, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '') + + # conn2 should survive + send_cmd(conn2, '*2\r\n$3\r\nGET\r\n$1\r\nk\r\n', '$1\r\nv\r\n') + + r = redis.Redis(nc.host(), nc.port()) + rst = r.set('k', 'v') + assert(r.get('k') == 'v') + +@with_setup(_setup, _teardown) +def test_new_port(): + if nc.version() < '0.4.2': + print 'Ignore test_reload for version %s' % nc.version() + return + r = redis.Redis(nc.host(), nc.port()) + r.set('k', 'v') + + content = ''' +reload_test: + listen: 0.0.0.0:4101 + hash: fnv1a_64 + distribution: modula + redis: true + timeout: 400 + servers: + - 127.0.0.1:2100:1 redis-2100 + - 127.0.0.1:2101:1 redis-2101 +''' + + nc.set_config(content) + time.sleep(T_RELOAD_DELAY) + + r1 = redis.Redis(nc.host(), nc.port()) + r2 = redis.Redis(nc.host(), 4101) + + assert_fail('Connection refused', r1.get, 'k') + assert(r2.get('k') == 'v') + +@with_setup(_setup, _teardown) +def test_pool_add_del(): + if nc.version() < '0.4.2': + print 'Ignore test_reload for version %s' % nc.version() + return + + r = redis.Redis(nc.host(), nc.port()) + + r.set('k', 'v') + content = ''' +reload_test: + listen: 0.0.0.0:4100 + hash: fnv1a_64 + distribution: modula + redis: true + servers: + - 127.0.0.1:2100:1 redis-2100 + - 127.0.0.1:2101:1 redis-2101 + +reload_test2: + listen: 0.0.0.0:4101 + hash: fnv1a_64 + distribution: modula + redis: true + servers: + - 127.0.0.1:2100:1 redis-2100 + - 127.0.0.1:2101:1 redis-2101 +''' + + nc.set_config(content) + time.sleep(T_RELOAD_DELAY) + + r1 = redis.Redis(nc.host(), nc.port()) + r2 = redis.Redis(nc.host(), 4101) + + assert(r1.get('k') == 'v') + assert(r2.get('k') == 'v') + + content = ''' +reload_test: + listen: 0.0.0.0:4102 + hash: fnv1a_64 + distribution: modula + redis: true + preconnect: true + servers: + - 127.0.0.1:2100:1 redis-2100 + - 127.0.0.1:2101:1 redis-2101 +''' + nc.set_config(content) + time.sleep(T_RELOAD_DELAY) + pid = nc.pid() + print system('ls -l /proc/%s/fd/' % pid) + + r3 = redis.Redis(nc.host(), 4102) + + assert_fail('Connection refused', r1.get, 'k') + assert_fail('Connection refused', r2.get, 'k') + assert(r3.get('k') == 'v') + + fds = system('ls -l /proc/%s/fd/' % pid) + sockets = [s for s in fds.split('\n') if strstr(s, 'socket:') ] + # pool + stat + 2 backend + 1 client + assert(len(sockets) == 5) + diff -Nru nutcracker-0.4.0+dfsg/travis.sh nutcracker-0.4.1+dfsg/travis.sh --- nutcracker-0.4.0+dfsg/travis.sh 1970-01-01 00:00:00.000000000 +0000 +++ nutcracker-0.4.1+dfsg/travis.sh 2015-06-23 06:06:05.000000000 +0000 @@ -0,0 +1,28 @@ +#!/bin/bash +#file : travis.sh +#author : ning +#date : 2014-05-10 16:54:43 + +#install deps if we are in travis +if [ -n "$TRAVIS" ]; then + sudo apt-get install socat + + #python libs + sudo pip install redis + sudo pip install nose + + sudo pip install git+https://github.com/andymccurdy/redis-py.git@2.9.0 + sudo pip install git+https://github.com/idning/python-memcached.git#egg=memcache +fi + +#build twemproxy +CFLAGS="-ggdb3 -O0" autoreconf -fvi && ./configure --enable-debug=log && make + +ln -s `pwd`/src/nutcracker tests/_binaries/ +cp `which redis-server` tests/_binaries/ +cp `which redis-cli` tests/_binaries/ +cp `which memcached` tests/_binaries/ + +#run test +cd tests/ && nosetests --nologcapture -x -v + diff -Nru nutcracker-0.4.0+dfsg/.travis.yml nutcracker-0.4.1+dfsg/.travis.yml --- nutcracker-0.4.0+dfsg/.travis.yml 2014-10-21 00:25:20.000000000 +0000 +++ nutcracker-0.4.1+dfsg/.travis.yml 2015-06-23 06:06:05.000000000 +0000 @@ -1,3 +1,3 @@ language: c -script: CFLAGS="-ggdb3 -O0" autoreconf -fvi && ./configure --enable-debug=log && make && sudo make install +script: bash ./travis.sh