From 362032e43a11e946018839a9344a78b9b76ba5a3 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 May 2015 11:16:15 +0200 Subject: [PATCH 01/35] Release notes: Sentinel upgrade urgency moved to moderate --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 33adfac6..22d96190 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -12,7 +12,7 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. --[ Redis 3.0.1 ] Release date: 5 May 2015 -Upgrade urgency: LOW for Redis, Sentinel, Cluster. +Upgrade urgency: LOW for Redis and Cluster, MODERATE for Sentinel. * [FIX] Sentinel memory leak due to hiredis fixed. (Salvatore Sanfilippo) * [FIX] Sentinel memory leak on duplicated instance. (Charsyam) From e7422ef1662f676d6fdb12014b99b3f1bfc36acd Mon Sep 17 00:00:00 2001 From: Ryan Schwartz Date: Thu, 7 May 2015 14:28:21 -0500 Subject: [PATCH 02/35] Update 00-RELEASENOTES Fix typo in 00-RELEASENOTES. --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 22d96190..a32930a4 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -32,7 +32,7 @@ Upgrade urgency: LOW for Redis and Cluster, MODERATE for Sentinel. * WAIT command to block waiting for a write to be transmitted to the specified number of slaves. * MIGRATE connection caching. Much faster keys migraitons. -* MIGARTE new options COPY and REPLACE. +* MIGRATE new options COPY and REPLACE. * CLIENT PAUSE command: stop processing client requests for a specified amount of time. * BITCOUNT performance improvements. From 08d4df8d313ccd5147ac0ad2f92129b37cf44fa2 Mon Sep 17 00:00:00 2001 From: Jungtaek Lim Date: Tue, 12 May 2015 10:04:52 +0900 Subject: [PATCH 03/35] protocol error log should be seen debug/verbose level --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 5fb9e878..a358a467 100644 --- a/src/networking.c +++ b/src/networking.c @@ -974,7 +974,7 @@ int processInlineBuffer(redisClient *c) { /* Helper function. Trims query buffer to make the function that processes * multi bulk requests idempotent. */ static void setProtocolError(redisClient *c, int pos) { - if (server.verbosity >= REDIS_VERBOSE) { + if (server.verbosity <= REDIS_VERBOSE) { sds client = catClientInfoString(sdsempty(),c); redisLog(REDIS_VERBOSE, "Protocol error from client: %s", client); From 3a9f41ad867059869c3d608c776234a909bd4045 Mon Sep 17 00:00:00 2001 From: Glenn Nethercutt Date: Fri, 17 Apr 2015 09:27:54 -0400 Subject: [PATCH 04/35] uphold the smove contract to return 0 when the element is not a member of the source set, even if source=dest --- src/t_set.c | 5 ++++- tests/unit/type/set.tcl | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/t_set.c b/src/t_set.c index c530d692..90819924 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -321,7 +321,10 @@ void smoveCommand(redisClient *c) { /* If srcset and dstset are equal, SMOVE is a no-op */ if (srcset == dstset) { - addReply(c,shared.cone); + if (setTypeIsMember(srcset,ele)) + addReply(c,shared.cone); + else + addReply(c,shared.czero); return; } diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index 162de0af..f294c01f 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -450,6 +450,7 @@ start_server { test "SMOVE non existing key" { setup_move assert_equal 0 [r smove myset1 myset2 foo] + assert_equal 0 [r smove myset1 myset1 foo] assert_equal {1 a b} [lsort [r smembers myset1]] assert_equal {2 3 4} [lsort [r smembers myset2]] } From 138e7af57bc5fa9a9276dbe89e8a03e303dee3e2 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 15 May 2015 17:38:48 +0200 Subject: [PATCH 05/35] Rewrite smoveCommand test with ternary operator --- src/t_set.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/t_set.c b/src/t_set.c index 90819924..64d43f7f 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -321,10 +321,7 @@ void smoveCommand(redisClient *c) { /* If srcset and dstset are equal, SMOVE is a no-op */ if (srcset == dstset) { - if (setTypeIsMember(srcset,ele)) - addReply(c,shared.cone); - else - addReply(c,shared.czero); + addReply(c,setTypeIsMember(srcset,ele) ? shared.cone : shared.czero); return; } From 5844f5d0d1d9e54848330dc9c4b7b189973ebaba Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 May 2015 12:52:03 +0200 Subject: [PATCH 06/35] Sentinel: SENTINEL CKQUORUM command A way for monitoring systems to check that Sentinel is technically able to reach the quorum and failover, using the currently visible Sentinels. --- src/sentinel.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 85ae7c37..d295e16f 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2614,6 +2614,31 @@ sentinelRedisInstance *sentinelGetMasterByNameOrReplyError(redisClient *c, return ri; } +#define SENTINEL_ISQR_OK 0 +#define SENTINEL_ISQR_NOQUORUM (1<<0) +#define SENTINEL_ISQR_NOAUTH (1<<1) +int sentinelIsQuorumReachable(sentinelRedisInstance *master, int *usableptr) { + dictIterator *di; + dictEntry *de; + int usable = 1; /* Number of usable Sentinels. Init to 1 to count myself. */ + int result = SENTINEL_ISQR_OK; + int voters = dictSize(master->sentinels)+1; /* Known Sentinels + myself. */ + + di = dictGetIterator(master->sentinels); + while((de = dictNext(di)) != NULL) { + sentinelRedisInstance *ri = dictGetVal(de); + + if (ri->flags & (SRI_S_DOWN|SRI_O_DOWN)) continue; + usable++; + } + dictReleaseIterator(di); + + if (usable < (int)master->quorum) result |= SENTINEL_ISQR_NOQUORUM; + if (usable < voters/2+1) result |= SENTINEL_ISQR_NOAUTH; + if (usableptr) *usableptr = usable; + return result; +} + void sentinelCommand(redisClient *c) { if (!strcasecmp(c->argv[1]->ptr,"masters")) { /* SENTINEL MASTERS */ @@ -2774,6 +2799,32 @@ void sentinelCommand(redisClient *c) { dictDelete(sentinel.masters,c->argv[2]->ptr); sentinelFlushConfig(); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"ckquorum")) { + /* SENTINEL CKQUORUM */ + sentinelRedisInstance *ri; + int usable; + + if ((ri = sentinelGetMasterByNameOrReplyError(c,c->argv[2])) + == NULL) return; + int result = sentinelIsQuorumReachable(ri,&usable); + if (result == SENTINEL_ISQR_OK) { + addReplySds(c, sdscatfmt(sdsempty(), + "+OK %i usable Sentinels. Quorum and failover authorization " + "can be reached\r\n",usable)); + } else { + sds e = sdscatfmt(sdsempty(), + "-NOQUORUM %i usable Sentinels. ",usable); + if (result & SENTINEL_ISQR_NOQUORUM) + e = sdscat(e,"Not enough available Sentinels to reach the" + " specified quorum for this master"); + if (result & SENTINEL_ISQR_NOAUTH) { + if (result & SENTINEL_ISQR_NOQUORUM) e = sdscat(e,". "); + e = sdscat(e, "Not enough available Sentinels to reach the" + " majority and authorize a failover"); + } + e = sdscat(e,"\r\n"); + addReplySds(c,e); + } } else if (!strcasecmp(c->argv[1]->ptr,"set")) { if (c->argc < 3 || c->argc % 2 == 0) goto numargserr; sentinelSetCommand(c); From 9c0a68861e2d23f34ce4d88f38c87ac6025bdcdd Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 May 2015 12:26:09 +0200 Subject: [PATCH 07/35] Sentinel: CKQUORUM tests --- tests/sentinel/tests/06-ckquorum.tcl | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/sentinel/tests/06-ckquorum.tcl diff --git a/tests/sentinel/tests/06-ckquorum.tcl b/tests/sentinel/tests/06-ckquorum.tcl new file mode 100644 index 00000000..31e5fa2f --- /dev/null +++ b/tests/sentinel/tests/06-ckquorum.tcl @@ -0,0 +1,34 @@ +# Test for the SENTINEL CKQUORUM command + +source "../tests/includes/init-tests.tcl" +set num_sentinels [llength $::sentinel_instances] + +test "CKQUORUM reports OK and the right amount of Sentinels" { + foreach_sentinel_id id { + assert_match "*OK $num_sentinels usable*" [S $id SENTINEL CKQUORUM mymaster] + } +} + +test "CKQUORUM detects quorum cannot be reached" { + set orig_quorum [expr {$num_sentinels/2+1}] + S 0 SENTINEL SET mymaster quorum [expr {$num_sentinels+1}] + catch {[S 0 SENTINEL CKQUORUM mymaster]} err + assert_match "*NOQUORUM*" $err + S 0 SENTINEL SET mymaster quorum $orig_quorum +} + +test "CKQUORUM detects failover authorization cannot be reached" { + set orig_quorum [expr {$num_sentinels/2+1}] + S 0 SENTINEL SET mymaster quorum 1 + kill_instance sentinel 1 + kill_instance sentinel 2 + kill_instance sentinel 3 + after 5000 + catch {[S 0 SENTINEL CKQUORUM mymaster]} err + assert_match "*NOQUORUM*" $err + S 0 SENTINEL SET mymaster quorum $orig_quorum + restart_instance sentinel 1 + restart_instance sentinel 2 + restart_instance sentinel 3 +} + From e252e9f231988b19fcb4baa8bf572525b1f7b14d Mon Sep 17 00:00:00 2001 From: therealbill Date: Mon, 11 May 2015 14:08:57 -0500 Subject: [PATCH 08/35] adding a sentinel command: "flushconfig" This new command triggers a config flush to save the in-memory config to disk. This is useful for cases of a configuration management system or a package manager wiping out your sentinel config while the process is still running - and has not yet been restarted. It can also be useful for scripting a backup and migrate or clone of a running sentinel. --- src/sentinel.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index d295e16f..3e8dfcdd 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2789,6 +2789,10 @@ void sentinelCommand(redisClient *c) { sentinelEvent(REDIS_WARNING,"+monitor",ri,"%@ quorum %d",ri->quorum); addReply(c,shared.ok); } + } else if (!strcasecmp(c->argv[1]->ptr,"flushconfig")) { + sentinelFlushConfig(); + addReply(c,shared.ok); + return; } else if (!strcasecmp(c->argv[1]->ptr,"remove")) { /* SENTINEL REMOVE */ sentinelRedisInstance *ri; From ff7c1faa1297648522de9adc2df516bea78eeff4 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 May 2015 18:06:16 +0200 Subject: [PATCH 09/35] ZADD implemenation able to take options. --- src/t_zset.c | 57 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index d3c7214b..0d3e7dd3 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1171,27 +1171,65 @@ void zsetConvert(robj *zobj, int encoding) { *----------------------------------------------------------------------------*/ /* This generic command implements both ZADD and ZINCRBY. */ -void zaddGenericCommand(redisClient *c, int incr) { +#define ZADD_NONE 0 +#define ZADD_INCR (1<<0) /* Increment the score instead of setting it. */ +#define ZADD_NX (1<<1) /* Don't touch elements not already existing. */ +#define ZADD_XX (1<<2) /* Only touch elements already exisitng. */ +void zaddGenericCommand(redisClient *c, int flags) { static char *nanerr = "resulting score is not a number (NaN)"; robj *key = c->argv[1]; robj *ele; robj *zobj; robj *curobj; double score = 0, *scores = NULL, curscore = 0.0; - int j, elements = (c->argc-2)/2; - int added = 0, updated = 0; + int j, elements; + int added = 0, updated = 0, scoreidx = 0; - if (c->argc % 2) { + /* Parse options. At the end 'scoreidx' is set to the argument position + * of the score of the first score-element pair. */ + scoreidx = 2; + while(scoreidx < c->argc) { + char *opt = c->argv[scoreidx]->ptr; + if (!strcasecmp(opt,"nx")) flags |= ZADD_NX; + else if (!strcasecmp(opt,"xx")) flags |= ZADD_XX; + else if (!strcasecmp(opt,"incr")) flags |= ZADD_INCR; + else break; + scoreidx++; + } + + /* Turn options into simple to check vars. */ + int incr = (flags & ZADD_INCR) != 0; + int nx = (flags & ZADD_NX) != 0; + int xx = (flags & ZADD_XX) != 0; + + /* After the options, we expect to have an even number of args, since + * we expect any number of score-element pairs. */ + elements = c->argc-scoreidx; + if (elements % 2) { addReply(c,shared.syntaxerr); return; } + elements /= 2; /* Now this holds the number of score-element pairs. */ + + /* Check for incompatible options. */ + if (nx && xx) { + addReplyError(c, + "XX and NX options at the same time are not compatible"); + return; + } + + if (incr && elements > 1) { + addReplyError(c, + "INCR option supports a single increment-element pair"); + return; + } /* Start parsing all the scores, we need to emit any syntax error * before executing additions to the sorted set, as the command should * either execute fully or nothing at all. */ scores = zmalloc(sizeof(double)*elements); for (j = 0; j < elements; j++) { - if (getDoubleFromObjectOrReply(c,c->argv[2+j*2],&scores[j],NULL) + if (getDoubleFromObjectOrReply(c,c->argv[scoreidx+j*2],&scores[j],NULL) != REDIS_OK) goto cleanup; } @@ -1220,7 +1258,7 @@ void zaddGenericCommand(redisClient *c, int incr) { unsigned char *eptr; /* Prefer non-encoded element when dealing with ziplists. */ - ele = c->argv[3+j*2]; + ele = c->argv[scoreidx+1+j*2]; if ((eptr = zzlFind(zobj->ptr,ele,&curscore)) != NULL) { if (incr) { score += curscore; @@ -1253,7 +1291,8 @@ void zaddGenericCommand(redisClient *c, int incr) { zskiplistNode *znode; dictEntry *de; - ele = c->argv[3+j*2] = tryObjectEncoding(c->argv[3+j*2]); + ele = c->argv[scoreidx+1+j*2] = + tryObjectEncoding(c->argv[scoreidx+1+j*2]); de = dictFind(zs->dict,ele); if (de != NULL) { curobj = dictGetKey(de); @@ -1307,11 +1346,11 @@ cleanup: } void zaddCommand(redisClient *c) { - zaddGenericCommand(c,0); + zaddGenericCommand(c,ZADD_NONE); } void zincrbyCommand(redisClient *c) { - zaddGenericCommand(c,1); + zaddGenericCommand(c,ZADD_INCR); } void zremCommand(redisClient *c) { From fa17e2daf08e4d4107f87db24a6ac0d93d197b9f Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 29 May 2015 09:59:42 +0200 Subject: [PATCH 10/35] ZADD NX and XX options --- src/t_zset.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index 0d3e7dd3..64355eb1 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1183,7 +1183,14 @@ void zaddGenericCommand(redisClient *c, int flags) { robj *curobj; double score = 0, *scores = NULL, curscore = 0.0; int j, elements; - int added = 0, updated = 0, scoreidx = 0; + int scoreidx = 0; + /* The following vars are used in order to track what the command actually + * did during the execution, to reply to the client and to trigger the + * notification of keyspace change. */ + int added = 0; /* Number of new elements added. */ + int updated = 0; /* Number of elements with updated score. */ + int processed = 0; /* Number of elements processed, may remain zero with + options like XX. */ /* Parse options. At the end 'scoreidx' is set to the argument position * of the score of the first score-element pair. */ @@ -1236,6 +1243,7 @@ void zaddGenericCommand(redisClient *c, int flags) { /* Lookup the key and create the sorted set if does not exist. */ zobj = lookupKeyWrite(c->db,key); if (zobj == NULL) { + if (xx) goto reply_to_client; /* No key + XX option: nothing to do. */ if (server.zset_max_ziplist_entries == 0 || server.zset_max_ziplist_value < sdslen(c->argv[3]->ptr)) { @@ -1260,6 +1268,7 @@ void zaddGenericCommand(redisClient *c, int flags) { /* Prefer non-encoded element when dealing with ziplists. */ ele = c->argv[scoreidx+1+j*2]; if ((eptr = zzlFind(zobj->ptr,ele,&curscore)) != NULL) { + if (nx) continue; if (incr) { score += curscore; if (isnan(score)) { @@ -1275,7 +1284,8 @@ void zaddGenericCommand(redisClient *c, int flags) { server.dirty++; updated++; } - } else { + processed++; + } else if (!xx) { /* Optimize: check if the element is too large or the list * becomes too long *before* executing zzlInsert. */ zobj->ptr = zzlInsert(zobj->ptr,ele,score); @@ -1285,6 +1295,7 @@ void zaddGenericCommand(redisClient *c, int flags) { zsetConvert(zobj,REDIS_ENCODING_SKIPLIST); server.dirty++; added++; + processed++; } } else if (zobj->encoding == REDIS_ENCODING_SKIPLIST) { zset *zs = zobj->ptr; @@ -1295,6 +1306,7 @@ void zaddGenericCommand(redisClient *c, int flags) { tryObjectEncoding(c->argv[scoreidx+1+j*2]); de = dictFind(zs->dict,ele); if (de != NULL) { + if (nx) continue; curobj = dictGetKey(de); curscore = *(double*)dictGetVal(de); @@ -1319,22 +1331,30 @@ void zaddGenericCommand(redisClient *c, int flags) { server.dirty++; updated++; } - } else { + processed++; + } else if (!xx) { znode = zslInsert(zs->zsl,score,ele); incrRefCount(ele); /* Inserted in skiplist. */ redisAssertWithInfo(c,NULL,dictAdd(zs->dict,ele,&znode->score) == DICT_OK); incrRefCount(ele); /* Added to dictionary. */ server.dirty++; added++; + processed++; } } else { redisPanic("Unknown sorted set encoding"); } } - if (incr) /* ZINCRBY */ - addReplyDouble(c,score); - else /* ZADD */ + +reply_to_client: + if (incr) { /* ZINCRBY or INCR option. */ + if (processed) + addReplyDouble(c,score); + else + addReply(c,shared.nullbulk); + } else { /* ZADD. */ addReplyLongLong(c,added); + } cleanup: zfree(scores); From a13d6378c1b53d7adc631de4293334a0092452c2 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 29 May 2015 11:22:03 +0200 Subject: [PATCH 11/35] ZADD RETCH option: Return number of elements added or updated Normally ZADD only returns the number of elements added to a sorted set, using the RETCH option it returns the sum of elements added or for which the score was updated. --- src/t_zset.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/t_zset.c b/src/t_zset.c index 64355eb1..50874e13 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1175,6 +1175,7 @@ void zsetConvert(robj *zobj, int encoding) { #define ZADD_INCR (1<<0) /* Increment the score instead of setting it. */ #define ZADD_NX (1<<1) /* Don't touch elements not already existing. */ #define ZADD_XX (1<<2) /* Only touch elements already exisitng. */ +#define ZADD_RETCH (1<<3) /* Return the number of elements added or updated.*/ void zaddGenericCommand(redisClient *c, int flags) { static char *nanerr = "resulting score is not a number (NaN)"; robj *key = c->argv[1]; @@ -1199,6 +1200,7 @@ void zaddGenericCommand(redisClient *c, int flags) { char *opt = c->argv[scoreidx]->ptr; if (!strcasecmp(opt,"nx")) flags |= ZADD_NX; else if (!strcasecmp(opt,"xx")) flags |= ZADD_XX; + else if (!strcasecmp(opt,"retch")) flags |= ZADD_RETCH; else if (!strcasecmp(opt,"incr")) flags |= ZADD_INCR; else break; scoreidx++; @@ -1208,6 +1210,7 @@ void zaddGenericCommand(redisClient *c, int flags) { int incr = (flags & ZADD_INCR) != 0; int nx = (flags & ZADD_NX) != 0; int xx = (flags & ZADD_XX) != 0; + int retch = (flags & ZADD_RETCH) != 0; /* After the options, we expect to have an even number of args, since * we expect any number of score-element pairs. */ @@ -1353,7 +1356,7 @@ reply_to_client: else addReply(c,shared.nullbulk); } else { /* ZADD. */ - addReplyLongLong(c,added); + addReplyLongLong(c,retch ? added+updated : added); } cleanup: From 9003483d43f3dfa1bd96659a70cf7cd63857b838 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 29 May 2015 11:23:49 +0200 Subject: [PATCH 12/35] Test: ZADD NX and XX options tests --- tests/unit/type/zset.tcl | 59 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 238eebb9..d33b4bb3 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -43,6 +43,63 @@ start_server {tags {"zset"}} { assert_error "*not*float*" {r zadd myzset nan abc} } + test "ZADD with options syntax error with incomplete pair" { + r del ztmp + catch {r zadd ztmp xx 10 x 20} err + set err + } {ERR*} + + test "ZADD XX option without key - $encoding" { + r del ztmp + assert {[r zadd ztmp xx 10 x] == 0} + assert {[r type ztmp] eq {none}} + } + + test "ZADD XX existing key - $encoding" { + r del ztmp + r zadd ztmp 10 x + assert {[r zadd ztmp xx 20 y] == 0} + assert {[r zcard ztmp] == 1} + } + + test "ZADD XX returns the number of elements actually added" { + r del ztmp + r zadd ztmp 10 x + set retval [r zadd ztmp 10 x 20 y 30 z] + assert {$retval == 2} + } + + test "ZADD XX updates existing elements score" { + r del ztmp + r zadd ztmp 10 x 20 y 30 z + r zadd ztmp xx 5 foo 11 x 21 y 40 zap + assert {[r zcard ztmp] == 3} + assert {[r zscore ztmp x] == 11} + assert {[r zscore ztmp y] == 21} + } + + test "ZADD XX and NX are not compatible" { + r del ztmp + catch {r zadd ztmp xx nx 10 x} err + set err + } {ERR*} + + test "ZADD NX with non exisitng key" { + r del ztmp + r zadd ztmp nx 10 x 20 y 30 z + assert {[r zcard ztmp] == 3} + } + + test "ZADD NX only add new elements without updating old ones" { + r del ztmp + r zadd ztmp 10 x 20 y 30 z + assert {[r zadd ztmp nx 11 x 21 y 100 a 200 b] == 2} + assert {[r zscore ztmp x] == 10} + assert {[r zscore ztmp y] == 20} + assert {[r zscore ztmp a] == 100} + assert {[r zscore ztmp b] == 200} + } + test "ZINCRBY calls leading to NaN result in error" { r zincrby myzset +inf abc assert_error "*NaN*" {r zincrby myzset -inf abc} @@ -77,6 +134,8 @@ start_server {tags {"zset"}} { } test "ZCARD basics - $encoding" { + r del ztmp + r zadd ztmp 10 a 20 b 30 c assert_equal 3 [r zcard ztmp] assert_equal 0 [r zcard zdoesntexist] } From df7add9e70daebdeefc6743d1de35ae918c2a947 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 29 May 2015 11:28:49 +0200 Subject: [PATCH 13/35] Test: ZADD INCR test --- tests/unit/type/zset.tcl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index d33b4bb3..ea2d4e37 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -100,6 +100,20 @@ start_server {tags {"zset"}} { assert {[r zscore ztmp b] == 200} } + test "ZADD INCR works like ZINCRBY" { + r del ztmp + r zadd ztmp 10 x 20 y 30 z + r zadd ztmp INCR 15 x + assert {[r zscore ztmp x] == 25} + } + + test "ZADD INCR works with a single score-elemenet pair" { + r del ztmp + r zadd ztmp 10 x 20 y 30 z + catch {r zadd ztmp INCR 15 x 10 y} err + set err + } {ERR*} + test "ZINCRBY calls leading to NaN result in error" { r zincrby myzset +inf abc assert_error "*NaN*" {r zincrby myzset -inf abc} From 1d8973c47df9c58cdca08951b5ee119fb5288d35 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 29 May 2015 11:32:22 +0200 Subject: [PATCH 14/35] ZADD RETCH option renamed CH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From Twitter: "@antirez that’s an awfully-named command :( http://en.wikipedia.org/wiki/Retching" --- src/t_zset.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index 50874e13..24f8d309 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1175,7 +1175,7 @@ void zsetConvert(robj *zobj, int encoding) { #define ZADD_INCR (1<<0) /* Increment the score instead of setting it. */ #define ZADD_NX (1<<1) /* Don't touch elements not already existing. */ #define ZADD_XX (1<<2) /* Only touch elements already exisitng. */ -#define ZADD_RETCH (1<<3) /* Return the number of elements added or updated.*/ +#define ZADD_CH (1<<3) /* Return num of elements added or updated. */ void zaddGenericCommand(redisClient *c, int flags) { static char *nanerr = "resulting score is not a number (NaN)"; robj *key = c->argv[1]; @@ -1200,7 +1200,7 @@ void zaddGenericCommand(redisClient *c, int flags) { char *opt = c->argv[scoreidx]->ptr; if (!strcasecmp(opt,"nx")) flags |= ZADD_NX; else if (!strcasecmp(opt,"xx")) flags |= ZADD_XX; - else if (!strcasecmp(opt,"retch")) flags |= ZADD_RETCH; + else if (!strcasecmp(opt,"ch")) flags |= ZADD_CH; else if (!strcasecmp(opt,"incr")) flags |= ZADD_INCR; else break; scoreidx++; @@ -1210,7 +1210,7 @@ void zaddGenericCommand(redisClient *c, int flags) { int incr = (flags & ZADD_INCR) != 0; int nx = (flags & ZADD_NX) != 0; int xx = (flags & ZADD_XX) != 0; - int retch = (flags & ZADD_RETCH) != 0; + int ch = (flags & ZADD_CH) != 0; /* After the options, we expect to have an even number of args, since * we expect any number of score-element pairs. */ @@ -1356,7 +1356,7 @@ reply_to_client: else addReply(c,shared.nullbulk); } else { /* ZADD. */ - addReplyLongLong(c,retch ? added+updated : added); + addReplyLongLong(c,ch ? added+updated : added); } cleanup: From 0da453160bfef1b69f9e035363e15f8c56cf1e71 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 29 May 2015 11:34:43 +0200 Subject: [PATCH 15/35] Test: ZADD CH tests --- tests/unit/type/zset.tcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index ea2d4e37..cc560143 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -114,6 +114,13 @@ start_server {tags {"zset"}} { set err } {ERR*} + test "ZADD CH option changes return value to all changed elements" { + r del ztmp + r zadd ztmp 10 x 20 y 30 z + assert {[r zadd ztmp 11 x 21 y 30 z] == 0} + assert {[r zadd ztmp ch 12 x 22 y 30 z] == 2} + } + test "ZINCRBY calls leading to NaN result in error" { r zincrby myzset +inf abc assert_error "*NaN*" {r zincrby myzset -inf abc} From e0b2e24830a66485efbb577520e1f15bcea5f7fa Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Thu, 21 May 2015 13:24:51 +0300 Subject: [PATCH 16/35] Removed incorrect suggestion DEL/INCR/DECR and others could be NTH but apparently never made it to the implementation of SORT --- src/sort.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sort.c b/src/sort.c index 2b327644..8516fdc7 100644 --- a/src/sort.c +++ b/src/sort.c @@ -209,7 +209,7 @@ void sortCommand(redisClient *c) { } /* Create a list of operations to perform for every sorted element. - * Operations can be GET/DEL/INCR/DECR */ + * Operations can be GET */ operations = listCreate(); listSetFreeMethod(operations,zfree); j = 2; /* options start at argv[2] */ From 702f9147718a32f9cfc2158d9948a84d17cc2e7c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 3 Jun 2015 08:44:43 +0200 Subject: [PATCH 17/35] Scripting: Lua cmsgpack lib updated to include str8 support --- deps/lua/src/lua_cmsgpack.c | 71 ++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/deps/lua/src/lua_cmsgpack.c b/deps/lua/src/lua_cmsgpack.c index e13f053d..0b82d008 100644 --- a/deps/lua/src/lua_cmsgpack.c +++ b/deps/lua/src/lua_cmsgpack.c @@ -66,7 +66,7 @@ /* Reverse memory bytes if arch is little endian. Given the conceptual * simplicity of the Lua build system we prefer check for endianess at runtime. * The performance difference should be acceptable. */ -static void memrevifle(void *ptr, size_t len) { +void memrevifle(void *ptr, size_t len) { unsigned char *p = (unsigned char *)ptr, *e = (unsigned char *)p+len-1, aux; @@ -96,7 +96,7 @@ typedef struct mp_buf { size_t len, free; } mp_buf; -static void *mp_realloc(lua_State *L, void *target, size_t osize,size_t nsize) { +void *mp_realloc(lua_State *L, void *target, size_t osize,size_t nsize) { void *(*local_realloc) (void *, void *, size_t osize, size_t nsize) = NULL; void *ud; @@ -105,7 +105,7 @@ static void *mp_realloc(lua_State *L, void *target, size_t osize,size_t nsize) { return local_realloc(ud, target, osize, nsize); } -static mp_buf *mp_buf_new(lua_State *L) { +mp_buf *mp_buf_new(lua_State *L) { mp_buf *buf = NULL; /* Old size = 0; new size = sizeof(*buf) */ @@ -117,7 +117,7 @@ static mp_buf *mp_buf_new(lua_State *L) { return buf; } -static void mp_buf_append(mp_buf *buf, const unsigned char *s, size_t len) { +void mp_buf_append(mp_buf *buf, const unsigned char *s, size_t len) { if (buf->free < len) { size_t newlen = buf->len+len; @@ -153,7 +153,7 @@ typedef struct mp_cur { int err; } mp_cur; -static void mp_cur_init(mp_cur *cursor, const unsigned char *s, size_t len) { +void mp_cur_init(mp_cur *cursor, const unsigned char *s, size_t len) { cursor->p = s; cursor->left = len; cursor->err = MP_CUR_ERROR_NONE; @@ -173,13 +173,17 @@ static void mp_cur_init(mp_cur *cursor, const unsigned char *s, size_t len) { /* ------------------------- Low level MP encoding -------------------------- */ -static void mp_encode_bytes(mp_buf *buf, const unsigned char *s, size_t len) { +void mp_encode_bytes(mp_buf *buf, const unsigned char *s, size_t len) { unsigned char hdr[5]; int hdrlen; if (len < 32) { hdr[0] = 0xa0 | (len&0xff); /* fix raw */ hdrlen = 1; + } else if (len <= 0xff) { + hdr[0] = 0xd9; + hdr[1] = len; + hdrlen = 2; } else if (len <= 0xffff) { hdr[0] = 0xda; hdr[1] = (len&0xff00)>>8; @@ -198,7 +202,7 @@ static void mp_encode_bytes(mp_buf *buf, const unsigned char *s, size_t len) { } /* we assume IEEE 754 internal format for single and double precision floats. */ -static void mp_encode_double(mp_buf *buf, double d) { +void mp_encode_double(mp_buf *buf, double d) { unsigned char b[9]; float f = d; @@ -216,7 +220,7 @@ static void mp_encode_double(mp_buf *buf, double d) { } } -static void mp_encode_int(mp_buf *buf, int64_t n) { +void mp_encode_int(mp_buf *buf, int64_t n) { unsigned char b[9]; int enclen; @@ -288,7 +292,7 @@ static void mp_encode_int(mp_buf *buf, int64_t n) { mp_buf_append(buf,b,enclen); } -static void mp_encode_array(mp_buf *buf, int64_t n) { +void mp_encode_array(mp_buf *buf, int64_t n) { unsigned char b[5]; int enclen; @@ -311,7 +315,7 @@ static void mp_encode_array(mp_buf *buf, int64_t n) { mp_buf_append(buf,b,enclen); } -static void mp_encode_map(mp_buf *buf, int64_t n) { +void mp_encode_map(mp_buf *buf, int64_t n) { unsigned char b[5]; int enclen; @@ -336,7 +340,7 @@ static void mp_encode_map(mp_buf *buf, int64_t n) { /* --------------------------- Lua types encoding --------------------------- */ -static void mp_encode_lua_string(lua_State *L, mp_buf *buf) { +void mp_encode_lua_string(lua_State *L, mp_buf *buf) { size_t len; const char *s; @@ -344,13 +348,13 @@ static void mp_encode_lua_string(lua_State *L, mp_buf *buf) { mp_encode_bytes(buf,(const unsigned char*)s,len); } -static void mp_encode_lua_bool(lua_State *L, mp_buf *buf) { +void mp_encode_lua_bool(lua_State *L, mp_buf *buf) { unsigned char b = lua_toboolean(L,-1) ? 0xc3 : 0xc2; mp_buf_append(buf,&b,1); } /* Lua 5.3 has a built in 64-bit integer type */ -static void mp_encode_lua_integer(lua_State *L, mp_buf *buf) { +void mp_encode_lua_integer(lua_State *L, mp_buf *buf) { #if (LUA_VERSION_NUM < 503) && BITS_32 lua_Number i = lua_tonumber(L,-1); #else @@ -362,7 +366,7 @@ static void mp_encode_lua_integer(lua_State *L, mp_buf *buf) { /* Lua 5.2 and lower only has 64-bit doubles, so we need to * detect if the double may be representable as an int * for Lua < 5.3 */ -static void mp_encode_lua_number(lua_State *L, mp_buf *buf) { +void mp_encode_lua_number(lua_State *L, mp_buf *buf) { lua_Number n = lua_tonumber(L,-1); if (IS_INT64_EQUIVALENT(n)) { @@ -372,10 +376,10 @@ static void mp_encode_lua_number(lua_State *L, mp_buf *buf) { } } -static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level); +void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level); /* Convert a lua table into a message pack list. */ -static void mp_encode_lua_table_as_array(lua_State *L, mp_buf *buf, int level) { +void mp_encode_lua_table_as_array(lua_State *L, mp_buf *buf, int level) { #if LUA_VERSION_NUM < 502 size_t len = lua_objlen(L,-1), j; #else @@ -391,7 +395,7 @@ static void mp_encode_lua_table_as_array(lua_State *L, mp_buf *buf, int level) { } /* Convert a lua table into a message pack key-value map. */ -static void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { +void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { size_t len = 0; /* First step: count keys into table. No other way to do it with the @@ -418,7 +422,7 @@ static void mp_encode_lua_table_as_map(lua_State *L, mp_buf *buf, int level) { /* Returns true if the Lua table on top of the stack is exclusively composed * of keys from numerical keys from 1 up to N, with N being the total number * of elements, without any hole in the middle. */ -static int table_is_an_array(lua_State *L) { +int table_is_an_array(lua_State *L) { int count = 0, max = 0; #if LUA_VERSION_NUM < 503 lua_Number n; @@ -461,14 +465,14 @@ static int table_is_an_array(lua_State *L) { /* If the length operator returns non-zero, that is, there is at least * an object at key '1', we serialize to message pack list. Otherwise * we use a map. */ -static void mp_encode_lua_table(lua_State *L, mp_buf *buf, int level) { +void mp_encode_lua_table(lua_State *L, mp_buf *buf, int level) { if (table_is_an_array(L)) mp_encode_lua_table_as_array(L,buf,level); else mp_encode_lua_table_as_map(L,buf,level); } -static void mp_encode_lua_null(lua_State *L, mp_buf *buf) { +void mp_encode_lua_null(lua_State *L, mp_buf *buf) { unsigned char b[1]; (void)L; @@ -476,7 +480,7 @@ static void mp_encode_lua_null(lua_State *L, mp_buf *buf) { mp_buf_append(buf,b,1); } -static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { +void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { int t = lua_type(L,-1); /* Limit the encoding of nested tables to a specified maximum depth, so that @@ -506,7 +510,7 @@ static void mp_encode_lua_type(lua_State *L, mp_buf *buf, int level) { * Packs all arguments as a stream for multiple upacking later. * Returns error if no arguments provided. */ -static int mp_pack(lua_State *L) { +int mp_pack(lua_State *L) { int nargs = lua_gettop(L); int i; mp_buf *buf; @@ -687,6 +691,15 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { mp_cur_consume(c,9); } break; + case 0xd9: /* raw 8 */ + mp_cur_need(c,2); + { + size_t l = c->p[1]; + mp_cur_need(c,2+l); + lua_pushlstring(L,(char*)c->p+2,l); + mp_cur_consume(c,2+l); + } + break; case 0xda: /* raw 16 */ mp_cur_need(c,3); { @@ -773,7 +786,7 @@ void mp_decode_to_lua_type(lua_State *L, mp_cur *c) { } } -static int mp_unpack_full(lua_State *L, int limit, int offset) { +int mp_unpack_full(lua_State *L, int limit, int offset) { size_t len; const char *s; mp_cur c; @@ -826,18 +839,18 @@ static int mp_unpack_full(lua_State *L, int limit, int offset) { return cnt; } -static int mp_unpack(lua_State *L) { +int mp_unpack(lua_State *L) { return mp_unpack_full(L, 0, 0); } -static int mp_unpack_one(lua_State *L) { +int mp_unpack_one(lua_State *L) { int offset = luaL_optinteger(L, 2, 0); /* Variable pop because offset may not exist */ lua_pop(L, lua_gettop(L)-1); return mp_unpack_full(L, 1, offset); } -static int mp_unpack_limit(lua_State *L) { +int mp_unpack_limit(lua_State *L) { int limit = luaL_checkinteger(L, 2); int offset = luaL_optinteger(L, 3, 0); /* Variable pop because offset may not exist */ @@ -846,7 +859,7 @@ static int mp_unpack_limit(lua_State *L) { return mp_unpack_full(L, limit, offset); } -static int mp_safe(lua_State *L) { +int mp_safe(lua_State *L) { int argc, err, total_results; argc = lua_gettop(L); @@ -869,7 +882,7 @@ static int mp_safe(lua_State *L) { } /* -------------------------------------------------------------------------- */ -static const struct luaL_Reg cmds[] = { +const struct luaL_Reg cmds[] = { {"pack", mp_pack}, {"unpack", mp_unpack}, {"unpack_one", mp_unpack_one}, @@ -877,7 +890,7 @@ static const struct luaL_Reg cmds[] = { {0} }; -static int luaopen_create(lua_State *L) { +int luaopen_create(lua_State *L) { int i; /* Manually construct our module table instead of * relying on _register or _newlib */ From 49efe300af258e83f377cd8142d2c67d66fc2e3a Mon Sep 17 00:00:00 2001 From: Ben Murphy Date: Mon, 11 May 2015 23:24:24 +0100 Subject: [PATCH 18/35] disable loading lua bytecode --- deps/lua/src/ldo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/lua/src/ldo.c b/deps/lua/src/ldo.c index d1bf786c..514f7a2a 100644 --- a/deps/lua/src/ldo.c +++ b/deps/lua/src/ldo.c @@ -495,7 +495,7 @@ static void f_parser (lua_State *L, void *ud) { struct SParser *p = cast(struct SParser *, ud); int c = luaZ_lookahead(p->z); luaC_checkGC(L); - tf = ((c == LUA_SIGNATURE[0]) ? luaU_undump : luaY_parser)(L, p->z, + tf = (luaY_parser)(L, p->z, &p->buff, p->name); cl = luaF_newLclosure(L, tf->nups, hvalue(gt(L))); cl->l.p = tf; From 30278061cc834b4073b004cb1a2bfb0f195734f7 Mon Sep 17 00:00:00 2001 From: Ben Murphy Date: Mon, 11 May 2015 23:24:37 +0100 Subject: [PATCH 19/35] hide access to debug table --- src/scripting.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 4f807f4e..53c0c9ed 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -612,11 +612,12 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { /* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html. * Modified to be adapted to Redis. */ + s[j++]="local dbg=debug\n"; s[j++]="local mt = {}\n"; s[j++]="setmetatable(_G, mt)\n"; s[j++]="mt.__newindex = function (t, n, v)\n"; - s[j++]=" if debug.getinfo(2) then\n"; - s[j++]=" local w = debug.getinfo(2, \"S\").what\n"; + s[j++]=" if dbg.getinfo(2) then\n"; + s[j++]=" local w = dbg.getinfo(2, \"S\").what\n"; s[j++]=" if w ~= \"main\" and w ~= \"C\" then\n"; s[j++]=" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n"; s[j++]=" end\n"; @@ -624,11 +625,12 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { s[j++]=" rawset(t, n, v)\n"; s[j++]="end\n"; s[j++]="mt.__index = function (t, n)\n"; - s[j++]=" if debug.getinfo(2) and debug.getinfo(2, \"S\").what ~= \"C\" then\n"; + s[j++]=" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n"; s[j++]=" error(\"Script attempted to access unexisting global variable '\"..tostring(n)..\"'\", 2)\n"; s[j++]=" end\n"; s[j++]=" return rawget(t, n)\n"; s[j++]="end\n"; + s[j++]="debug = nil\n"; s[j++]=NULL; for (j = 0; s[j] != NULL; j++) code = sdscatlen(code,s[j],strlen(s[j])); @@ -732,10 +734,11 @@ void scriptingInit(void) { * information about the caller, that's what makes sense from the point * of view of the user debugging a script. */ { - char *errh_func = "function __redis__err__handler(err)\n" - " local i = debug.getinfo(2,'nSl')\n" + char *errh_func = "local dbg = debug\n" + "function __redis__err__handler(err)\n" + " local i = dbg.getinfo(2,'nSl')\n" " if i and i.what == 'C' then\n" - " i = debug.getinfo(3,'nSl')\n" + " i = dbg.getinfo(3,'nSl')\n" " end\n" " if i then\n" " return i.source .. ':' .. i.currentline .. ': ' .. err\n" From 01888d1e587c681e80117d7457b0c643a0719a21 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 4 Jun 2015 11:35:56 +0200 Subject: [PATCH 20/35] Redis 3.0.2 --- 00-RELEASENOTES | 14 ++++++++++++++ src/version.h | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index a32930a4..95f86e3a 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -10,6 +10,20 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 3.0.2 ] Release date: 4 Jun 2015 + +Upgrade urgency: HIGH for Redis because of a security issue. + LOW for Sentinel. + +* [FIX] Critical security issue fix by Ben Murphy: http://t.co/LpGTyZmfS7 +* [FIX] SMOVE reply fixed when src and dst keys are the same. (Glenn Nethercutt) +* [FIX] Lua cmsgpack lib updated to support str8 type. (Sebastian Waisbrot) + +* [NEW] ZADD support for options: NX, XX, CH. See new doc at redis.io. + (Salvatore Sanfilippo) +* [NEW] Senitnel: CKQUORUM and FLUSHCONFIG commands back ported. + (Salvatore Sanfilippo) + --[ Redis 3.0.1 ] Release date: 5 May 2015 Upgrade urgency: LOW for Redis and Cluster, MODERATE for Sentinel. diff --git a/src/version.h b/src/version.h index 44ac9c45..ee8c63ce 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "3.0.1" +#define REDIS_VERSION "3.0.2" From f58d67b015f416a97239a75bc32e3323f0eec345 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 4 Jun 2015 11:59:45 +0200 Subject: [PATCH 21/35] Fix 3.0.2 release notes to give full credits. --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 95f86e3a..c1cbed36 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -22,7 +22,7 @@ Upgrade urgency: HIGH for Redis because of a security issue. * [NEW] ZADD support for options: NX, XX, CH. See new doc at redis.io. (Salvatore Sanfilippo) * [NEW] Senitnel: CKQUORUM and FLUSHCONFIG commands back ported. - (Salvatore Sanfilippo) + (Salvatore Sanfilippo and Bill Anderson) --[ Redis 3.0.1 ] Release date: 5 May 2015 From d815289d54b3b34a8ef5e7e749e0bc2513ccdec7 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 11 Jun 2015 12:29:37 +0200 Subject: [PATCH 22/35] Don't try to bind the source address for MIGRATE Related to issues #2609 and #2612. --- src/cluster.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 916a4be6..fb45bd06 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4486,8 +4486,8 @@ migrateCachedSocket* migrateGetSocket(redisClient *c, robj *host, robj *port, lo } /* Create the socket */ - fd = anetTcpNonBlockBindConnect(server.neterr,c->argv[1]->ptr, - atoi(c->argv[2]->ptr),REDIS_BIND_ADDR); + fd = anetTcpNonBlockConnect(server.neterr,c->argv[1]->ptr, + atoi(c->argv[2]->ptr)); if (fd == -1) { sdsfree(name); addReplyErrorFormat(c,"Can't connect to target node: %s", From af8a9de663844e4310cc9c5bef2f879852d01592 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 11 Jun 2015 12:46:55 +0200 Subject: [PATCH 23/35] anetTcpGenericConnect(), jump to error not end on error Two code paths jumped to the "ok, return the socket to the user" code path to handle error conditions. Related to issues #2609 and #2612. --- src/anet.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/anet.c b/src/anet.c index 1e5d8549..bd7d634b 100644 --- a/src/anet.c +++ b/src/anet.c @@ -295,7 +295,7 @@ static int anetTcpGenericConnect(char *err, char *addr, int port, if ((rv = getaddrinfo(source_addr, NULL, &hints, &bservinfo)) != 0) { anetSetError(err, "%s", gai_strerror(rv)); - goto end; + goto error; } for (b = bservinfo; b != NULL; b = b->ai_next) { if (bind(s,b->ai_addr,b->ai_addrlen) != -1) { @@ -306,7 +306,7 @@ static int anetTcpGenericConnect(char *err, char *addr, int port, freeaddrinfo(bservinfo); if (!bound) { anetSetError(err, "bind: %s", strerror(errno)); - goto end; + goto error; } } if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { From a8d7f00e2ee26a5cb374e513e4f0a4a01ef40f88 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 11 Jun 2015 12:55:58 +0200 Subject: [PATCH 24/35] anet.c: new API anetTcpNonBlockBestEffortBindConnect() This performs a best effort source address binding attempt. If it is possible to bind the local address and still have a successful connect(), then this socket is returned. Otherwise the call is retried without source address binding attempt. Related to issues #2609 and #2612. --- src/anet.c | 24 +++++++++++++++++++++--- src/anet.h | 1 + 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/anet.c b/src/anet.c index bd7d634b..0f6da4b4 100644 --- a/src/anet.c +++ b/src/anet.c @@ -264,6 +264,7 @@ static int anetCreateSocket(char *err, int domain) { #define ANET_CONNECT_NONE 0 #define ANET_CONNECT_NONBLOCK 1 +#define ANET_CONNECT_BE_BINDING 2 /* Best effort binding. */ static int anetTcpGenericConnect(char *err, char *addr, int port, char *source_addr, int flags) { @@ -331,9 +332,17 @@ error: close(s); s = ANET_ERR; } + end: freeaddrinfo(servinfo); - return s; + + /* Handle best effort binding: if a binding address was used, but it is + * not possible to create a socket, try again without a binding address. */ + if (s == ANET_ERR && source_addr && (flags & ANET_CONNECT_BE_BINDING)) { + return anetTcpGenericConnect(err,addr,port,NULL,flags); + } else { + return s; + } } int anetTcpConnect(char *err, char *addr, int port) @@ -346,9 +355,18 @@ int anetTcpNonBlockConnect(char *err, char *addr, int port) return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBindConnect(char *err, char *addr, int port, char *source_addr) +int anetTcpNonBlockBindConnect(char *err, char *addr, int port, + char *source_addr) { - return anetTcpGenericConnect(err,addr,port,source_addr,ANET_CONNECT_NONBLOCK); + return anetTcpGenericConnect(err,addr,port,source_addr, + ANET_CONNECT_NONBLOCK); +} + +int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, + char *source_addr) +{ + return anetTcpGenericConnect(err,addr,port,source_addr, + ANET_CONNECT_NONBLOCK|ANET_CONNECT_BE_BINDING); } int anetUnixGenericConnect(char *err, char *path, int flags) diff --git a/src/anet.h b/src/anet.h index b94a0cd1..ac5c36ee 100644 --- a/src/anet.h +++ b/src/anet.h @@ -50,6 +50,7 @@ int anetTcpConnect(char *err, char *addr, int port); int anetTcpNonBlockConnect(char *err, char *addr, int port); int anetTcpNonBlockBindConnect(char *err, char *addr, int port, char *source_addr); +int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, char *source_addr); int anetUnixConnect(char *err, char *path); int anetUnixNonBlockConnect(char *err, char *path); int anetRead(int fd, char *buf, int count); From 4e8759c65e03f5fb8ff2851a9bb45b6327d92f84 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 11 Jun 2015 12:57:53 +0200 Subject: [PATCH 25/35] Use best effort address binding to connect to the master We usually want to reach the master using the address of the interface Redis is bound to (via the "bind" config option). That's useful since the master will get (and publish) the slave address getting the peer name of the incoming socket connection from the slave. However, when this is not possible, for example because the slave is bound to the loopback interface but repliaces from a master accessed via an external interface, we want to still connect with the master even from a different interface: in this case it is not really important that the master will provide any other address, while it is vital to be able to replicate correctly. Related to issues #2609 and #2612. --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 3aefc83e..335a48f4 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1377,7 +1377,7 @@ error: int connectWithMaster(void) { int fd; - fd = anetTcpNonBlockBindConnect(NULL, + fd = anetTcpNonBlockBestEffortBindConnect(NULL, server.masterhost,server.masterport,REDIS_BIND_ADDR); if (fd == -1) { redisLog(REDIS_WARNING,"Unable to connect to MASTER: %s", From d4a7c9e1abd4fae24c47951f51bc00da1c090189 Mon Sep 17 00:00:00 2001 From: linfangrong Date: Tue, 2 Jun 2015 18:12:57 +0800 Subject: [PATCH 26/35] Update t_zset.c --- src/t_zset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_zset.c b/src/t_zset.c index 24f8d309..6ae12ec0 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1248,7 +1248,7 @@ void zaddGenericCommand(redisClient *c, int flags) { if (zobj == NULL) { if (xx) goto reply_to_client; /* No key + XX option: nothing to do. */ if (server.zset_max_ziplist_entries == 0 || - server.zset_max_ziplist_value < sdslen(c->argv[3]->ptr)) + server.zset_max_ziplist_value < sdslen(c->argv[scoreidx+1]->ptr)) { zobj = createZsetObject(); } else { From 55e8d4cf1b1b23b329d69d82e94dac285d2bcb1c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 29 Jun 2015 17:23:47 +0200 Subject: [PATCH 27/35] Add 3.0 changed config default for maxmemory policy to release notes. --- 00-RELEASENOTES | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index c1cbed36..0f2e901c 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -608,6 +608,11 @@ non-backward compatible changes introduced in the 3.0 release: '?' is actually the role of the instance. M for master, S for slave, C if this process is a saving child (for RDB/AOF), and X for Sentinel. +* The default maxmemory policy in Redis 3.0 is no longer "volatile-lru" as + it used to be in 2.8, but "noeviction". The policies behavior is the same + (but LRU eviction is much more precise in 3.0), so only the default value + changed. Just make sure to specify in your redis.conf what you mean. + -------------------------------------------------------------------------------- Credits: Where not specified the implementation and design is done by From 7ae1d4d6f50fa627a32eee261743d41d64a13e96 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jul 2015 18:06:24 +0200 Subject: [PATCH 28/35] EXISTS is now variadic. The new return value is the number of keys existing, among the ones specified in the command line, counting the same key multiple times if given multiple times (and if it exists). See PR #2667. --- src/db.c | 14 +++++++++----- src/redis.c | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/db.c b/src/db.c index 2982c016..b7cbbfd0 100644 --- a/src/db.c +++ b/src/db.c @@ -293,13 +293,17 @@ void delCommand(redisClient *c) { addReplyLongLong(c,deleted); } +/* EXISTS key1 key2 ... key_N. + * Return value is the number of keys existing. */ void existsCommand(redisClient *c) { - expireIfNeeded(c->db,c->argv[1]); - if (dbExists(c->db,c->argv[1])) { - addReply(c, shared.cone); - } else { - addReply(c, shared.czero); + long long count = 0; + int j; + + for (j = 1; j < c->argc; j++) { + expireIfNeeded(c->db,c->argv[j]); + if (dbExists(c->db,c->argv[j])) count++; } + addReplyLongLong(c,count); } void selectCommand(redisClient *c) { diff --git a/src/redis.c b/src/redis.c index 309e9088..4623479a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -129,7 +129,7 @@ struct redisCommand redisCommandTable[] = { {"append",appendCommand,3,"wm",0,NULL,1,1,1,0,0}, {"strlen",strlenCommand,2,"rF",0,NULL,1,1,1,0,0}, {"del",delCommand,-2,"w",0,NULL,1,-1,1,0,0}, - {"exists",existsCommand,2,"rF",0,NULL,1,1,1,0,0}, + {"exists",existsCommand,-2,"rF",0,NULL,1,-1,1,0,0}, {"setbit",setbitCommand,4,"wm",0,NULL,1,1,1,0,0}, {"getbit",getbitCommand,3,"rF",0,NULL,1,1,1,0,0}, {"setrange",setrangeCommand,4,"wm",0,NULL,1,1,1,0,0}, From 5dcba26b3183ebfb65f8add1b44cb3e4d780ca57 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 16 Jul 2015 09:26:36 +0200 Subject: [PATCH 29/35] Clarify a comment in clientsCron(). --- src/redis.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/redis.c b/src/redis.c index 4623479a..fcd770ab 100644 --- a/src/redis.c +++ b/src/redis.c @@ -963,11 +963,11 @@ int clientsCronResizeQueryBuffer(redisClient *c) { } void clientsCron(void) { - /* Make sure to process at least 1/(server.hz*10) of clients per call. - * Since this function is called server.hz times per second we are sure that - * in the worst case we process all the clients in 10 seconds. - * In normal conditions (a reasonable number of clients) we process - * all the clients in a shorter time. */ + /* Make sure to process at least numclients/(server.hz*10) of clients + * per call. Since this function is called server.hz times per second + * we are sure that in the worst case we process all the clients in 10 + * seconds. In normal conditions (a reasonable number of clients) we + * process all the clients in a shorter time. */ int numclients = listLength(server.clients); int iterations = numclients/(server.hz*10); From b029ff11b6bb6cac438c3833657a7b0d7172df30 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 16 Jul 2015 10:54:12 +0200 Subject: [PATCH 30/35] Client timeout handling improved. The previos attempt to process each client at least once every ten seconds was not a good idea, because: 1. Usually because of the past min iterations set to 50, you get much better processing period most of the times. 2. However when there are many clients and a normal setting for server.hz, the edge case is triggered, and waiting 10 seconds for a BLPOP that asked for 1 second is not ok. 3. Moreover, because of the high min-itereations limit of 50, when HZ was set to an high value, the actual behavior was to process a lot of clients per second. Also the function checking for timeouts called gettimeofday() at each iteration which can be costly. The new implementation will try to process each client once per second, gets the current time as argument, and does not attempt to process more than 5 clients per iteration if not needed. So now: 1. The CPU usage of an idle Redis process is the same or better. 2. The CPU usage of a busy Redis process is the same or better. 3. However a non trivial amount of work may be performed per iteration when there are many many clients. In this particular case the user may want to raise the "HZ" value if needed. Btw with 4000 clients it was still not possible to noticy any actual latency created by processing 400 clients per second, since the work performed for each client is pretty small. --- src/redis.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/redis.c b/src/redis.c index fcd770ab..27e7aa15 100644 --- a/src/redis.c +++ b/src/redis.c @@ -902,9 +902,12 @@ long long getInstantaneousMetric(int metric) { return sum / REDIS_METRIC_SAMPLES; } -/* Check for timeouts. Returns non-zero if the client was terminated */ -int clientsCronHandleTimeout(redisClient *c) { - time_t now = server.unixtime; +/* Check for timeouts. Returns non-zero if the client was terminated. + * The function gets the current time in milliseconds as argument since + * it gets called multiple times in a loop, so calling gettimeofday() for + * each iteration would be costly without any actual gain. */ +int clientsCronHandleTimeout(redisClient *c, mstime_t now_ms) { + time_t now = now_ms/1000; if (server.maxidletime && !(c->flags & REDIS_SLAVE) && /* no timeout for slaves */ @@ -920,7 +923,6 @@ int clientsCronHandleTimeout(redisClient *c) { /* Blocked OPS timeout is handled with milliseconds resolution. * However note that the actual resolution is limited by * server.hz. */ - mstime_t now_ms = mstime(); if (c->bpop.timeout != 0 && c->bpop.timeout < now_ms) { /* Handle blocking operation specific timeout. */ @@ -962,17 +964,23 @@ int clientsCronResizeQueryBuffer(redisClient *c) { return 0; } +#define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { - /* Make sure to process at least numclients/(server.hz*10) of clients + /* Make sure to process at least numclients/server.hz of clients * per call. Since this function is called server.hz times per second - * we are sure that in the worst case we process all the clients in 10 - * seconds. In normal conditions (a reasonable number of clients) we - * process all the clients in a shorter time. */ + * we are sure that in the worst case we process all the clients in 1 + * second. */ int numclients = listLength(server.clients); - int iterations = numclients/(server.hz*10); + int iterations = numclients/server.hz; + mstime_t now = mstime(); + + /* Process at least a few clients while we are at it, even if we need + * to process less than CLIENTS_CRON_MIN_ITERATIONS to meet our contract + * of processing each client once per second. */ + if (iterations < CLIENTS_CRON_MIN_ITERATIONS) + iterations = (numclients < CLIENTS_CRON_MIN_ITERATIONS) ? + numclients : CLIENTS_CRON_MIN_ITERATIONS; - if (iterations < 50) - iterations = (numclients < 50) ? numclients : 50; while(listLength(server.clients) && iterations--) { redisClient *c; listNode *head; @@ -986,7 +994,7 @@ void clientsCron(void) { /* The following functions do different service checks on the client. * The protocol is that they return non-zero if the client was * terminated. */ - if (clientsCronHandleTimeout(c)) continue; + if (clientsCronHandleTimeout(c,now)) continue; if (clientsCronResizeQueryBuffer(c)) continue; } } From ad0082fde43afbb0b74dbf75ad5912536d038f88 Mon Sep 17 00:00:00 2001 From: Tom Kiemes Date: Wed, 15 Jul 2015 16:11:40 +0200 Subject: [PATCH 31/35] Fix: aof_delayed_fsync is not reset aof_delayed_fsync was not set to 0 when calling CONFIG RESETSTAT --- src/redis.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redis.c b/src/redis.c index 27e7aa15..cbc015f1 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1748,6 +1748,7 @@ void resetServerStats(void) { } server.stat_net_input_bytes = 0; server.stat_net_output_bytes = 0; + server.aof_delayed_fsync = 0; } void initServer(void) { From a57aa8831ee8cdb36b47d501db48a0554acd8aa8 Mon Sep 17 00:00:00 2001 From: Yongyue Sun Date: Fri, 10 Jul 2015 15:25:40 +0800 Subject: [PATCH 32/35] bugfix: errno might change before logging Signed-off-by: Yongyue Sun --- src/aof.c | 2 +- src/rdb.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aof.c b/src/aof.c index c6b84040..806a7d00 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1169,9 +1169,9 @@ int rewriteAppendOnlyFile(char *filename) { return REDIS_OK; werr: + redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); fclose(fp); unlink(tmpfile); - redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); if (di) dictReleaseIterator(di); return REDIS_ERR; } diff --git a/src/rdb.c b/src/rdb.c index 3dd69f28..f695f414 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -755,9 +755,9 @@ int rdbSave(char *filename) { return REDIS_OK; werr: + redisLog(REDIS_WARNING,"Write error saving DB on disk: %s", strerror(errno)); fclose(fp); unlink(tmpfile); - redisLog(REDIS_WARNING,"Write error saving DB on disk: %s", strerror(errno)); return REDIS_ERR; } From f22a9c0f78310a4b941df81569d88f8fac89368f Mon Sep 17 00:00:00 2001 From: MOON_CLJ Date: Fri, 26 Jun 2015 17:58:45 +0800 Subject: [PATCH 33/35] pfcount support multi keys --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index cbc015f1..09cdd63c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -280,7 +280,7 @@ struct redisCommand redisCommandTable[] = { {"command",commandCommand,0,"rlt",0,NULL,0,0,0,0,0}, {"pfselftest",pfselftestCommand,1,"r",0,NULL,0,0,0,0,0}, {"pfadd",pfaddCommand,-2,"wmF",0,NULL,1,1,1,0,0}, - {"pfcount",pfcountCommand,-2,"r",0,NULL,1,1,1,0,0}, + {"pfcount",pfcountCommand,-2,"r",0,NULL,1,-1,1,0,0}, {"pfmerge",pfmergeCommand,-2,"wm",0,NULL,1,-1,1,0,0}, {"pfdebug",pfdebugCommand,-3,"w",0,NULL,0,0,0,0,0}, {"latency",latencyCommand,-2,"arslt",0,NULL,0,0,0,0,0} From 9b0a47cbc83920f2b6d20e51908978df6ab2e6cd Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Wed, 24 Jun 2015 12:55:00 +0200 Subject: [PATCH 34/35] Do not attempt to lock on Solaris --- src/cluster.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/cluster.c b/src/cluster.c index fb45bd06..6280677a 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -358,6 +358,11 @@ void clusterSaveConfigOrDie(int do_fsync) { * On success REDIS_OK is returned, otherwise an error is logged and * the function returns REDIS_ERR to signal a lock was not acquired. */ int clusterLockConfig(char *filename) { +/* flock() does not exist on Solaris + * and a fcntl-based solution won't help, as we constantly re-open that file, + * which will release _all_ locks anyway + */ +#if !defined(__sun) /* To lock it, we need to open the file in a way it is created if * it does not exist, otherwise there is a race condition with other * processes. */ @@ -385,6 +390,8 @@ int clusterLockConfig(char *filename) { } /* Lock acquired: leak the 'fd' by not closing it, so that we'll retain the * lock to the file as long as the process exists. */ +#endif /* __sun */ + return REDIS_OK; } From f15bf6c8f87f3008ea95dddca6be51261f411b44 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 17 Jul 2015 11:50:21 +0200 Subject: [PATCH 35/35] Redis 3.0.3. --- 00-RELEASENOTES | 22 ++++++++++++++++++++++ src/version.h | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 0f2e901c..bffd7a12 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -10,6 +10,28 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 3.0.3 ] Release date: 17 Jul 2015 + +Upgrade urgency: LOW for Redis and Sentinel. + +* [FIX] Fix blocking operations timeout precision when HZ is at its default + value (not increased) and there are thousands of clients connected + at the same time. This bug affected Sidekiq users that experienced + a very long delay for BLPOP and similar commands to return for + timeout. Check commit b029ff1 for more info. (Salvatore Sanfilippo) +* [FIX] MIGRATE "creating socket: Invalid argument" error fix. Check + issues #2609 and #2612 for more info. (Salvatore Sanfilippo) +* [FIX] Be able to connect to the master even when the slave is bound to + just the loopback interface and has no valid public address in the + network the master is reacahble. (Salvatore Sanfilippo) +* [FIX] ZADD with options encoding promotion fixed. (linfangrong) +* [FIX] Reset aof_delayed_fsync on CONFIG RESETSTATS. (Tom Kiemes) +* [FIX] PFCOUNT key parsing in cluster fixed. (MOON_CLJ) +* [FIX] Fix Solaris compilation of Redis 3.0. (Jan-Erik Rediger) + +* [NEW] Variadic EXISTS command. Now the command accepts multiple arguments + and returns the total count of existing keys. + --[ Redis 3.0.2 ] Release date: 4 Jun 2015 Upgrade urgency: HIGH for Redis because of a security issue. diff --git a/src/version.h b/src/version.h index ee8c63ce..93c9f285 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "3.0.2" +#define REDIS_VERSION "3.0.3"