Redis 2.8.9.

The internal HLL raw encoding used by PFCOUNT when merging multiple keys
is aligned to 8 bits (1 byte per register) so we can exploit this to
improve performances by processing multiple bytes per iteration.

In benchmarks the new code was several times faster with HLLs with many
registers set to zero, while no slowdown was observed with populated
HLLs.

00-RELEASENOTES

@@ -14,6 +14,16 @@ HIGH:     There is a critical bug that may affect a subset of users. Upgrade!
 CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
 --------------------------------------------------------------------------------
 
+--[ Redis 2.8.9 ] Release date: 22 Apr 2014
+
+# UPGRADE URGENCY: LOW, only new features introduced, no bugs fixed.
+
+* [NEW] The HyperLogLog data structure. You can read more about it
+        in this blog post. http://antirez.com/news/75
+* [NEW] The Sorted Set data type has now support for lexicographic range
+        queries, check the new commands ZRANGEBYLEX, ZLEXCOUNT and
+        ZREMRANGEBYLEX, which are documented at http://redis.io.
+
 --[ Redis 2.8.8 ] Release date: 25 Mar 2014
 
 # UPGRADE URGENCY: HIGH for Redis, LOW for Sentinel. There is a potentially
@@ -308,6 +318,9 @@ that you should be aware of:
 
 The following commands changed behavior:
 
+    * The TTL and PTTL commands now return -2 if the key does not exist and
+      -1 if it exists but has no associated expire. Redis 2.6 and previous
+      versions used to return -1 for both the conditions.
     * SORT with ALPHA now sorts according to local collation locale if no STORE
       option is used.
     * ZADD/ZINCRBY are now able to accept a bigger range of values as valid

redis.conf

@@ -635,6 +635,20 @@ set-max-intset-entries 512
 zset-max-ziplist-entries 128
 zset-max-ziplist-value 64
 
+# HyperLogLog sparse representation bytes limit. The limit includes the
+# 16 bytes header. When an HyperLogLog using the sparse representation crosses
+# this limit, it is convereted into the dense representation.
+#
+# A value greater than 16000 is totally useless, since at that point the
+# dense representation is more memory efficient.
+# 
+# The suggested value is ~ 3000 in order to have the benefits of
+# the space efficient encoding without slowing down too much PFADD,
+# which is O(N) with the sparse encoding. Thev value can be raised to
+# ~ 10000 when CPU is not a concern, but space is, and the data set is
+# composed of many HyperLogLogs with cardinality in the 0 - 15000 range.
+hll-sparse-max-bytes 3000
+
 # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
 # order to help rehashing the main Redis hash table (the one mapping top-level
 # keys to values). The hash table implementation Redis uses (see dict.c)

src/Makefile

@@ -107,7 +107,7 @@ endif
 
 REDIS_SERVER_NAME=redis-server
 REDIS_SENTINEL_NAME=redis-sentinel
-REDIS_SERVER_OBJ=adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o
+REDIS_SERVER_OBJ=adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o hyperloglog.o
 REDIS_CLI_NAME=redis-cli
 REDIS_CLI_OBJ=anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o
 REDIS_BENCHMARK_NAME=redis-benchmark

src/Makefile.dep

@@ -26,6 +26,10 @@ debug.o: debug.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \
   ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h crc64.h bio.h
 dict.o: dict.c fmacros.h dict.h zmalloc.h redisassert.h
 endianconv.o: endianconv.c
+hyperloglog.o: hyperloglog.c redis.h fmacros.h config.h \
+ ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h sds.h dict.h \
+ adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \
+ rio.h
 intset.o: intset.c intset.h zmalloc.h endianconv.h config.h
 lzf_c.o: lzf_c.c lzfP.h
 lzf_d.o: lzf_d.c lzfP.h

src/bitops.c

@@ -223,14 +223,7 @@ void setbitCommand(redisClient *c) {
         dbAdd(c->db,c->argv[1],o);
     } else {
         if (checkType(c,o,REDIS_STRING)) return;
-
-        /* Create a copy when the object is shared or encoded. */
-        if (o->refcount != 1 || o->encoding != REDIS_ENCODING_RAW) {
-            robj *decoded = getDecodedObject(o);
-            o = createStringObject(decoded->ptr, sdslen(decoded->ptr));
-            decrRefCount(decoded);
-            dbOverwrite(c->db,c->argv[1],o);
-        }
+        o = dbUnshareStringValue(c->db,c->argv[1],o);
     }
 
     /* Grow sds value to the right length if necessary */

src/config.c

@@ -388,6 +388,8 @@ void loadServerConfigFromString(char *config) {
             server.zset_max_ziplist_entries = memtoll(argv[1], NULL);
         } else if (!strcasecmp(argv[0],"zset-max-ziplist-value") && argc == 2) {
             server.zset_max_ziplist_value = memtoll(argv[1], NULL);
+        } else if (!strcasecmp(argv[0],"hll-sparse-max-bytes") && argc == 2) {
+            server.hll_sparse_max_bytes = memtoll(argv[1], NULL);
         } else if (!strcasecmp(argv[0],"rename-command") && argc == 3) {
             struct redisCommand *cmd = lookupCommand(argv[1]);
             int retval;
@@ -733,6 +735,9 @@ void configSetCommand(redisClient *c) {
     } else if (!strcasecmp(c->argv[2]->ptr,"zset-max-ziplist-value")) {
         if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
         server.zset_max_ziplist_value = ll;
+    } else if (!strcasecmp(c->argv[2]->ptr,"hll-sparse-max-bytes")) {
+        if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
+        server.hll_sparse_max_bytes = ll;
     } else if (!strcasecmp(c->argv[2]->ptr,"lua-time-limit")) {
         if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
         server.lua_time_limit = ll;
@@ -934,6 +939,8 @@ void configGetCommand(redisClient *c) {
             server.zset_max_ziplist_entries);
     config_get_numerical_field("zset-max-ziplist-value",
             server.zset_max_ziplist_value);
+    config_get_numerical_field("hll-sparse-max-bytes",
+            server.hll_sparse_max_bytes);
     config_get_numerical_field("lua-time-limit",server.lua_time_limit);
     config_get_numerical_field("slowlog-log-slower-than",
             server.slowlog_log_slower_than);
@@ -1726,6 +1733,7 @@ int rewriteConfig(char *path) {
     rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,REDIS_SET_MAX_INTSET_ENTRIES);
     rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,REDIS_ZSET_MAX_ZIPLIST_ENTRIES);
     rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,REDIS_ZSET_MAX_ZIPLIST_VALUE);
+    rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",server.hll_sparse_max_bytes,REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES);
     rewriteConfigYesNoOption(state,"activerehashing",server.activerehashing,REDIS_DEFAULT_ACTIVE_REHASHING);
     rewriteConfigClientoutputbufferlimitOption(state);
     rewriteConfigNumericalOption(state,"hz",server.hz,REDIS_DEFAULT_HZ);

src/db.c

@@ -166,6 +166,44 @@ int dbDelete(redisDb *db, robj *key) {
     }
 }
 
+/* Prepare the string object stored at 'key' to be modified destructively
+ * to implement commands like SETBIT or APPEND.
+ *
+ * An object is usually ready to be modified unless one of the two conditions
+ * are true:
+ *
+ * 1) The object 'o' is shared (refcount > 1), we don't want to affect
+ *    other users.
+ * 2) The object encoding is not "RAW".
+ *
+ * If the object is found in one of the above conditions (or both) by the
+ * function, an unshared / not-encoded copy of the string object is stored
+ * at 'key' in the specified 'db'. Otherwise the object 'o' itself is
+ * returned.
+ *
+ * USAGE:
+ *
+ * The object 'o' is what the caller already obtained by looking up 'key'
+ * in 'db', the usage pattern looks like this:
+ *
+ * o = lookupKeyWrite(db,key);
+ * if (checkType(c,o,REDIS_STRING)) return;
+ * o = dbUnshareStringValue(db,key,o);
+ *
+ * At this point the caller is ready to modify the object, for example
+ * using an sdscat() call to append some data, or anything else.
+ */
+robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) {
+    redisAssert(o->type == REDIS_STRING);
+    if (o->refcount != 1 || o->encoding != REDIS_ENCODING_RAW) {
+        robj *decoded = getDecodedObject(o);
+        o = createStringObject(decoded->ptr, sdslen(decoded->ptr));
+        decrRefCount(decoded);
+        dbOverwrite(db,key,o);
+    }
+    return o;
+}
+
 long long emptyDb(void(callback)(void*)) {
     int j;
     long long removed = 0;

src/object.c

@@ -587,7 +587,7 @@ unsigned long estimateObjectIdleTime(robj *o) {
     }
 }
 
-/* This is a helper function for the DEBUG command. We need to lookup keys
+/* This is a helper function for the OBJECT command. We need to lookup keys
  * without any modification of LRU or other parameters. */
 robj *objectCommandLookup(redisClient *c, robj *key) {
     dictEntry *de;

src/redis.c

@@ -168,12 +168,16 @@ struct redisCommand redisCommandTable[] = {
     {"zrem",zremCommand,-3,"w",0,NULL,1,1,1,0,0},
     {"zremrangebyscore",zremrangebyscoreCommand,4,"w",0,NULL,1,1,1,0,0},
     {"zremrangebyrank",zremrangebyrankCommand,4,"w",0,NULL,1,1,1,0,0},
+    {"zremrangebylex",zremrangebylexCommand,4,"w",0,NULL,1,1,1,0,0},
     {"zunionstore",zunionstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
     {"zinterstore",zinterstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
     {"zrange",zrangeCommand,-4,"r",0,NULL,1,1,1,0,0},
     {"zrangebyscore",zrangebyscoreCommand,-4,"r",0,NULL,1,1,1,0,0},
     {"zrevrangebyscore",zrevrangebyscoreCommand,-4,"r",0,NULL,1,1,1,0,0},
+    {"zrangebylex",zrangebylexCommand,-4,"r",0,NULL,1,1,1,0,0},
+    {"zrevrangebylex",zrevrangebylexCommand,-4,"r",0,NULL,1,1,1,0,0},
     {"zcount",zcountCommand,4,"r",0,NULL,1,1,1,0,0},
+    {"zlexcount",zlexcountCommand,4,"r",0,NULL,1,1,1,0,0},
     {"zrevrange",zrevrangeCommand,-4,"r",0,NULL,1,1,1,0,0},
     {"zcard",zcardCommand,2,"r",0,NULL,1,1,1,0,0},
     {"zscore",zscoreCommand,3,"r",0,NULL,1,1,1,0,0},
@@ -258,7 +262,12 @@ struct redisCommand redisCommandTable[] = {
     {"time",timeCommand,1,"rR",0,NULL,0,0,0,0,0},
     {"bitop",bitopCommand,-4,"wm",0,NULL,2,-1,1,0,0},
     {"bitcount",bitcountCommand,-2,"r",0,NULL,1,1,1,0,0},
-    {"bitpos",bitposCommand,-3,"r",0,NULL,1,1,1,0,0}
+    {"bitpos",bitposCommand,-3,"r",0,NULL,1,1,1,0,0},
+    {"pfselftest",pfselftestCommand,1,"r",0,NULL,0,0,0,0,0},
+    {"pfadd",pfaddCommand,-2,"wm",0,NULL,1,1,1,0,0},
+    {"pfcount",pfcountCommand,-2,"w",0,NULL,1,1,1,0,0},
+    {"pfmerge",pfmergeCommand,-2,"wm",0,NULL,1,-1,1,0,0},
+    {"pfdebug",pfdebugCommand,-3,"w",0,NULL,0,0,0,0,0}
 };
 
 /*============================ Utility functions ============================ */
@@ -1273,6 +1282,12 @@ void createSharedObjects(void) {
         shared.bulkhdr[j] = createObject(REDIS_STRING,
             sdscatprintf(sdsempty(),"$%d\r\n",j));
     }
+    /* The following two shared objects, minstring and maxstrings, are not
+     * actually used for their value but as a special object meaning
+     * respectively the minimum possible string and the maximum possible
+     * string in string comparisons for the ZRANGEBYLEX command. */
+    shared.minstring = createStringObject("minstring",9);
+    shared.maxstring = createStringObject("maxstring",9);
 }
 
 void initServerConfig() {
@@ -1340,6 +1355,7 @@ void initServerConfig() {
     server.set_max_intset_entries = REDIS_SET_MAX_INTSET_ENTRIES;
     server.zset_max_ziplist_entries = REDIS_ZSET_MAX_ZIPLIST_ENTRIES;
     server.zset_max_ziplist_value = REDIS_ZSET_MAX_ZIPLIST_VALUE;
+    server.hll_sparse_max_bytes = REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES;
     server.shutdown_asap = 0;
     server.repl_ping_slave_period = REDIS_REPL_PING_SLAVE_PERIOD;
     server.repl_timeout = REDIS_REPL_TIMEOUT;
@@ -1469,24 +1485,28 @@ void adjustOpenFilesLimit(void) {
                     redisLog(REDIS_WARNING,"Your current 'ulimit -n' "
                         "of %llu is not enough for Redis to start. "
                         "Please increase your open file limit to at least "
-                        "%llu. Exiting.", oldlimit, maxfiles);
+                        "%llu. Exiting.",
+                        (unsigned long long) oldlimit,
+                        (unsigned long long) maxfiles);
                     exit(1);
                 }
                 redisLog(REDIS_WARNING,"You requested maxclients of %d "
                     "requiring at least %llu max file descriptors.",
-                    old_maxclients, maxfiles);
+                    old_maxclients,
+                    (unsigned long long) maxfiles);
                 redisLog(REDIS_WARNING,"Redis can't set maximum open files "
                     "to %llu because of OS error: %s.",
-                    maxfiles, strerror(setrlimit_error));
+                    (unsigned long long) maxfiles, strerror(setrlimit_error));
                 redisLog(REDIS_WARNING,"Current maximum open files is %llu. "
                     "maxclients has been reduced to %d to compensate for "
                     "low ulimit. "
                     "If you need higher maxclients increase 'ulimit -n'.",
-                    oldlimit, server.maxclients);
+                    (unsigned long long) oldlimit, server.maxclients);
             } else {
                 redisLog(REDIS_NOTICE,"Increased maximum number of open files "
                     "to %llu (it was originally set to %llu).",
-                    maxfiles, oldlimit);
+                    (unsigned long long) maxfiles,
+                    (unsigned long long) oldlimit);
             }
         }
     }

src/redis.h

@@ -303,6 +303,9 @@
 #define REDIS_ZSET_MAX_ZIPLIST_ENTRIES 128
 #define REDIS_ZSET_MAX_ZIPLIST_VALUE 64
 
+/* HyperLogLog defines */
+#define REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES 3000
+
 /* Sets operations codes */
 #define REDIS_OP_UNION 0
 #define REDIS_OP_DIFF 1
@@ -502,7 +505,7 @@ struct sharedObjectsStruct {
     *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr,
     *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk,
     *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *rpop, *lpop,
-    *lpush, *emptyscan,
+    *lpush, *emptyscan, *minstring, *maxstring,
     *select[REDIS_SHARED_SELECT_CMDS],
     *integers[REDIS_SHARED_INTEGERS],
     *mbulkhdr[REDIS_SHARED_BULKHDR_LEN], /* "*<value>\r\n" */
@@ -755,6 +758,7 @@ struct redisServer {
     size_t set_max_intset_entries;
     size_t zset_max_ziplist_entries;
     size_t zset_max_ziplist_value;
+    size_t hll_sparse_max_bytes;
     time_t unixtime;        /* Unix time sampled every cron cycle. */
     long long mstime;       /* Like 'unixtime' but with milliseconds resolution. */
     /* Pubsub */
@@ -1057,18 +1061,24 @@ unsigned long aofRewriteBufferSize(void);
 
 /* Sorted sets data type */
 
-/* Struct to hold a inclusive/exclusive range spec. */
+/* Struct to hold a inclusive/exclusive range spec by score comparison. */
 typedef struct {
     double min, max;
     int minex, maxex; /* are min or max exclusive? */
 } zrangespec;
 
+/* Struct to hold an inclusive/exclusive range spec by lexicographic comparison. */
+typedef struct {
+    robj *min, *max;  /* May be set to shared.(minstring|maxstring) */
+    int minex, maxex; /* are min or max exclusive? */
+} zlexrangespec;
+
 zskiplist *zslCreate(void);
 void zslFree(zskiplist *zsl);
 zskiplistNode *zslInsert(zskiplist *zsl, double score, robj *obj);
 unsigned char *zzlInsert(unsigned char *zl, robj *ele, double score);
 int zslDelete(zskiplist *zsl, double score, robj *obj);
-zskiplistNode *zslFirstInRange(zskiplist *zsl, zrangespec range);
+zskiplistNode *zslFirstInRange(zskiplist *zsl, zrangespec *range);
 double zzlGetScore(unsigned char *sptr);
 void zzlNext(unsigned char *zl, unsigned char **eptr, unsigned char **sptr);
 void zzlPrev(unsigned char *zl, unsigned char **eptr, unsigned char **sptr);
@@ -1176,6 +1186,7 @@ void setKey(redisDb *db, robj *key, robj *val);
 int dbExists(redisDb *db, robj *key);
 robj *dbRandomKey(redisDb *db);
 int dbDelete(redisDb *db, robj *key);
+robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o);
 long long emptyDb(void(callback)(void*));
 int selectDb(redisClient *c, int id);
 void signalModifiedKey(redisDb *db, robj *key);
@@ -1294,12 +1305,16 @@ void zincrbyCommand(redisClient *c);
 void zrangeCommand(redisClient *c);
 void zrangebyscoreCommand(redisClient *c);
 void zrevrangebyscoreCommand(redisClient *c);
+void zrangebylexCommand(redisClient *c);
+void zrevrangebylexCommand(redisClient *c);
 void zcountCommand(redisClient *c);
+void zlexcountCommand(redisClient *c);
 void zrevrangeCommand(redisClient *c);
 void zcardCommand(redisClient *c);
 void zremCommand(redisClient *c);
 void zscoreCommand(redisClient *c);
 void zremrangebyscoreCommand(redisClient *c);
+void zremrangebylexCommand(redisClient *c);
 void multiCommand(redisClient *c);
 void execCommand(redisClient *c);
 void discardCommand(redisClient *c);
@@ -1350,6 +1365,11 @@ void bitopCommand(redisClient *c);
 void bitcountCommand(redisClient *c);
 void bitposCommand(redisClient *c);
 void replconfCommand(redisClient *c);
+void pfselftestCommand(redisClient *c);
+void pfaddCommand(redisClient *c);
+void pfcountCommand(redisClient *c);
+void pfmergeCommand(redisClient *c);
+void pfdebugCommand(redisClient *c);
 
 #if defined(__GNUC__)
 void *calloc(size_t count, size_t size) __attribute__ ((deprecated));

src/t_string.c

@@ -215,12 +215,7 @@ void setrangeCommand(redisClient *c) {
             return;
 
         /* Create a copy when the object is shared or encoded. */
-        if (o->refcount != 1 || o->encoding != REDIS_ENCODING_RAW) {
-            robj *decoded = getDecodedObject(o);
-            o = createStringObject(decoded->ptr, sdslen(decoded->ptr));
-            decrRefCount(decoded);
-            dbOverwrite(c->db,c->argv[1],o);
-        }
+        o = dbUnshareStringValue(c->db,c->argv[1],o);
     }
 
     if (sdslen(value) > 0) {
@@ -433,15 +428,8 @@ void appendCommand(redisClient *c) {
         if (checkStringLength(c,totlen) != REDIS_OK)
             return;
 
-        /* If the object is shared or encoded, we have to make a copy */
-        if (o->refcount != 1 || o->encoding != REDIS_ENCODING_RAW) {
-            robj *decoded = getDecodedObject(o);
-            o = createStringObject(decoded->ptr, sdslen(decoded->ptr));
-            decrRefCount(decoded);
-            dbOverwrite(c->db,c->argv[1],o);
-        }
-
         /* Append the value */
+        o = dbUnshareStringValue(c->db,c->argv[1],o);
         o->ptr = sdscatlen(o->ptr,append->ptr,sdslen(append->ptr));
         totlen = sdslen(o->ptr);
     }

src/version.h

@@ -1 +1 @@
-#define REDIS_VERSION "2.8.8"
+#define REDIS_VERSION "2.8.9"

tests/test_helper.tcl

@@ -47,6 +47,7 @@ set ::all_tests {
     unit/dump
     unit/bitops
     unit/memefficiency
+    unit/hyperloglog
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0

tests/unit/hyperloglog.tcl

@@ -0,0 +1,159 @@
+start_server {tags {"hll"}} {
+    test {HyperLogLog self test passes} {
+        catch {r pfselftest} e
+        set e
+    } {OK}
+
+    test {PFADD without arguments creates an HLL value} {
+        r pfadd hll
+        r exists hll
+    } {1}
+
+    test {Approximated cardinality after creation is zero} {
+        r pfcount hll
+    } {0}
+
+    test {PFADD returns 1 when at least 1 reg was modified} {
+        r pfadd hll a b c
+    } {1}
+
+    test {PFADD returns 0 when no reg was modified} {
+        r pfadd hll a b c
+    } {0}
+
+    test {PFADD works with empty string (regression)} {
+        r pfadd hll ""
+    }
+
+    # Note that the self test stresses much better the
+    # cardinality estimation error. We are testing just the
+    # command implementation itself here.
+    test {PFCOUNT returns approximated cardinality of set} {
+        r del hll
+        set res {}
+        r pfadd hll 1 2 3 4 5
+        lappend res [r pfcount hll]
+        # Call it again to test cached value invalidation.
+        r pfadd hll 6 7 8 8 9 10
+        lappend res [r pfcount hll]
+        set res
+    } {5 10}
+
+    test {HyperLogLogs are promote from sparse to dense} {
+        r del hll
+        r config set hll-sparse-max-bytes 3000
+        set n 0
+        while {$n < 100000} {
+            set elements {}
+            for {set j 0} {$j < 100} {incr j} {lappend elements [expr rand()]}
+            incr n 100
+            r pfadd hll {*}$elements
+            set card [r pfcount hll]
+            set err [expr {abs($card-$n)}]
+            assert {$err < (double($card)/100)*5}
+            if {$n < 1000} {
+                assert {[r pfdebug encoding hll] eq {sparse}}
+            } elseif {$n > 10000} {
+                assert {[r pfdebug encoding hll] eq {dense}}
+            }
+        }
+    }
+
+    test {HyperLogLog sparse encoding stress test} {
+        for {set x 0} {$x < 1000} {incr x} {
+            r del hll1 hll2
+            set numele [randomInt 100]
+            set elements {}
+            for {set j 0} {$j < $numele} {incr j} {
+                lappend elements [expr rand()]
+            }
+            # Force dense representation of hll2
+            r pfadd hll2
+            r pfdebug todense hll2
+            r pfadd hll1 {*}$elements
+            r pfadd hll2 {*}$elements
+            assert {[r pfdebug encoding hll1] eq {sparse}}
+            assert {[r pfdebug encoding hll2] eq {dense}}
+            # Cardinality estimated should match exactly.
+            assert {[r pfcount hll1] eq [r pfcount hll2]}
+        }
+    }
+
+    test {Corrupted sparse HyperLogLogs are detected: Additionl at tail} {
+        r del hll
+        r pfadd hll a b c
+        r append hll "hello"
+        set e {}
+        catch {r pfcount hll} e
+        set e
+    } {*INVALIDOBJ*}
+
+    test {Corrupted sparse HyperLogLogs are detected: Broken magic} {
+        r del hll
+        r pfadd hll a b c
+        r setrange hll 0 "0123"
+        set e {}
+        catch {r pfcount hll} e
+        set e
+    } {*WRONGTYPE*}
+
+    test {Corrupted sparse HyperLogLogs are detected: Invalid encoding} {
+        r del hll
+        r pfadd hll a b c
+        r setrange hll 4 "x"
+        set e {}
+        catch {r pfcount hll} e
+        set e
+    } {*WRONGTYPE*}
+
+    test {Corrupted dense HyperLogLogs are detected: Wrong length} {
+        r del hll
+        r pfadd hll a b c
+        r setrange hll 4 "\x00"
+        set e {}
+        catch {r pfcount hll} e
+        set e
+    } {*WRONGTYPE*}
+
+    test {PFADD, PFCOUNT, PFMERGE type checking works} {
+        r set foo bar
+        catch {r pfadd foo 1} e
+        assert_match {*WRONGTYPE*} $e
+        catch {r pfcount foo} e
+        assert_match {*WRONGTYPE*} $e
+        catch {r pfmerge bar foo} e
+        assert_match {*WRONGTYPE*} $e
+        catch {r pfmerge foo bar} e
+        assert_match {*WRONGTYPE*} $e
+    }
+
+    test {PFMERGE results on the cardinality of union of sets} {
+        r del hll hll1 hll2 hll3
+        r pfadd hll1 a b c
+        r pfadd hll2 b c d
+        r pfadd hll3 c d e
+        r pfmerge hll hll1 hll2 hll3
+        r pfcount hll
+    } {5}
+
+    test {PFCOUNT multiple-keys merge returns cardinality of union} {
+        r del hll1 hll2 hll3
+        for {set x 1} {$x < 10000} {incr x} {
+            # Force dense representation of hll2
+            r pfadd hll1 "foo-$x"
+            r pfadd hll2 "bar-$x"
+            r pfadd hll3 "zap-$x"
+
+            set card [r pfcount hll1 hll2 hll3]
+            set realcard [expr {$x*3}]
+            set err [expr {abs($card-$realcard)}]
+            assert {$err < (double($card)/100)*5}
+        }
+    }
+
+    test {PFDEBUG GETREG returns the HyperLogLog raw registers} {
+        r del hll
+        r pfadd hll 1 2 3
+        llength [r pfdebug getreg hll]
+    } {16384}
+}

tests/unit/type/zset.tcl

@@ -296,6 +296,62 @@ start_server {tags {"zset"}} {
             assert_error "*not*float*" {r zrangebyscore fooz 1 NaN}
         }
 
+        proc create_default_lex_zset {} {
+            create_zset zset {0 alpha 0 bar 0 cool 0 down
+                              0 elephant 0 foo 0 great 0 hill
+                              0 omega}
+        }
+
+        test "ZRANGEBYLEX/ZREVRANGEBYLEX/ZCOUNT basics" {
+            create_default_lex_zset
+
+            # inclusive range
+            assert_equal {alpha bar cool} [r zrangebylex zset - \[cool]
+            assert_equal {bar cool down} [r zrangebylex zset \[bar \[down]
+            assert_equal {great hill omega} [r zrangebylex zset \[g +]
+            assert_equal {cool bar alpha} [r zrevrangebylex zset \[cool -]
+            assert_equal {down cool bar} [r zrevrangebylex zset \[down \[bar]
+            assert_equal {omega hill great foo elephant down} [r zrevrangebylex zset + \[d]
+            assert_equal 3 [r zlexcount zset \[ele \[h]
+
+            # exclusive range
+            assert_equal {alpha bar} [r zrangebylex zset - (cool]
+            assert_equal {cool} [r zrangebylex zset (bar (down]
+            assert_equal {hill omega} [r zrangebylex zset (great +]
+            assert_equal {bar alpha} [r zrevrangebylex zset (cool -]
+            assert_equal {cool} [r zrevrangebylex zset (down (bar]
+            assert_equal {omega hill} [r zrevrangebylex zset + (great]
+            assert_equal 2 [r zlexcount zset (ele (great]
+
+            # inclusive and exclusive
+            assert_equal {} [r zrangebylex zset (az (b]
+            assert_equal {} [r zrangebylex zset (z +]
+            assert_equal {} [r zrangebylex zset - \[aaaa]
+            assert_equal {} [r zrevrangebylex zset \[elez \[elex]
+            assert_equal {} [r zrevrangebylex zset (hill (omega]
+        }
+
+        test "ZRANGEBYSLEX with LIMIT" {
+            create_default_lex_zset
+            assert_equal {alpha bar} [r zrangebylex zset - \[cool LIMIT 0 2]
+            assert_equal {bar cool} [r zrangebylex zset - \[cool LIMIT 1 2]
+            assert_equal {} [r zrangebylex zset \[bar \[down LIMIT 0 0]
+            assert_equal {} [r zrangebylex zset \[bar \[down LIMIT 2 0]
+            assert_equal {bar} [r zrangebylex zset \[bar \[down LIMIT 0 1]
+            assert_equal {cool} [r zrangebylex zset \[bar \[down LIMIT 1 1]
+            assert_equal {bar cool down} [r zrangebylex zset \[bar \[down LIMIT 0 100]
+            assert_equal {omega hill great foo elephant} [r zrevrangebylex zset + \[d LIMIT 0 5]
+            assert_equal {omega hill great foo} [r zrevrangebylex zset + \[d LIMIT 0 4]
+        }
+
+        test "ZRANGEBYLEX with invalid lex range specifiers" {
+            assert_error "*not*string*" {r zrangebylex fooz foo bar}
+            assert_error "*not*string*" {r zrangebylex fooz \[foo bar}
+            assert_error "*not*string*" {r zrangebylex fooz foo \[bar}
+            assert_error "*not*string*" {r zrangebylex fooz +x \[bar}
+            assert_error "*not*string*" {r zrangebylex fooz -x \[bar}
+        }
+
         test "ZREMRANGEBYSCORE basics" {
             proc remrangebyscore {min max} {
                 create_zset zset {1 a 2 b 3 c 4 d 5 e}
@@ -708,6 +764,111 @@ start_server {tags {"zset"}} {
             assert_equal {} $err
         }
 
+        test "ZRANGEBYLEX fuzzy test, 100 ranges in $elements element sorted set - $encoding" {
+            set lexset {}
+            r del zset
+            for {set j 0} {$j < $elements} {incr j} {
+                set e [randstring 0 30 alpha]
+                lappend lexset $e
+                r zadd zset 0 $e
+            }
+            set lexset [lsort -unique $lexset]
+            for {set j 0} {$j < 100} {incr j} {
+                set min [randstring 0 30 alpha]
+                set max [randstring 0 30 alpha]
+                set mininc [randomInt 2]
+                set maxinc [randomInt 2]
+                if {$mininc} {set cmin "\[$min"} else {set cmin "($min"}
+                if {$maxinc} {set cmax "\[$max"} else {set cmax "($max"}
+                set rev [randomInt 2]
+                if {$rev} {
+                    set cmd zrevrangebylex
+                } else {
+                    set cmd zrangebylex
+                }
+
+                # Make sure data is the same in both sides
+                assert {[r zrange zset 0 -1] eq $lexset}
+
+                # Get the Redis output
+                set output [r $cmd zset $cmin $cmax]
+                if {$rev} {
+                    set outlen [r zlexcount zset $cmax $cmin]
+                } else {
+                    set outlen [r zlexcount zset $cmin $cmax]
+                }
+
+                # Compute the same output via Tcl
+                set o {}
+                set copy $lexset
+                if {(!$rev && [string compare $min $max] > 0) ||
+                    ($rev && [string compare $max $min] > 0)} {
+                    # Empty output when ranges are inverted.
+                } else {
+                    if {$rev} {
+                        # Invert the Tcl array using Redis itself.
+                        set copy [r zrevrange zset 0 -1]
+                        # Invert min / max as well
+                        lassign [list $min $max $mininc $maxinc] \
+                            max min maxinc mininc
+                    }
+                    foreach e $copy {
+                        set mincmp [string compare $e $min]
+                        set maxcmp [string compare $e $max]
+                        if {
+                             ($mininc && $mincmp >= 0 || !$mininc && $mincmp > 0)
+                             &&
+                             ($maxinc && $maxcmp <= 0 || !$maxinc && $maxcmp < 0)
+                        } {
+                            lappend o $e
+                        }
+                    }
+                }
+                assert {$o eq $output}
+                assert {$outlen eq [llength $output]}
+            }
+        }
+
+        test "ZREMRANGEBYLEX fuzzy test, 100 ranges in $elements element sorted set - $encoding" {
+            set lexset {}
+            r del zset zsetcopy
+            for {set j 0} {$j < $elements} {incr j} {
+                set e [randstring 0 30 alpha]
+                lappend lexset $e
+                r zadd zset 0 $e
+            }
+            set lexset [lsort -unique $lexset]
+            for {set j 0} {$j < 100} {incr j} {
+                # Copy...
+                r zunionstore zsetcopy 1 zset
+                set lexsetcopy $lexset
+
+                set min [randstring 0 30 alpha]
+                set max [randstring 0 30 alpha]
+                set mininc [randomInt 2]
+                set maxinc [randomInt 2]
+                if {$mininc} {set cmin "\[$min"} else {set cmin "($min"}
+                if {$maxinc} {set cmax "\[$max"} else {set cmax "($max"}
+
+                # Make sure data is the same in both sides
+                assert {[r zrange zset 0 -1] eq $lexset}
+
+                # Get the range we are going to remove
+                set torem [r zrangebylex zset $cmin $cmax]
+                set toremlen [r zlexcount zset $cmin $cmax]
+                r zremrangebylex zsetcopy $cmin $cmax
+                set output [r zrange zsetcopy 0 -1]
+
+                # Remove the range with Tcl from the original list
+                if {$toremlen} {
+                    set first [lsearch -exact $lexsetcopy [lindex $torem 0]]
+                    set last [expr {$first+$toremlen-1}]
+                    set lexsetcopy [lreplace $lexsetcopy $first $last]
+                }
+                assert {$lexsetcopy eq $output}
+            }
+        }
+
         test "ZSETs skiplist implementation backlink consistency test - $encoding" {
             set diff 0
             for {set j 0} {$j < $elements} {incr j} {

utils/hyperloglog/.gitignore

@@ -0,0 +1 @@
+*.txt

utils/hyperloglog/hll-err.rb

@@ -0,0 +1,27 @@
+# hll-err.rb - Copyright (C) 2014 Salvatore Sanfilippo
+# BSD license, See the COPYING file for more information.
+#
+# Check error of HyperLogLog Redis implementation for different set sizes.
+
+require 'rubygems'
+require 'redis'
+require 'digest/sha1'
+
+r = Redis.new
+r.del('hll')
+i = 0
+while true do
+    100.times {
+        elements = []
+        1000.times {
+            ele = Digest::SHA1.hexdigest(i.to_s)
+            elements << ele
+            i += 1
+        }
+        r.pfadd('hll',*elements)
+    }
+    approx = r.pfcount('hll')
+    abs_err = (approx-i).abs
+    rel_err = 100.to_f*abs_err/i
+    puts "#{i} vs #{approx}: #{rel_err}%"
+end

utils/hyperloglog/hll-gnuplot-graph.rb

@@ -0,0 +1,88 @@
+# hll-err.rb - Copyright (C) 2014 Salvatore Sanfilippo
+# BSD license, See the COPYING file for more information.
+#
+# This program is suited to output average and maximum errors of
+# the Redis HyperLogLog implementation in a format suitable to print
+# graphs using gnuplot.
+
+require 'rubygems'
+require 'redis'
+require 'digest/sha1'
+
+# Generate an array of [cardinality,relative_error] pairs
+# in the 0 - max range, with the specified step.
+#
+# 'r' is the Redis object used to perform the queries.
+# 'seed' must be different every time you want a test performed
+# with a different set. The function guarantees that if 'seed' is the
+# same, exactly the same dataset is used, and when it is different,
+# a totally unrelated different data set is used (without any common
+# element in practice).
+def run_experiment(r,seed,max,step)
+    r.del('hll')
+    i = 0
+    samples = []
+    step = 1000 if step > 1000
+    while i < max do
+        elements = []
+        step.times {
+            ele = Digest::SHA1.hexdigest(i.to_s+seed.to_s)
+            elements << ele
+            i += 1
+        }
+        r.pfadd('hll',*elements)
+        approx = r.pfcount('hll')
+        err = approx-i
+        rel_err = 100.to_f*err/i
+        samples << [i,rel_err]
+    end
+    samples
+end
+
+def filter_samples(numsets,max,step,filter)
+    r = Redis.new
+    dataset = {}
+    (0...numsets).each{|i|
+        dataset[i] = run_experiment(r,i,max,step)
+        STDERR.puts "Set #{i}"
+    }
+    dataset[0].each_with_index{|ele,index|
+        if filter == :max
+            card=ele[0]
+            err=ele[1].abs
+            (1...numsets).each{|i|
+                err = dataset[i][index][1] if err < dataset[i][index][1]
+            }
+            puts "#{card} #{err}"
+        elsif filter == :avg
+            card=ele[0]
+            err = 0
+            (0...numsets).each{|i|
+                err += dataset[i][index][1]
+            }
+            err /= numsets
+            puts "#{card} #{err}"
+        elsif filter == :absavg
+            card=ele[0]
+            err = 0
+            (0...numsets).each{|i|
+                err += dataset[i][index][1].abs
+            }
+            err /= numsets
+            puts "#{card} #{err}"
+        elsif filter == :all
+            (0...numsets).each{|i|
+                card,err = dataset[i][index]
+                puts "#{card} #{err}"
+            }
+        else
+            raise "Unknown filter #{filter}"
+        end
+    }
+end
+
+if ARGV.length != 4
+    puts "Usage: hll-gnuplot-graph <samples> <max> <step> (max|avg|absavg|all)"
+    exit 1
+end
+filter_samples(ARGV[0].to_i,ARGV[1].to_i,ARGV[2].to_i,ARGV[3].to_sym)