Redis/redis-For-Windows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

integrated most of changes from redis/4.0.2 onto win-3.2.100 source

Browse Source 
- TODO: modules support (currently turned off), fix in fork() implementation that crashes at the moment, update of libraries in "deps" folder
This commit is contained in:
Tomasz Poradowski
2017-10-27 13:13:09 +02:00
parent 26accdfb00
commit 0267dd21c3
111 changed files with 20205 additions and 6647 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deps/hiredis/README.md
Vendored
+2 -2
View File
@@ -98,7 +98,7 @@ was received:
* **`REDIS_REPLY_INTEGER`**:
* The command replied with an integer. The integer value can be accessed using the
`reply->integer` field of type `long long`.
`reply->integer` field of type `PORT_LONGLONG`.
* **`REDIS_REPLY_NIL`**:
* The command replied with a **nil** object. There is no data to access.
@@ -300,7 +300,7 @@ An asynchronous connection can be terminated using:
void redisAsyncDisconnect(redisAsyncContext *ac);
When this function is called, the connection is **not** immediately terminated. Instead, new
commands are no longer accepted and the connection is only terminated when all pending commands
commands are no PORT_LONGer accepted and the connection is only terminated when all pending commands
have been written to the socket, their respective replies have been read and their respective
callbacks have been executed. After this, the disconnection callback is executed with the
`REDIS_OK` status and the context object is free'd.
deps/hiredis/adapters/libevent.h
Vendored
+12 -12
View File
@@ -30,13 +30,13 @@
#ifndef __HIREDIS_LIBEVENT_H__
#define __HIREDIS_LIBEVENT_H__
#include <event.h>
#include <event2/event.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct redisLibeventEvents {
redisAsyncContext *context;
struct event rev, wev;
struct event *rev, *wev;
} redisLibeventEvents;
static void redisLibeventReadEvent(int fd, short event, void *arg) {
@@ -53,28 +53,28 @@ static void redisLibeventWriteEvent(int fd, short event, void *arg) {
static void redisLibeventAddRead(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_add(&e->rev,NULL);
event_add(e->rev,NULL);
}
static void redisLibeventDelRead(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_del(&e->rev);
event_del(e->rev);
}
static void redisLibeventAddWrite(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_add(&e->wev,NULL);
event_add(e->wev,NULL);
}
static void redisLibeventDelWrite(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_del(&e->wev);
event_del(e->wev);
}
static void redisLibeventCleanup(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_del(&e->rev);
event_del(&e->wev);
event_del(e->rev);
event_del(e->wev);
free(e);
}
@@ -99,10 +99,10 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) {
ac->ev.data = e;
/* Initialize and install read/write events */
event_set(&e->rev,c->fd,EV_READ,redisLibeventReadEvent,e);
event_set(&e->wev,c->fd,EV_WRITE,redisLibeventWriteEvent,e);
event_base_set(base,&e->rev);
event_base_set(base,&e->wev);
e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e);
e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e);
event_add(e->rev, NULL);
event_add(e->wev, NULL);
return REDIS_OK;
}
#endif
deps/hiredis/adapters/libuv.h
Vendored
+4 -3
View File
@@ -1,5 +1,6 @@
#ifndef __HIREDIS_LIBUV_H__
#define __HIREDIS_LIBUV_H__
#include <stdlib.h>
#include <uv.h>
#include "../hiredis.h"
#include "../async.h"
@@ -11,7 +12,6 @@ typedef struct redisLibuvEvents {
int events;
} redisLibuvEvents;
int redisLibuvAttach(redisAsyncContext*, uv_loop_t*);
static void redisLibuvPoll(uv_poll_t* handle, int status, int events) {
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
@@ -20,10 +20,10 @@ static void redisLibuvPoll(uv_poll_t* handle, int status, int events) {
return;
}
if (events & UV_READABLE) {
if (p->context != NULL && (events & UV_READABLE)) {
redisAsyncHandleRead(p->context);
}
if (events & UV_WRITABLE) {
if (p->context != NULL && (events & UV_WRITABLE)) {
redisAsyncHandleWrite(p->context);
}
}
@@ -83,6 +83,7 @@ static void on_close(uv_handle_t* handle) {
static void redisLibuvCleanup(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->context = NULL; // indicate that context might no PORT_LONGer exist
uv_close((uv_handle_t*)&p->handle, on_close);
}
deps/hiredis/async.c
Vendored
+514 -479
View File
File diff suppressed because it is too large Load Diff
deps/hiredis/async.h
Vendored
+4 -1
View File
@@ -86,7 +86,7 @@ typedef struct redisAsyncContext {
} ev;
/* Called when either the connection is terminated due to an error or per
* user request. The status is set accordingly (C_OK, C_ERR). */
* user request. The status is set accordingly (REDIS_OK, REDIS_ERR). */
redisDisconnectCallback *onDisconnect;
/* Called when the first write event was received. */
@@ -106,6 +106,8 @@ typedef struct redisAsyncContext {
/* Functions that proxy to hiredis */
redisAsyncContext *redisAsyncConnect(const char *ip, int port);
redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr);
redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
const char *source_addr);
redisAsyncContext *redisAsyncConnectUnix(const char *path);
int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn);
int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
@@ -125,6 +127,7 @@ int redisAsyncHandleWriteComplete(redisAsyncContext *ac, int written);
int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap);
int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...);
int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen);
int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len);
#ifdef __cplusplus
}
deps/hiredis/dict.c
Vendored
+2 -2
View File
@@ -164,7 +164,7 @@ static int dictReplace(dict *ht, void *key, void *val) {
dictEntry *entry, auxentry;
/* Try to add the element. If the key
* does not exists dictAdd will suceed. */
* does not exists dictAdd will succeed. */
if (dictAdd(ht, key, val) == DICT_OK)
return 1;
/* It already exists, get the entry */
@@ -296,7 +296,7 @@ static void dictReleaseIterator(dictIterator *iter) {
/* Expand the hash table if needed */
static int _dictExpandIfNeeded(dict *ht) {
/* If the hash table is empty expand it to the intial size,
/* If the hash table is empty expand it to the initial size,
* if the table is "full" dobule its size. */
if (ht->size == 0)
return dictExpand(ht, DICT_HT_INITIAL_SIZE);
deps/hiredis/fmacros.h
Vendored
+8 -7
View File
@@ -1,23 +1,24 @@
#ifndef __HIREDIS_FMACRO_H
#define __HIREDIS_FMACRO_H
#if !defined(_BSD_SOURCE)
#if defined(__linux__)
#define _BSD_SOURCE
#define _DEFAULT_SOURCE
#endif
#if defined(_AIX)
#define _ALL_SOURCE
#if defined(__CYGWIN__)
#include <sys/cdefs.h>
#endif
#if defined(__sun__)
#define _POSIX_C_SOURCE 200112L
#elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__)
#define _XOPEN_SOURCE 600
#else
#define _XOPEN_SOURCE
#if !(defined(__APPLE__) && defined(__MACH__))
#define _XOPEN_SOURCE 600
#endif
#endif
#if __APPLE__ && __MACH__
#if defined(__APPLE__) && defined(__MACH__)
#define _OSX
#endif
deps/hiredis/hiredis.c
Vendored
+1073 -1001
View File
File diff suppressed because it is too large Load Diff
deps/hiredis/hiredis.h
Vendored
+77 -73
View File
@@ -1,6 +1,8 @@
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
* Jan-Erik Rediger <janerik at fnordig dot com>
*
* All rights reserved.
*
@@ -31,6 +33,7 @@
#ifndef __HIREDIS_H
#define __HIREDIS_H
#include "read.h"
#include <stdio.h> /* for size_t */
#include <stdarg.h> /* for va_list */
#ifndef _WIN32
@@ -38,23 +41,13 @@
#else
#include "../../src/Win32_Interop/win32_types_hiredis.h"
#endif
#include <stdint.h> /* uintXX_t, etc */
#include "sds.h" /* for sds */
#define HIREDIS_MAJOR 0
#define HIREDIS_MINOR 11
#define HIREDIS_PATCH 0
#define REDIS_ERR -1
#define REDIS_OK 0
/* When an error occurs, the err flag in a context is set to hold the type of
* error that occured. REDIS_ERR_IO means there was an I/O error and you
* should use the "errno" variable to find out what is wrong.
* For other values, the "errstr" field will hold a description. */
#define REDIS_ERR_IO 1 /* Error in read or write */
#define REDIS_ERR_EOF 3 /* End of file */
#define REDIS_ERR_PROTOCOL 4 /* Protocol error */
#define REDIS_ERR_OOM 5 /* Out of memory */
#define REDIS_ERR_OTHER 2 /* Everything else... */
#define HIREDIS_MINOR 13
#define HIREDIS_PATCH 3
#define HIREDIS_SONAME 0.13
/* Connection type can be blocking or non-blocking and is set in the
* least significant bit of the flags field in redisContext. */
@@ -83,17 +76,39 @@
/* Flag that is set when monitor mode is active */
#define REDIS_MONITORING 0x40
#define REDIS_REPLY_STRING 1
#define REDIS_REPLY_ARRAY 2
#define REDIS_REPLY_INTEGER 3
#define REDIS_REPLY_NIL 4
#define REDIS_REPLY_STATUS 5
#define REDIS_REPLY_ERROR 6
#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */
/* Flag that is set when we should set SO_REUSEADDR before calling bind() */
#define REDIS_REUSEADDR 0x80
#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */
/* number of times we retry to connect in the case of EADDRNOTAVAIL and
* SO_REUSEADDR is being used. */
#define REDIS_CONNECT_RETRIES 10
/* strerror_r has two completely different prototypes and behaviors
* depending on system issues, so we need to operate on the error buffer
* differently depending on which strerror_r we're using. */
#ifndef _GNU_SOURCE
/* "regular" POSIX strerror_r that does the right thing. */
#define __redis_strerror_r(errno, buf, len) \
do { \
strerror_r((errno), (buf), (len)); \
} while (0)
#else
/* "bad" GNU strerror_r we need to clean up after. */
#define __redis_strerror_r(errno, buf, len) \
do { \
char *err_str = strerror_r((errno), (buf), (len)); \
/* If return value _isn't_ the start of the buffer we passed in, \
* then GNU strerror_r returned an internal static buffer and we \
* need to copy the result into our private buffer. */ \
if (err_str != (buf)) { \
strncpy((buf), err_str, ((len) - 1)); \
buf[(len)-1] = '\0'; \
} \
} while (0)
#endif
#ifdef __cplusplus
extern "C" {
#endif
@@ -108,55 +123,7 @@ typedef struct redisReply {
struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */
} redisReply;
typedef struct redisReadTask {
int type;
int elements; /* number of elements in multibulk container */
int idx; /* index in parent (array) object */
void *obj; /* holds user-generated value for a read task */
struct redisReadTask *parent; /* parent task */
void *privdata; /* user-settable arbitrary field */
} redisReadTask;
typedef struct redisReplyObjectFunctions {
void *(*createString)(const redisReadTask*, char*, size_t);
void *(*createArray)(const redisReadTask*, int);
void *(*createInteger)(const redisReadTask*, PORT_LONGLONG);
void *(*createNil)(const redisReadTask*);
void (*freeObject)(void*);
} redisReplyObjectFunctions;
/* State for the protocol parser */
typedef struct redisReader {
int err; /* Error flags, 0 when there is no error */
char errstr[128]; /* String representation of error when applicable */
char *buf; /* Read buffer */
size_t pos; /* Buffer cursor */
size_t len; /* Buffer length */
size_t maxbuf; /* Max length of unused buffer */
redisReadTask rstack[9];
int ridx; /* Index of current read task */
void *reply; /* Temporary reply pointer */
redisReplyObjectFunctions *fn;
void *privdata;
} redisReader;
/* Public API for the protocol parser. */
redisReader *redisReaderCreate(void);
void redisReaderFree(redisReader *r);
int redisReaderFeed(redisReader *r, const char *buf, size_t len);
int redisReaderGetReply(redisReader *r, void **reply);
/* Backwards compatibility, can be removed on big version bump. */
#define redisReplyReaderCreate redisReaderCreate
#define redisReplyReaderFree redisReaderFree
#define redisReplyReaderFeed redisReaderFeed
#define redisReplyReaderGetReply redisReaderGetReply
#define redisReplyReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p))
#define redisReplyReaderGetObject(_r) (((redisReader*)(_r))->reply)
#define redisReplyReaderGetError(_r) (((redisReader*)(_r))->errstr)
/* Function to free the reply objects hiredis returns by default. */
void freeReplyObject(void *reply);
@@ -165,6 +132,14 @@ void freeReplyObject(void *reply);
int redisvFormatCommand(char **target, const char *format, va_list ap);
int redisFormatCommand(char **target, const char *format, ...);
int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen);
int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen);
void redisFreeCommand(char *cmd);
void redisFreeSdsCommand(sds cmd);
enum redisConnectionType {
REDIS_CONN_TCP,
REDIS_CONN_UNIX
};
/* Context for a connection to Redis */
typedef struct redisContext {
@@ -174,16 +149,45 @@ typedef struct redisContext {
int flags;
char *obuf; /* Write buffer */
redisReader *reader; /* Protocol reader */
enum redisConnectionType connection_type;
struct timeval *timeout;
struct {
char *host;
char *source_addr;
int port;
} tcp;
struct {
char *path;
} unix_sock;
} redisContext;
redisContext *redisConnect(const char *ip, int port);
redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv);
redisContext *redisConnectNonBlock(const char *ip, int port);
redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr);
redisContext *redisConnectBindNonBlock(const char *ip, int port,
const char *source_addr);
redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
const char *source_addr);
redisContext *redisConnectUnix(const char *path);
redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv);
redisContext *redisConnectUnixNonBlock(const char *path);
redisContext *redisConnectFd(int fd);
/**
* Reconnect the given context using the saved information.
*
* This re-uses the exact same connect options as in the initial connection.
* host, ip (or path), timeout and bind address are reused,
* flags are used unmodified from the existing context.
*
* Returns REDIS_OK on successful connect or REDIS_ERR otherwise.
*/
int redisReconnect(redisContext *c);
int redisSetTimeout(redisContext *c, const struct timeval tv);
int redisEnableKeepAlive(redisContext *c);
void redisFree(redisContext *c);
deps/hiredis/net.c
Vendored
+396 -293
View File
@@ -1,7 +1,9 @@
/* Extracted from anet.c to work properly with Hiredis error reporting.
*
* Copyright (c) 2006-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
* Jan-Erik Rediger <janerik at fnordig dot com>
*
* All rights reserved.
*
@@ -58,397 +60,498 @@
#include "net.h"
#include "sds.h"
#ifdef _WIN32
#include "win32_hiredis.h"
#include "mstcpip.h"
#include "win32_hiredis.h"
#include "mstcpip.h"
#endif
/* Defined in hiredis.c */
/* Defined in hiredis.c */
void __redisSetError(redisContext *c, int type, const char *str);
static void redisContextCloseFd(redisContext *c) {
if (c && c->fd >= 0) {
close(c->fd);
c->fd = -1;
}
if (c && c->fd >= 0) {
close(c->fd);
c->fd = -1;
}
}
static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) {
char buf[128] = { 0 };
size_t len = 0;
char buf[128] = { 0 };
size_t len = 0;
if (prefix != NULL)
len = snprintf(buf,sizeof(buf),"%s: ",prefix);
strerror_r(errno,buf+len,sizeof(buf)-len);
__redisSetError(c,type,buf);
if (prefix != NULL)
len = snprintf(buf, sizeof(buf), "%s: ", prefix);
__redis_strerror_r(errno, (char *)(buf + len), sizeof(buf) - len);
__redisSetError(c, type, buf);
}
static int redisSetReuseAddr(redisContext *c) {
int on = 1;
if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
__redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
redisContextCloseFd(c);
return REDIS_ERR;
}
return REDIS_OK;
int on = 1;
if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL);
redisContextCloseFd(c);
return REDIS_ERR;
}
return REDIS_OK;
}
static int redisCreateSocket(redisContext *c, int type) {
int s;
if ((s = socket(type, SOCK_STREAM, 0)) == -1) {
__redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
return REDIS_ERR;
}
c->fd = s;
if (type == AF_INET) {
if (redisSetReuseAddr(c) == REDIS_ERR) {
return REDIS_ERR;
}
}
return REDIS_OK;
int s;
if ((s = socket(type, SOCK_STREAM, 0)) == -1) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL);
return REDIS_ERR;
}
c->fd = s;
if (type == AF_INET) {
if (redisSetReuseAddr(c) == REDIS_ERR) {
return REDIS_ERR;
}
}
return REDIS_OK;
}
static int redisSetBlocking(redisContext *c, int blocking) {
int flags;
int flags;
/* Set the socket nonblocking.
* Note that fcntl(2) for F_GETFL and F_SETFL can't be
* interrupted by a signal. */
if ((flags = fcntl(c->fd, F_GETFL,0)) == -1) {
__redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)");
redisContextCloseFd(c);
return REDIS_ERR;
}
/* Set the socket nonblocking.
* Note that fcntl(2) for F_GETFL and F_SETFL can't be
* interrupted by a signal. */
if ((flags = fcntl(c->fd, F_GETFL, 0)) == -1) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, "fcntl(F_GETFL)");
redisContextCloseFd(c);
return REDIS_ERR;
}
if (blocking)
flags &= ~O_NONBLOCK;
else
flags |= O_NONBLOCK;
if (blocking)
flags &= ~O_NONBLOCK;
else
flags |= O_NONBLOCK;
if (fcntl(c->fd, F_SETFL, flags) == -1) {
__redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)");
redisContextCloseFd(c);
return REDIS_ERR;
}
return REDIS_OK;
if (fcntl(c->fd, F_SETFL, flags) == -1) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, "fcntl(F_SETFL)");
redisContextCloseFd(c);
return REDIS_ERR;
}
return REDIS_OK;
}
int redisKeepAlive(redisContext *c, int interval) {
int val = 1;
int fd = c->fd;
int val = 1;
int fd = c->fd;
if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){
__redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
return REDIS_ERR;
}
if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1) {
__redisSetError(c, REDIS_ERR_OTHER, strerror(errno));
return REDIS_ERR;
}
val = interval;
val = interval;
#ifdef _OSX
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) {
__redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
return REDIS_ERR;
}
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &val, sizeof(val)) < 0) {
__redisSetError(c, REDIS_ERR_OTHER, strerror(errno));
return REDIS_ERR;
}
#else
#ifndef __sun
#ifdef _WIN32
{
struct tcp_keepalive settings;
DWORD bytesReturned;
WSAOVERLAPPED overlapped;
settings.onoff = 1;
settings.keepalivetime = interval*1000;
settings.keepaliveinterval = interval*1000/3;
overlapped.hEvent = NULL;
FDAPI_WSAIoctl(fd,
SIO_KEEPALIVE_VALS,
&settings,
sizeof(struct tcp_keepalive),
NULL,
0,
&bytesReturned,
&overlapped,
NULL);
}
{
struct tcp_keepalive settings;
DWORD bytesReturned;
WSAOVERLAPPED overlapped;
settings.onoff = 1;
settings.keepalivetime = interval * 1000;
settings.keepaliveinterval = interval * 1000 / 3;
overlapped.hEvent = NULL;
FDAPI_WSAIoctl(fd,
SIO_KEEPALIVE_VALS,
&settings,
sizeof(struct tcp_keepalive),
NULL,
0,
&bytesReturned,
&overlapped,
NULL);
}
#else
val = interval;
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) {
__redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
return REDIS_ERR;
}
val = interval;
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) {
__redisSetError(c, REDIS_ERR_OTHER, strerror(errno));
return REDIS_ERR;
}
val = interval/3;
if (val == 0) val = 1;
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) {
__redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
return REDIS_ERR;
}
val = interval / 3;
if (val == 0) val = 1;
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) {
__redisSetError(c, REDIS_ERR_OTHER, strerror(errno));
return REDIS_ERR;
}
val = 3;
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) {
__redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
return REDIS_ERR;
}
val = 3;
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) {
__redisSetError(c, REDIS_ERR_OTHER, strerror(errno));
return REDIS_ERR;
}
#endif
#endif
#endif
return REDIS_OK;
return REDIS_OK;
}
static int redisSetTcpNoDelay(redisContext *c) {
int yes = 1;
if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) {
__redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)");
redisContextCloseFd(c);
return REDIS_ERR;
}
return REDIS_OK;
int yes = 1;
if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, "setsockopt(TCP_NODELAY)");
redisContextCloseFd(c);
return REDIS_ERR;
}
return REDIS_OK;
}
#define __MAX_MSEC (((LONG_MAX) - 999) / 1000)
static int redisContextWaitReady(redisContext *c, const struct timeval *timeout) {
struct pollfd wfd[1];
PORT_LONG msec;
static int redisContextTimeoutMsec(redisContext *c, PORT_LONG *result)
{
const struct timeval *timeout = c->timeout;
PORT_LONG msec = -1;
msec = -1;
wfd[0].fd = c->fd;
wfd[0].events = POLLOUT;
/* Only use timeout when not NULL. */
if (timeout != NULL) {
if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) {
*result = msec;
return REDIS_ERR;
}
/* Only use timeout when not NULL. */
if (timeout != NULL) {
if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL);
redisContextCloseFd(c);
return REDIS_ERR;
}
msec = (timeout->tv_sec * 1000) + ((timeout->tv_usec + 999) / 1000);
msec = (timeout->tv_sec * 1000) + ((timeout->tv_usec + 999) / 1000);
if (msec < 0 || msec > INT_MAX) {
msec = INT_MAX;
}
}
if (msec < 0 || msec > INT_MAX) {
msec = INT_MAX;
}
}
*result = msec;
return REDIS_OK;
}
if (errno == EINPROGRESS) {
int res;
static int redisContextWaitReady(redisContext *c, PORT_LONG msec) {
struct pollfd wfd[1];
if ((res = poll(wfd, 1, (int) msec)) == -1) { WIN_PORT_FIX /* cast (int) */
__redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)");
redisContextCloseFd(c);
return REDIS_ERR;
} else if (res == 0) {
errno = ETIMEDOUT;
__redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
redisContextCloseFd(c);
return REDIS_ERR;
}
wfd[0].fd = c->fd;
wfd[0].events = POLLOUT;
if (redisCheckSocketError(c) != REDIS_OK)
return REDIS_ERR;
if (errno == EINPROGRESS) {
int res;
return REDIS_OK;
}
if ((res = poll(wfd, 1, (int)msec)) == -1) {
WIN_PORT_FIX /* cast (int) */
__redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)");
redisContextCloseFd(c);
return REDIS_ERR;
}
else if (res == 0) {
errno = ETIMEDOUT;
__redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL);
redisContextCloseFd(c);
return REDIS_ERR;
}
__redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
redisContextCloseFd(c);
return REDIS_ERR;
if (redisCheckSocketError(c) != REDIS_OK)
return REDIS_ERR;
return REDIS_OK;
}
__redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL);
redisContextCloseFd(c);
return REDIS_ERR;
}
int redisCheckSocketError(redisContext *c) {
int err = 0;
socklen_t errlen = sizeof(err);
int err = 0;
socklen_t errlen = sizeof(err);
if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) {
__redisSetErrorFromErrno(c,REDIS_ERR_IO,"getsockopt(SO_ERROR)");
return REDIS_ERR;
}
if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, &err, &errlen) == -1) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, "getsockopt(SO_ERROR)");
return REDIS_ERR;
}
if (err) {
errno = err;
__redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL);
return REDIS_ERR;
}
if (err) {
errno = err;
__redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL);
return REDIS_ERR;
}
return REDIS_OK;
return REDIS_OK;
}
int redisContextSetTimeout(redisContext *c, const struct timeval tv) {
if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) {
__redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)");
return REDIS_ERR;
}
if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) {
__redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)");
return REDIS_ERR;
}
return REDIS_OK;
if (setsockopt(c->fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) == -1) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, "setsockopt(SO_RCVTIMEO)");
return REDIS_ERR;
}
if (setsockopt(c->fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) == -1) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, "setsockopt(SO_SNDTIMEO)");
return REDIS_ERR;
}
return REDIS_OK;
}
#ifdef _WIN32
int redisContextPreConnectTcp(
redisContext *c,
const char *addr,
int port,
struct timeval *timeout,
SOCKADDR_STORAGE* ss) {
int blocking = (c->flags & REDIS_BLOCK);
redisContext *c,
const char *addr,
int port,
struct timeval *timeout,
SOCKADDR_STORAGE* ss) {
int blocking = (c->flags & REDIS_BLOCK);
if (ParseStorageAddress(addr, port, ss) == FALSE) {
return REDIS_ERR;
}
if (ParseStorageAddress(addr, port, ss) == FALSE) {
return REDIS_ERR;
}
if (REDIS_OK != redisCreateSocket(c, ss->ss_family)) {
return REDIS_ERR;
}
if (REDIS_OK != redisCreateSocket(c, ss->ss_family)) {
return REDIS_ERR;
}
if (redisSetTcpNoDelay(c) != REDIS_OK)
return REDIS_ERR;
if (redisSetTcpNoDelay(c) != REDIS_OK)
return REDIS_ERR;
if (blocking == 0) {
if (redisSetBlocking(c, 0) != REDIS_OK)
return REDIS_ERR;
}
if (blocking == 0) {
if (redisSetBlocking(c, 0) != REDIS_OK)
return REDIS_ERR;
}
return REDIS_OK;
return REDIS_OK;
}
#endif
static int _redisContextConnectTcp(redisContext *c, const char *addr, int port,
const struct timeval *timeout,
const char *source_addr) {
int s, rv;
char _port[6]; /* strlen("65535"); */
struct addrinfo hints, *servinfo, *bservinfo, *p, *b;
int blocking = (c->flags & REDIS_BLOCK);
const struct timeval *timeout,
const char *source_addr) {
int s, rv, n;
char _port[6]; /* strlen("65535"); */
struct addrinfo hints, *servinfo, *bservinfo, *p, *b;
int blocking = (c->flags & REDIS_BLOCK);
int reuseaddr = (c->flags & REDIS_REUSEADDR);
int reuses = 0;
PORT_LONG timeout_msec = -1;
snprintf(_port, 6, "%d", port);
memset(&hints,0,sizeof(hints));
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
servinfo = NULL;
c->connection_type = REDIS_CONN_TCP;
c->tcp.port = port;
/* Try with IPv6 if no IPv4 address was found. We do it in this order since
* in a Redis client you can't afford to test if you have IPv6 connectivity
* as this would add latency to every connect. Otherwise a more sensible
* route could be: Use IPv6 if both addresses are available and there is IPv6
* connectivity. */
if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) {
hints.ai_family = AF_INET6;
if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) {
__redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv));
return REDIS_ERR;
}
}
for (p = servinfo; p != NULL; p = p->ai_next) {
if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1)
continue;
/* We need to take possession of the passed parameters
* to make them reusable for a reconnect.
* We also carefully check we don't free data we already own,
* as in the case of the reconnect method.
*
* This is a bit ugly, but atleast it works and doesn't leak memory.
**/
if (c->tcp.host != addr) {
if (c->tcp.host)
free(c->tcp.host);
c->fd = s;
if (redisSetBlocking(c,0) != REDIS_OK)
goto error;
if (source_addr) {
int bound = 0;
/* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */
if ((rv = getaddrinfo(source_addr, NULL, &hints, &bservinfo)) != 0) {
char buf[128];
snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv));
__redisSetError(c,REDIS_ERR_OTHER,buf);
goto error;
}
for (b = bservinfo; b != NULL; b = b->ai_next) {
if (bind(s,b->ai_addr,(socklen_t)b->ai_addrlen) != -1) {
bound = 1;
break;
}
}
freeaddrinfo(bservinfo);
if (!bound) {
char buf[128];
snprintf(buf,sizeof(buf),"Can't bind socket: %s",strerror(errno));
__redisSetError(c,REDIS_ERR_OTHER,buf);
goto error;
}
}
if (connect(s,p->ai_addr,p->ai_addrlen) == -1) {
if (errno == EHOSTUNREACH) {
redisContextCloseFd(c);
continue;
} else if (errno == EINPROGRESS && !blocking) {
/* This is ok. */
} else {
if (redisContextWaitReady(c,timeout) != REDIS_OK)
goto error;
}
}
if (blocking && redisSetBlocking(c,1) != REDIS_OK)
goto error;
if (redisSetTcpNoDelay(c) != REDIS_OK)
goto error;
c->tcp.host = strdup(addr);
}
c->flags |= REDIS_CONNECTED;
rv = REDIS_OK;
goto end;
}
if (p == NULL) {
char buf[128];
snprintf(buf,sizeof(buf),"Can't create socket: %s",strerror(errno));
__redisSetError(c,REDIS_ERR_OTHER,buf);
goto error;
}
if (timeout) {
if (c->timeout != timeout) {
if (c->timeout == NULL)
c->timeout = malloc(sizeof(struct timeval));
memcpy(c->timeout, timeout, sizeof(struct timeval));
}
}
else {
if (c->timeout)
free(c->timeout);
c->timeout = NULL;
}
if (redisContextTimeoutMsec(c, &timeout_msec) != REDIS_OK) {
__redisSetError(c, REDIS_ERR_IO, "Invalid timeout specified");
goto error;
}
if (source_addr == NULL) {
free(c->tcp.source_addr);
c->tcp.source_addr = NULL;
}
else if (c->tcp.source_addr != source_addr) {
free(c->tcp.source_addr);
c->tcp.source_addr = strdup(source_addr);
}
snprintf(_port, 6, "%d", port);
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
/* Try with IPv6 if no IPv4 address was found. We do it in this order since
* in a Redis client you can't afford to test if you have IPv6 connectivity
* as this would add latency to every connect. Otherwise a more sensible
* route could be: Use IPv6 if both addresses are available and there is IPv6
* connectivity. */
if ((rv = getaddrinfo(c->tcp.host, _port, &hints, &servinfo)) != 0) {
hints.ai_family = AF_INET6;
if ((rv = getaddrinfo(addr, _port, &hints, &servinfo)) != 0) {
__redisSetError(c, REDIS_ERR_OTHER, gai_strerror(rv));
return REDIS_ERR;
}
}
for (p = servinfo; p != NULL; p = p->ai_next) {
addrretry:
if ((s = socket(p->ai_family, p->ai_socktype, p->ai_protocol)) == -1)
continue;
c->fd = s;
if (redisSetBlocking(c, 0) != REDIS_OK)
goto error;
if (c->tcp.source_addr) {
int bound = 0;
/* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */
if ((rv = getaddrinfo(c->tcp.source_addr, NULL, &hints, &bservinfo)) != 0) {
char buf[128];
snprintf(buf, sizeof(buf), "Can't get addr: %s", gai_strerror(rv));
__redisSetError(c, REDIS_ERR_OTHER, buf);
goto error;
}
if (reuseaddr) {
n = 1;
if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*)&n,
sizeof(n)) < 0) {
goto error;
}
}
for (b = bservinfo; b != NULL; b = b->ai_next) {
if (bind(s, b->ai_addr, b->ai_addrlen) != -1) {
bound = 1;
break;
}
}
freeaddrinfo(bservinfo);
if (!bound) {
char buf[128];
snprintf(buf, sizeof(buf), "Can't bind socket: %s", strerror(errno));
__redisSetError(c, REDIS_ERR_OTHER, buf);
goto error;
}
}
if (connect(s, p->ai_addr, p->ai_addrlen) == -1) {
if (errno == EHOSTUNREACH) {
redisContextCloseFd(c);
continue;
}
else if (errno == EINPROGRESS && !blocking) {
/* This is ok. */
}
else if (errno == EADDRNOTAVAIL && reuseaddr) {
if (++reuses >= REDIS_CONNECT_RETRIES) {
goto error;
}
else {
redisContextCloseFd(c);
goto addrretry;
}
}
else {
if (redisContextWaitReady(c, timeout_msec) != REDIS_OK)
goto error;
}
}
if (blocking && redisSetBlocking(c, 1) != REDIS_OK)
goto error;
if (redisSetTcpNoDelay(c) != REDIS_OK)
goto error;
c->flags |= REDIS_CONNECTED;
rv = REDIS_OK;
goto end;
}
if (p == NULL) {
char buf[128];
snprintf(buf, sizeof(buf), "Can't create socket: %s", strerror(errno));
__redisSetError(c, REDIS_ERR_OTHER, buf);
goto error;
}
error:
rv = REDIS_ERR;
rv = REDIS_ERR;
end:
freeaddrinfo(servinfo);
return rv; // Need to return REDIS_OK if alright
freeaddrinfo(servinfo);
return rv; // Need to return REDIS_OK if alright
}
int redisContextConnectTcp(redisContext *c, const char *addr, int port,
const struct timeval *timeout) {
return _redisContextConnectTcp(c, addr, port, timeout, NULL);
const struct timeval *timeout) {
return _redisContextConnectTcp(c, addr, port, timeout, NULL);
}
int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
const struct timeval *timeout,
const char *source_addr) {
return _redisContextConnectTcp(c, addr, port, timeout, source_addr);
const struct timeval *timeout,
const char *source_addr) {
return _redisContextConnectTcp(c, addr, port, timeout, source_addr);
}
#ifdef _WIN32
int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) {
(void) timeout;
__redisSetError(c,REDIS_ERR_IO,
sdscatprintf(sdsempty(),"Unix sockets are not suported on Windows platform. (%s)\n", path));
(void)timeout;
__redisSetError(c, REDIS_ERR_IO,
sdscatprintf(sdsempty(), "Unix sockets are not suported on Windows platform. (%s)\n", path));
return REDIS_ERR;
return REDIS_ERR;
}
#else
int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) {
int blocking = (c->flags & REDIS_BLOCK);
struct sockaddr_un sa;
int blocking = (c->flags & REDIS_BLOCK);
struct sockaddr_un sa;
PORT_LONG timeout_msec = -1;
if (redisCreateSocket(c,AF_LOCAL) < 0)
return REDIS_ERR;
if (redisSetBlocking(c,0) != REDIS_OK)
return REDIS_ERR;
if (redisCreateSocket(c, AF_LOCAL) < 0)
return REDIS_ERR;
if (redisSetBlocking(c, 0) != REDIS_OK)
return REDIS_ERR;
sa.sun_family = AF_LOCAL;
strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1);
if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) {
if (errno == EINPROGRESS && !blocking) {
/* This is ok. */
} else {
if (redisContextWaitReady(c,timeout) != REDIS_OK)
return REDIS_ERR;
}
}
c->connection_type = REDIS_CONN_UNIX;
if (c->unix_sock.path != path)
c->unix_sock.path = strdup(path);
/* Reset socket to be blocking after connect(2). */
if (blocking && redisSetBlocking(c,1) != REDIS_OK)
return REDIS_ERR;
if (timeout) {
if (c->timeout != timeout) {
if (c->timeout == NULL)
c->timeout = malloc(sizeof(struct timeval));
c->flags |= REDIS_CONNECTED;
return REDIS_OK;
memcpy(c->timeout, timeout, sizeof(struct timeval));
}
}
else {
if (c->timeout)
free(c->timeout);
c->timeout = NULL;
}
if (redisContextTimeoutMsec(c, &timeout_msec) != REDIS_OK)
return REDIS_ERR;
sa.sun_family = AF_LOCAL;
strncpy(sa.sun_path, path, sizeof(sa.sun_path) - 1);
if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) {
if (errno == EINPROGRESS && !blocking) {
/* This is ok. */
}
else {
if (redisContextWaitReady(c, timeout_msec) != REDIS_OK)
return REDIS_ERR;
}
}
/* Reset socket to be blocking after connect(2). */
if (blocking && redisSetBlocking(c, 1) != REDIS_OK)
return REDIS_ERR;
c->flags |= REDIS_CONNECTED;
return REDIS_OK;
}
#endif
#endif
deps/hiredis/net.h
Vendored
+5 -3
View File
@@ -1,7 +1,9 @@
/* Extracted from anet.c to work properly with Hiredis error reporting.
*
* Copyright (c) 2006-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
* Jan-Erik Rediger <janerik at fnordig dot com>
*
* All rights reserved.
*
@@ -35,7 +37,7 @@
#include "hiredis.h"
#if defined(__sun) || defined(_AIX)
#if defined(__sun)
#define AF_LOCAL AF_UNIX
#endif
deps/hiredis/read.c
Vendored
+525
View File
@@ -0,0 +1,525 @@
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "fmacros.h"
#include <string.h>
#include <stdlib.h>
#ifndef _MSC_VER
#include <unistd.h>
#endif
#include <assert.h>
#include <errno.h>
#include <ctype.h>
#include "read.h"
#include "sds.h"
static void __redisReaderSetError(redisReader *r, int type, const char *str) {
size_t len;
if (r->reply != NULL && r->fn && r->fn->freeObject) {
r->fn->freeObject(r->reply);
r->reply = NULL;
}
/* Clear input buffer on errors. */
if (r->buf != NULL) {
sdsfree(r->buf);
r->buf = NULL;
r->pos = r->len = 0;
}
/* Reset task stack. */
r->ridx = -1;
/* Set error. */
r->err = type;
len = strlen(str);
len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1);
memcpy(r->errstr,str,len);
r->errstr[len] = '\0';
}
static size_t chrtos(char *buf, size_t size, char byte) {
size_t len = 0;
switch(byte) {
case '\\':
case '"':
len = snprintf(buf,size,"\"\\%c\"",byte);
break;
case '\n': len = snprintf(buf,size,"\"\\n\""); break;
case '\r': len = snprintf(buf,size,"\"\\r\""); break;
case '\t': len = snprintf(buf,size,"\"\\t\""); break;
case '\a': len = snprintf(buf,size,"\"\\a\""); break;
case '\b': len = snprintf(buf,size,"\"\\b\""); break;
default:
if (isprint(byte))
len = snprintf(buf,size,"\"%c\"",byte);
else
len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte);
break;
}
return len;
}
static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) {
char cbuf[8], sbuf[128];
chrtos(cbuf,sizeof(cbuf),byte);
snprintf(sbuf,sizeof(sbuf),
"Protocol error, got %s as reply type byte", cbuf);
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf);
}
static void __redisReaderSetErrorOOM(redisReader *r) {
__redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory");
}
static char *readBytes(redisReader *r, unsigned int bytes) {
char *p;
if (r->len-r->pos >= bytes) {
p = r->buf+r->pos;
r->pos += bytes;
return p;
}
return NULL;
}
/* Find pointer to \r\n. */
static char *seekNewline(char *s, size_t len) {
int pos = 0;
int _len = len-1;
/* Position should be < len-1 because the character at "pos" should be
* followed by a \n. Note that strchr cannot be used because it doesn't
* allow to search a limited length and the buffer that is being searched
* might not have a trailing NULL character. */
while (pos < _len) {
while(pos < _len && s[pos] != '\r') pos++;
if (pos==_len) {
/* Not found. */
return NULL;
} else {
if (s[pos+1] == '\n') {
/* Found. */
return s+pos;
} else {
/* Continue searching. */
pos++;
}
}
}
return NULL;
}
/* Read a PORT_LONGLONG value starting at *s, under the assumption that it will be
* terminated by \r\n. Ambiguously returns -1 for unexpected input. */
static PORT_LONGLONG readLongLong(char *s) {
PORT_LONGLONG v = 0;
int dec, mult = 1;
char c;
if (*s == '-') {
mult = -1;
s++;
} else if (*s == '+') {
mult = 1;
s++;
}
while ((c = *(s++)) != '\r') {
dec = c - '0';
if (dec >= 0 && dec < 10) {
v *= 10;
v += dec;
} else {
/* Should not happen... */
return -1;
}
}
return mult*v;
}
static char *readLine(redisReader *r, int *_len) {
char *p, *s;
int len;
p = r->buf+r->pos;
s = seekNewline(p,(r->len-r->pos));
if (s != NULL) {
len = s-(r->buf+r->pos);
r->pos += len+2; /* skip \r\n */
if (_len) *_len = len;
return p;
}
return NULL;
}
static void moveToNextTask(redisReader *r) {
redisReadTask *cur, *prv;
while (r->ridx >= 0) {
/* Return a.s.a.p. when the stack is now empty. */
if (r->ridx == 0) {
r->ridx--;
return;
}
cur = &(r->rstack[r->ridx]);
prv = &(r->rstack[r->ridx-1]);
assert(prv->type == REDIS_REPLY_ARRAY);
if (cur->idx == prv->elements-1) {
r->ridx--;
} else {
/* Reset the type because the next item can be anything */
assert(cur->idx < prv->elements);
cur->type = -1;
cur->elements = -1;
cur->idx++;
return;
}
}
}
static int processLineItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj;
char *p;
int len;
if ((p = readLine(r,&len)) != NULL) {
if (cur->type == REDIS_REPLY_INTEGER) {
if (r->fn && r->fn->createInteger)
obj = r->fn->createInteger(cur,readLongLong(p));
else
obj = (void*)REDIS_REPLY_INTEGER;
} else {
/* Type will be error or status. */
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,p,len);
else
obj = (void*)(size_t)(cur->type);
}
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
/* Set reply if this is the root object. */
if (r->ridx == 0) r->reply = obj;
moveToNextTask(r);
return REDIS_OK;
}
return REDIS_ERR;
}
static int processBulkItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj = NULL;
char *p, *s;
PORT_LONG len;
PORT_ULONG bytelen;
int success = 0;
p = r->buf+r->pos;
s = seekNewline(p,r->len-r->pos);
if (s != NULL) {
p = r->buf+r->pos;
bytelen = s-(r->buf+r->pos)+2; /* include \r\n */
len = readLongLong(p);
if (len < 0) {
/* The nil object can always be created. */
if (r->fn && r->fn->createNil)
obj = r->fn->createNil(cur);
else
obj = (void*)REDIS_REPLY_NIL;
success = 1;
} else {
/* Only continue when the buffer contains the entire bulk item. */
bytelen += len+2; /* include \r\n */
if (r->pos+bytelen <= r->len) {
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,s+2,len);
else
obj = (void*)REDIS_REPLY_STRING;
success = 1;
}
}
/* Proceed when obj was created. */
if (success) {
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
r->pos += bytelen;
/* Set reply if this is the root object. */
if (r->ridx == 0) r->reply = obj;
moveToNextTask(r);
return REDIS_OK;
}
}
return REDIS_ERR;
}
static int processMultiBulkItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj;
char *p;
PORT_LONG elements;
int root = 0;
/* Set error for nested multi bulks with depth > 7 */
if (r->ridx == 8) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"No support for nested multi bulk replies with depth > 7");
return REDIS_ERR;
}
if ((p = readLine(r,NULL)) != NULL) {
elements = readLongLong(p);
root = (r->ridx == 0);
if (elements == -1) {
if (r->fn && r->fn->createNil)
obj = r->fn->createNil(cur);
else
obj = (void*)REDIS_REPLY_NIL;
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
moveToNextTask(r);
} else {
if (r->fn && r->fn->createArray)
obj = r->fn->createArray(cur,elements);
else
obj = (void*)REDIS_REPLY_ARRAY;
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
/* Modify task stack when there are more than 0 elements. */
if (elements > 0) {
cur->elements = elements;
cur->obj = obj;
r->ridx++;
r->rstack[r->ridx].type = -1;
r->rstack[r->ridx].elements = -1;
r->rstack[r->ridx].idx = 0;
r->rstack[r->ridx].obj = NULL;
r->rstack[r->ridx].parent = cur;
r->rstack[r->ridx].privdata = r->privdata;
} else {
moveToNextTask(r);
}
}
/* Set reply if this is the root object. */
if (root) r->reply = obj;
return REDIS_OK;
}
return REDIS_ERR;
}
static int processItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
char *p;
/* check if we need to read type */
if (cur->type < 0) {
if ((p = readBytes(r,1)) != NULL) {
switch (p[0]) {
case '-':
cur->type = REDIS_REPLY_ERROR;
break;
case '+':
cur->type = REDIS_REPLY_STATUS;
break;
case ':':
cur->type = REDIS_REPLY_INTEGER;
break;
case '$':
cur->type = REDIS_REPLY_STRING;
break;
case '*':
cur->type = REDIS_REPLY_ARRAY;
break;
default:
__redisReaderSetErrorProtocolByte(r,*p);
return REDIS_ERR;
}
} else {
/* could not consume 1 byte */
return REDIS_ERR;
}
}
/* process typed item */
switch(cur->type) {
case REDIS_REPLY_ERROR:
case REDIS_REPLY_STATUS:
case REDIS_REPLY_INTEGER:
return processLineItem(r);
case REDIS_REPLY_STRING:
return processBulkItem(r);
case REDIS_REPLY_ARRAY:
return processMultiBulkItem(r);
default:
assert(NULL);
return REDIS_ERR; /* Avoid warning. */
}
}
redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) {
redisReader *r;
r = calloc(sizeof(redisReader),1);
if (r == NULL)
return NULL;
r->err = 0;
r->errstr[0] = '\0';
r->fn = fn;
r->buf = sdsempty();
r->maxbuf = REDIS_READER_MAX_BUF;
if (r->buf == NULL) {
free(r);
return NULL;
}
r->ridx = -1;
return r;
}
void redisReaderFree(redisReader *r) {
if (r->reply != NULL && r->fn && r->fn->freeObject)
r->fn->freeObject(r->reply);
if (r->buf != NULL)
sdsfree(r->buf);
free(r);
}
int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
sds newbuf;
/* Return early when this reader is in an erroneous state. */
if (r->err)
return REDIS_ERR;
/* Copy the provided buffer. */
if (buf != NULL && len >= 1) {
/* Destroy internal buffer when it is empty and is quite large. */
if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) {
sdsfree(r->buf);
r->buf = sdsempty();
r->pos = 0;
/* r->buf should not be NULL since we just free'd a larger one. */
assert(r->buf != NULL);
}
newbuf = sdscatlen(r->buf,buf,len);
if (newbuf == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
r->buf = newbuf;
r->len = sdslen(r->buf);
}
return REDIS_OK;
}
int redisReaderGetReply(redisReader *r, void **reply) {
/* Default target pointer to NULL. */
if (reply != NULL)
*reply = NULL;
/* Return early when this reader is in an erroneous state. */
if (r->err)
return REDIS_ERR;
/* When the buffer is empty, there will never be a reply. */
if (r->len == 0)
return REDIS_OK;
/* Set first item to process when the stack is empty. */
if (r->ridx == -1) {
r->rstack[0].type = -1;
r->rstack[0].elements = -1;
r->rstack[0].idx = -1;
r->rstack[0].obj = NULL;
r->rstack[0].parent = NULL;
r->rstack[0].privdata = r->privdata;
r->ridx = 0;
}
/* Process items in reply. */
while (r->ridx >= 0)
if (processItem(r) != REDIS_OK)
break;
/* Return ASAP when an error occurred. */
if (r->err)
return REDIS_ERR;
/* Discard part of the buffer when we've consumed at least 1k, to avoid
* doing unnecessary calls to memmove() in sds.c. */
if (r->pos >= 1024) {
sdsrange(r->buf,r->pos,-1);
r->pos = 0;
r->len = sdslen(r->buf);
}
/* Emit a reply when there is one. */
if (r->ridx == -1) {
if (reply != NULL)
*reply = r->reply;
r->reply = NULL;
}
return REDIS_OK;
}
deps/hiredis/read.h
Vendored
+114
View File
@@ -0,0 +1,114 @@
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __HIREDIS_READ_H
#define __HIREDIS_READ_H
#include <stdio.h> /* for size_t */
#ifdef _WIN32
#include "../../src/Win32_Interop/win32_types_hiredis.h"
#endif
#define REDIS_ERR -1
#define REDIS_OK 0
/* When an error occurs, the err flag in a context is set to hold the type of
* error that occurred. REDIS_ERR_IO means there was an I/O error and you
* should use the "errno" variable to find out what is wrong.
* For other values, the "errstr" field will hold a description. */
#define REDIS_ERR_IO 1 /* Error in read or write */
#define REDIS_ERR_EOF 3 /* End of file */
#define REDIS_ERR_PROTOCOL 4 /* Protocol error */
#define REDIS_ERR_OOM 5 /* Out of memory */
#define REDIS_ERR_OTHER 2 /* Everything else... */
#define REDIS_REPLY_STRING 1
#define REDIS_REPLY_ARRAY 2
#define REDIS_REPLY_INTEGER 3
#define REDIS_REPLY_NIL 4
#define REDIS_REPLY_STATUS 5
#define REDIS_REPLY_ERROR 6
#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */
#ifdef __cplusplus
extern "C" {
#endif
typedef struct redisReadTask {
int type;
int elements; /* number of elements in multibulk container */
int idx; /* index in parent (array) object */
void *obj; /* holds user-generated value for a read task */
struct redisReadTask *parent; /* parent task */
void *privdata; /* user-settable arbitrary field */
} redisReadTask;
typedef struct redisReplyObjectFunctions {
void *(*createString)(const redisReadTask*, char*, size_t);
void *(*createArray)(const redisReadTask*, int);
void *(*createInteger)(const redisReadTask*, PORT_LONGLONG);
void *(*createNil)(const redisReadTask*);
void (*freeObject)(void*);
} redisReplyObjectFunctions;
typedef struct redisReader {
int err; /* Error flags, 0 when there is no error */
char errstr[128]; /* String representation of error when applicable */
char *buf; /* Read buffer */
size_t pos; /* Buffer cursor */
size_t len; /* Buffer length */
size_t maxbuf; /* Max length of unused buffer */
redisReadTask rstack[9];
int ridx; /* Index of current read task */
void *reply; /* Temporary reply pointer */
redisReplyObjectFunctions *fn;
void *privdata;
} redisReader;
/* Public API for the protocol parser. */
redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn);
void redisReaderFree(redisReader *r);
int redisReaderFeed(redisReader *r, const char *buf, size_t len);
int redisReaderGetReply(redisReader *r, void **reply);
#define redisReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p))
#define redisReaderGetObject(_r) (((redisReader*)(_r))->reply)
#define redisReaderGetError(_r) (((redisReader*)(_r))->errstr)
#ifdef __cplusplus
}
#endif
#endif
deps/hiredis/sds.c
Vendored
+794 -781
View File
File diff suppressed because it is too large Load Diff
deps/hiredis/sds.h
Vendored
+160 -160
View File
@@ -51,68 +51,68 @@ typedef char *sds;
/* Note: sdshdr5 is never used, we just access the flags byte directly.
* However is here to document the layout of type 5 SDS strings. */
PACK(
struct sdshdr5{
unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
char buf[];
struct sdshdr5 {
unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
char buf[];
};)
PACK(
struct sdshdr8 {
uint8_t len; /* used */
uint8_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
struct sdshdr8 {
uint8_t len; /* used */
uint8_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};)
PACK(
struct sdshdr16 {
uint16_t len; /* used */
uint16_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
struct sdshdr16 {
uint16_t len; /* used */
uint16_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};)
PACK(
struct sdshdr32 {
uint32_t len; /* used */
uint32_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
struct sdshdr32 {
uint32_t len; /* used */
uint32_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};)
PACK(
struct sdshdr64 {
uint64_t len; /* used */
uint64_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
struct sdshdr64 {
uint64_t len; /* used */
uint64_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};)
#else
/* Note: sdshdr5 is never used, we just access the flags byte directly.
* However is here to document the layout of type 5 SDS strings. */
struct __attribute__ ((__packed__)) sdshdr5 {
unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
char buf[];
struct __attribute__((__packed__)) sdshdr5 {
unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
char buf[];
};
struct __attribute__ ((__packed__)) sdshdr8 {
uint8_t len; /* used */
uint8_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
struct __attribute__((__packed__)) sdshdr8 {
uint8_t len; /* used */
uint8_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
struct __attribute__ ((__packed__)) sdshdr16 {
uint16_t len; /* used */
uint16_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
struct __attribute__((__packed__)) sdshdr16 {
uint16_t len; /* used */
uint16_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
struct __attribute__ ((__packed__)) sdshdr32 {
uint32_t len; /* used */
uint32_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
struct __attribute__((__packed__)) sdshdr32 {
uint32_t len; /* used */
uint32_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
struct __attribute__ ((__packed__)) sdshdr64 {
uint64_t len; /* used */
uint64_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
struct __attribute__((__packed__)) sdshdr64 {
uint64_t len; /* used */
uint64_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
#endif
@@ -123,7 +123,7 @@ struct __attribute__ ((__packed__)) sdshdr64 {
#define SDS_TYPE_64 4
#define SDS_TYPE_MASK 7
#define SDS_TYPE_BITS 3
#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (void*)((s)-(sizeof(struct sdshdr##T)));
#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)));
#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
@@ -132,134 +132,134 @@ struct __attribute__ ((__packed__)) sdshdr64 {
#endif
static inline size_t sdslen(const sds s) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
return SDS_TYPE_5_LEN(flags);
case SDS_TYPE_8:
return SDS_HDR(8,s)->len;
case SDS_TYPE_16:
return SDS_HDR(16,s)->len;
case SDS_TYPE_32:
return SDS_HDR(32,s)->len;
case SDS_TYPE_64:
return SDS_HDR(64,s)->len;
}
return 0;
unsigned char flags = s[-1];
switch (flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
return SDS_TYPE_5_LEN(flags);
case SDS_TYPE_8:
return SDS_HDR(8, s)->len;
case SDS_TYPE_16:
return SDS_HDR(16, s)->len;
case SDS_TYPE_32:
return SDS_HDR(32, s)->len;
case SDS_TYPE_64:
return SDS_HDR(64, s)->len;
}
return 0;
}
static inline size_t sdsavail(const sds s) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5: {
return 0;
}
case SDS_TYPE_8: {
SDS_HDR_VAR(8,s);
return sh->alloc - sh->len;
}
case SDS_TYPE_16: {
SDS_HDR_VAR(16,s);
return sh->alloc - sh->len;
}
case SDS_TYPE_32: {
SDS_HDR_VAR(32,s);
return sh->alloc - sh->len;
}
case SDS_TYPE_64: {
SDS_HDR_VAR(64,s);
return sh->alloc - sh->len;
}
}
return 0;
unsigned char flags = s[-1];
switch (flags&SDS_TYPE_MASK) {
case SDS_TYPE_5: {
return 0;
}
case SDS_TYPE_8: {
SDS_HDR_VAR(8, s);
return sh->alloc - sh->len;
}
case SDS_TYPE_16: {
SDS_HDR_VAR(16, s);
return sh->alloc - sh->len;
}
case SDS_TYPE_32: {
SDS_HDR_VAR(32, s);
return sh->alloc - sh->len;
}
case SDS_TYPE_64: {
SDS_HDR_VAR(64, s);
return sh->alloc - sh->len;
}
}
return 0;
}
static inline void sdssetlen(sds s, size_t newlen) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
{
unsigned char *fp = ((unsigned char*)s)-1;
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
}
break;
case SDS_TYPE_8:
SDS_HDR(8,s)->len = newlen;
break;
case SDS_TYPE_16:
SDS_HDR(16,s)->len = newlen;
break;
case SDS_TYPE_32:
SDS_HDR(32,s)->len = newlen;
break;
case SDS_TYPE_64:
SDS_HDR(64,s)->len = newlen;
break;
}
unsigned char flags = s[-1];
switch (flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
{
unsigned char *fp = ((unsigned char*)s) - 1;
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
}
break;
case SDS_TYPE_8:
SDS_HDR(8, s)->len = newlen;
break;
case SDS_TYPE_16:
SDS_HDR(16, s)->len = newlen;
break;
case SDS_TYPE_32:
SDS_HDR(32, s)->len = newlen;
break;
case SDS_TYPE_64:
SDS_HDR(64, s)->len = newlen;
break;
}
}
static inline void sdsinclen(sds s, size_t inc) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
{
unsigned char *fp = ((unsigned char*)s)-1;
unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc;
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
}
break;
case SDS_TYPE_8:
SDS_HDR(8,s)->len += inc;
break;
case SDS_TYPE_16:
SDS_HDR(16,s)->len += inc;
break;
case SDS_TYPE_32:
SDS_HDR(32,s)->len += inc;
break;
case SDS_TYPE_64:
SDS_HDR(64,s)->len += inc;
break;
}
unsigned char flags = s[-1];
switch (flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
{
unsigned char *fp = ((unsigned char*)s) - 1;
unsigned char newlen = SDS_TYPE_5_LEN(flags) + inc;
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
}
break;
case SDS_TYPE_8:
SDS_HDR(8, s)->len += inc;
break;
case SDS_TYPE_16:
SDS_HDR(16, s)->len += inc;
break;
case SDS_TYPE_32:
SDS_HDR(32, s)->len += inc;
break;
case SDS_TYPE_64:
SDS_HDR(64, s)->len += inc;
break;
}
}
/* sdsalloc() = sdsavail() + sdslen() */
static inline size_t sdsalloc(const sds s) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
return SDS_TYPE_5_LEN(flags);
case SDS_TYPE_8:
return SDS_HDR(8,s)->alloc;
case SDS_TYPE_16:
return SDS_HDR(16,s)->alloc;
case SDS_TYPE_32:
return SDS_HDR(32,s)->alloc;
case SDS_TYPE_64:
return SDS_HDR(64,s)->alloc;
}
return 0;
unsigned char flags = s[-1];
switch (flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
return SDS_TYPE_5_LEN(flags);
case SDS_TYPE_8:
return SDS_HDR(8, s)->alloc;
case SDS_TYPE_16:
return SDS_HDR(16, s)->alloc;
case SDS_TYPE_32:
return SDS_HDR(32, s)->alloc;
case SDS_TYPE_64:
return SDS_HDR(64, s)->alloc;
}
return 0;
}
static inline void sdssetalloc(sds s, size_t newlen) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
/* Nothing to do, this type has no total allocation info. */
break;
case SDS_TYPE_8:
SDS_HDR(8,s)->alloc = newlen;
break;
case SDS_TYPE_16:
SDS_HDR(16,s)->alloc = newlen;
break;
case SDS_TYPE_32:
SDS_HDR(32,s)->alloc = newlen;
break;
case SDS_TYPE_64:
SDS_HDR(64,s)->alloc = newlen;
break;
}
unsigned char flags = s[-1];
switch (flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
/* Nothing to do, this type has no total allocation info. */
break;
case SDS_TYPE_8:
SDS_HDR(8, s)->alloc = newlen;
break;
case SDS_TYPE_16:
SDS_HDR(16, s)->alloc = newlen;
break;
case SDS_TYPE_32:
SDS_HDR(32, s)->alloc = newlen;
break;
case SDS_TYPE_64:
SDS_HDR(64, s)->alloc = newlen;
break;
}
}
sds sdsnewlen(const void *init, size_t initlen);
@@ -277,7 +277,7 @@ sds sdscpy(sds s, const char *t);
sds sdscatvprintf(sds s, const char *fmt, va_list ap);
#ifdef __GNUC__
sds sdscatprintf(sds s, const char *fmt, ...)
__attribute__((format(printf, 2, 3)));
__attribute__((format(printf, 2, 3)));
#else
sds sdscatprintf(sds s, const char *fmt, ...);
#endif
deps/hiredis/sdsalloc.h
Vendored
+4 -4
View File
@@ -1,6 +1,7 @@
/* SDSLib 2.0 -- A C dynamic strings library
*
* Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2015, Oran Agra
* Copyright (c) 2015, Redis Labs, Inc
* All rights reserved.
*
@@ -36,7 +37,6 @@
* the include of your alternate allocator if needed (not needed in order
* to use the default libc allocator). */
#include "zmalloc.h"
#define s_malloc zmalloc
#define s_realloc zrealloc
#define s_free zfree
#define s_malloc malloc
#define s_realloc realloc
#define s_free free
deps/hiredis/test.c
Vendored
+103 -14
View File
@@ -43,7 +43,7 @@ struct config {
struct {
const char *path;
} unix;
} unix_sock;
};
/* The following lines make up our testing "framework" :) */
@@ -57,6 +57,13 @@ static PORT_LONGLONG usec(void) {
return (((PORT_LONGLONG)tv.tv_sec)*1000000)+tv.tv_usec;
}
/* The assert() calls below have side effects, so we need assert()
* even if we are compiling without asserts (-DNDEBUG). */
#ifdef NDEBUG
#undef assert
#define assert(e) (void)(e)
#endif
static redisContext *select_database(redisContext *c) {
redisReply *reply;
@@ -107,10 +114,10 @@ static redisContext *connect(struct config config) {
if (config.type == CONN_TCP) {
c = redisConnect(config.tcp.host, config.tcp.port);
} else if (config.type == CONN_UNIX) {
c = redisConnectUnix(config.unix.path);
c = redisConnectUnix(config.unix_sock.path);
} else if (config.type == CONN_FD) {
/* Create a dummy connection just to get an fd to inherit */
redisContext *dummy_ctx = redisConnectUnix(config.unix.path);
redisContext *dummy_ctx = redisConnectUnix(config.unix_sock.path);
if (dummy_ctx) {
int fd = disconnect(dummy_ctx, 1);
printf("Connecting to inherited fd %d\n", fd);
@@ -125,6 +132,7 @@ static redisContext *connect(struct config config) {
exit(1);
} else if (c->err) {
printf("Connection error: %s\n", c->errstr);
redisFree(c);
exit(1);
}
@@ -233,6 +241,22 @@ static void test_format_commands(void) {
test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(7+2)+4+(3+2));
free(cmd);
sds sds_cmd;
sds_cmd = sdsempty();
test("Format command into sds by passing argc/argv without lengths: ");
len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL);
test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(3+2)+4+(3+2));
sdsfree(sds_cmd);
sds_cmd = sdsempty();
test("Format command into sds by passing argc/argv with lengths: ");
len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens);
test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(7+2)+4+(3+2));
sdsfree(sds_cmd);
}
static void test_append_formatted_commands(struct config config) {
@@ -336,16 +360,31 @@ static void test_reply_reader(void) {
redisReaderFree(reader);
}
static void test_free_null(void) {
void *redisCtx = NULL;
void *reply = NULL;
test("Don't fail when redisFree is passed a NULL value: ");
redisFree(redisCtx);
test_cond(redisCtx == NULL);
test("Don't fail when freeReplyObject is passed a NULL value: ");
freeReplyObject(reply);
test_cond(reply == NULL);
}
static void test_blocking_connection_errors(void) {
redisContext *c;
test("Returns error when host cannot be resolved: ");
c = redisConnect((char*)"idontexist.local", 6379);
c = redisConnect((char*)"idontexist.test", 6379);
test_cond(c->err == REDIS_ERR_OTHER &&
(strcmp(c->errstr,"Name or service not known") == 0 ||
strcmp(c->errstr,"Can't resolve: idontexist.local") == 0 ||
strcmp(c->errstr,"Can't resolve: idontexist.test") == 0 ||
strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 ||
strcmp(c->errstr,"No address associated with hostname") == 0 ||
strcmp(c->errstr,"Temporary failure in name resolution") == 0 ||
strcmp(c->errstr,"hostname nor servname provided, or not known") == 0 ||
strcmp(c->errstr,"no address associated with name") == 0));
redisFree(c);
@@ -355,7 +394,7 @@ static void test_blocking_connection_errors(void) {
strcmp(c->errstr,"Connection refused") == 0);
redisFree(c);
test("Returns error when the unix socket path doesn't accept connections: ");
test("Returns error when the unix_sock socket path doesn't accept connections: ");
c = redisConnectUnix((char*)"/tmp/idontexist.sock");
test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */
redisFree(c);
@@ -439,6 +478,52 @@ static void test_blocking_connection(struct config config) {
disconnect(c, 0);
}
static void test_blocking_connection_timeouts(struct config config) {
redisContext *c;
redisReply *reply;
ssize_t s;
const char *cmd = "DEBUG SLEEP 3\r\n";
struct timeval tv;
c = IF_WIN32(_connect,connect)(config);
test("Successfully completes a command when the timeout is not exceeded: ");
reply = redisCommand(c,"SET foo fast");
freeReplyObject(reply);
tv.tv_sec = 0;
tv.tv_usec = 10000;
redisSetTimeout(c, tv);
reply = redisCommand(c, "GET foo");
test_cond(reply != NULL && reply->type == REDIS_REPLY_STRING && memcmp(reply->str, "fast", 4) == 0);
freeReplyObject(reply);
disconnect(c, 0);
c = IF_WIN32(_connect,connect)(config);
test("Does not return a reply when the command times out: ");
s = write(c->fd, cmd, strlen(cmd));
tv.tv_sec = 0;
tv.tv_usec = 10000;
redisSetTimeout(c, tv);
reply = redisCommand(c, "GET foo");
test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0);
freeReplyObject(reply);
test("Reconnect properly reconnects after a timeout: ");
redisReconnect(c);
reply = redisCommand(c, "PING");
test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
freeReplyObject(reply);
test("Reconnect properly uses owned parameters: ");
config.tcp.host = "foo";
config.unix_sock.path = "foo";
redisReconnect(c);
reply = redisCommand(c, "PING");
test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
freeReplyObject(reply);
disconnect(c, 0);
}
static void test_blocking_io_errors(struct config config) {
redisContext *c;
redisReply *reply;
@@ -462,7 +547,7 @@ static void test_blocking_io_errors(struct config config) {
test("Returns I/O error when the connection is lost: ");
reply = redisCommand(c,"QUIT");
if (major >= 2 && minor > 0) {
if (major > 2 || (major == 2 && minor > 0)) {
/* > 2.0 returns OK on QUIT and read() should be issued once more
* to know the descriptor is at EOF. */
test_cond(strcasecmp(reply->str,"OK") == 0 &&
@@ -500,7 +585,8 @@ static void test_invalid_timeout_errors(struct config config) {
c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
test_cond(c->err == REDIS_ERR_IO);
test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
redisFree(c);
test("Set error when an invalid timeout sec value is given to redisConnectWithTimeout: ");
@@ -509,8 +595,7 @@ static void test_invalid_timeout_errors(struct config config) {
c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
test_cond(c->err == REDIS_ERR_IO);
test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
redisFree(c);
}
@@ -684,7 +769,7 @@ int main(int argc, char **argv) {
.host = "127.0.0.1",
.port = 6379
},
.unix = {
.unix_sock = {
.path = "/tmp/redis.sock"
}
};
@@ -705,7 +790,7 @@ int main(int argc, char **argv) {
cfg.tcp.port = atoi(argv[0]);
} else if (argc >= 2 && !strcmp(argv[0],"-s")) {
argv++; argc--;
cfg.unix.path = argv[0];
cfg.unix_sock.path = argv[0];
} else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) {
throughput = 0;
} else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) {
@@ -720,27 +805,31 @@ int main(int argc, char **argv) {
test_format_commands();
test_reply_reader();
test_blocking_connection_errors();
test_free_null();
printf("\nTesting against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port);
cfg.type = CONN_TCP;
test_blocking_connection(cfg);
test_blocking_connection_timeouts(cfg);
test_blocking_io_errors(cfg);
test_invalid_timeout_errors(cfg);
test_append_formatted_commands(cfg);
if (throughput) test_throughput(cfg);
printf("\nTesting against Unix socket connection (%s):\n", cfg.unix.path);
printf("\nTesting against Unix socket connection (%s):\n", cfg.unix_sock.path);
cfg.type = CONN_UNIX;
test_blocking_connection(cfg);
test_blocking_connection_timeouts(cfg);
test_blocking_io_errors(cfg);
if (throughput) test_throughput(cfg);
if (test_inherit_fd) {
printf("\nTesting against inherited fd (%s):\n", cfg.unix.path);
printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path);
cfg.type = CONN_FD;
test_blocking_connection(cfg);
}
if (fails) {
printf("*** %d TESTS FAILED ***\n", fails);
return 1;
deps/hiredis/win32_hiredis.h
Vendored
+29 -2
View File
@@ -32,9 +32,36 @@
#include "hiredis.h"
#define snprintf _snprintf
#ifndef snprintf
#define snprintf c99_snprintf
__inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap)
{
int count = -1;
if (size != 0)
count = _vsnprintf_s(str, size, _TRUNCATE, format, ap);
if (count == -1)
count = _vscprintf(format, ap);
return count;
}
__inline int c99_snprintf(char* str, size_t size, const char* format, ...)
{
int count;
va_list ap;
va_start(ap, format);
count = c99_vsnprintf(str, size, format, ap);
va_end(ap);
return count;
}
#endif
#ifndef va_copy
#define va_copy(d,s) d = (s)
#define va_copy(d,s) ((d) = (s))
#endif
redisContext *redisPreConnectNonBlock(const char *ip, int port, SOCKADDR_STORAGE *sa);
deps/jemalloc-win/src/arena.c
Vendored
+2 -2
View File
@@ -1513,8 +1513,8 @@ arena_redzone_corruption(void *ptr, size_t usize, bool after,
size_t offset, uint8_t byte)
{
malloc_printf("<jemalloc>: Corrupt redzone %zu byte%s %s %p "
"(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s",
malloc_printf("<jemalloc>: Corrupt redzone %Iu byte%s %s %p " /*WIN_PORT_FIX %zu -> %Iu */
"(size %Iu), byte=%#x\n", offset, (offset == 1) ? "" : "s", /*WIN_PORT_FIX %zu -> %Iu */
after ? "after" : "before", ptr, usize, byte);
}
#ifdef JEMALLOC_JET
deps/jemalloc-win/src/prof.c
Vendored
+1 -1
View File
@@ -992,7 +992,7 @@ prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_nctx,
if (cnt_all->curbytes != 0) {
malloc_printf("<jemalloc>: Leak summary: %"PRId64" byte%s, %"
PRId64" object%s, %zu context%s\n",
PRId64" object%s, %Iu context%s\n", /*WIN_PORT_FIX %zu -> %Iu */
cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "",
cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "",
leak_nctx, (leak_nctx != 1) ? "s" : "");
deps/jemalloc/include/jemalloc/jemalloc_macros.h.in
Vendored
+4
View File
@@ -100,3 +100,7 @@
# define JEMALLOC_RESTRICT_RETURN
# define JEMALLOC_ALLOCATOR
#endif
/* This version of Jemalloc, modified for Redis, has the je_get_defrag_hint()
* function. */
#define JEMALLOC_FRAG_HINT
deps/jemalloc/src/jemalloc.c
Vendored
+32
View File
@@ -2591,3 +2591,35 @@ jemalloc_postfork_child(void)
}
/******************************************************************************/
/* Helps the application decide if a pointer is worth re-allocating in order to reduce fragmentation.
* returns 0 if the allocation is in the currently active run,
* or when it is not causing any frag issue (large or huge bin)
* returns the bin utilization and run utilization both in fixed point 16:16.
* If the application decides to re-allocate it should use MALLOCX_TCACHE_NONE when doing so. */
JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_get_defrag_hint(void* ptr, int *bin_util, int *run_util) {
int defrag = 0;
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (likely(chunk != ptr)) { /* indication that this is not a HUGE alloc */
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind);
if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { /* indication that this is not a LARGE alloc */
arena_t *arena = extent_node_arena_get(&chunk->node);
size_t rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind);
arena_run_t *run = &arena_miscelm_get(chunk, rpages_ind)->run;
arena_bin_t *bin = &arena->bins[run->binind];
malloc_mutex_lock(&bin->lock);
/* runs that are in the same chunk in as the current chunk, are likely to be the next currun */
if (chunk != (arena_chunk_t *)CHUNK_ADDR2BASE(bin->runcur)) {
arena_bin_info_t *bin_info = &arena_bin_info[run->binind];
size_t availregs = bin_info->nregs * bin->stats.curruns;
*bin_util = (bin->stats.curregs<<16) / availregs;
*run_util = ((bin_info->nregs - run->nfree)<<16) / bin_info->nregs;
defrag = 1;
}
malloc_mutex_unlock(&bin->lock);
}
}
return defrag;
}
msvs/RedisCheckAof/RedisCheckAof.vcxproj
+10 -2
View File
@@ -88,7 +88,7 @@
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_DEBUG;_CONSOLE;LACKS_STDLIB_H;%(PreprocessorDefinitions);NO_QFORKIMPL</PreprocessorDefinitions>
<PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_DEBUG;_CONSOLE;LACKS_STDLIB_H;%(PreprocessorDefinitions);NO_QFORKIMPL;_WIN32_REDIS_CHECK_AOF_EXE</PreprocessorDefinitions>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
<AdditionalIncludeDirectories>$(SolutionDir)..\deps\lua\src</AdditionalIncludeDirectories>
@@ -100,6 +100,10 @@
<RandomizedBaseAddress>false</RandomizedBaseAddress>
<IgnoreSpecificDefaultLibraries>MSVCRT</IgnoreSpecificDefaultLibraries>
<LinkTimeCodeGeneration>Default</LinkTimeCodeGeneration>
<SubSystem>
</SubSystem>
<EntryPointSymbol>
</EntryPointSymbol>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
@@ -127,7 +131,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_CONSOLE;%(PreprocessorDefinitions);LACKS_STDLIB_H;NO_QFORKIMPL</PreprocessorDefinitions>
<PreprocessorDefinitions>_OFF_T_DEFINED;WIN32;_CONSOLE;%(PreprocessorDefinitions);LACKS_STDLIB_H;NO_QFORKIMPL;_WIN32_REDIS_CHECK_AOF_EXE</PreprocessorDefinitions>
<DisableSpecificWarnings>4996</DisableSpecificWarnings>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<AdditionalIncludeDirectories>$(SolutionDir)..\deps\lua\src</AdditionalIncludeDirectories>
@@ -165,12 +169,16 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\src\redis-check-aof.c" />
<ClCompile Include="..\..\src\zmalloc.c" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Win32_Interop\Win32_Interop.vcxproj">
<Project>{8c07f811-c81c-432c-b334-1ae6faecf951}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\src\zmalloc.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
msvs/RedisCli/RedisCli.vcxproj
+3
View File
@@ -187,6 +187,9 @@
<Project>{13e85053-54b3-487b-8ddb-3430b1c1b3bf}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\deps\linenoise\linenoise.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
msvs/RedisServer.sln
+5 -2
View File
@@ -1,7 +1,7 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2013
VisualStudioVersion = 12.0.40629.0
# Visual Studio 15
VisualStudioVersion = 15.0.27004.2005
MinimumVisualStudioVersion = 10.0.40219.1
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "RedisServer", "RedisServer.vcxproj", "{46842776-68A5-EC98-6A09-1859BBFC73AA}"
ProjectSection(ProjectDependencies) = postProject
@@ -195,4 +195,7 @@ Global
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {5698B266-499D-48CD-8488-5FF35FF5C30A}
EndGlobalSection
EndGlobal
msvs/RedisServer.vcxproj
+27 -4
View File
@@ -94,7 +94,7 @@
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PreprocessorDefinitions>USE_JEMALLOC;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;_DEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\geohash-int;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>$(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\geohash-int;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include;$(SolutionDir)..\deps\linenoise</AdditionalIncludeDirectories>
<RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
<WarningLevel>Level3</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@@ -198,21 +198,29 @@
<ClCompile Include="..\src\bio.c" />
<ClCompile Include="..\src\bitops.c" />
<ClCompile Include="..\src\blocked.c" />
<ClCompile Include="..\src\childinfo.c" />
<ClCompile Include="..\src\cluster.c" />
<ClCompile Include="..\src\config.c" />
<ClCompile Include="..\src\crc16.c" />
<ClCompile Include="..\src\crc64.c" />
<ClCompile Include="..\src\db.c" />
<ClCompile Include="..\src\debug.c" />
<ClCompile Include="..\src\defrag.c" />
<ClCompile Include="..\src\dict.c" />
<ClCompile Include="..\src\endianconv.c" />
<ClCompile Include="..\src\evict.c" />
<ClCompile Include="..\src\expire.c" />
<ClCompile Include="..\src\geo.c" />
<ClCompile Include="..\src\geohash.c" />
<ClCompile Include="..\src\geohash_helper.c" />
<ClCompile Include="..\src\hyperloglog.c" />
<ClCompile Include="..\src\intset.c" />
<ClCompile Include="..\src\latency.c" />
<ClCompile Include="..\src\lazyfree.c" />
<ClCompile Include="..\src\lzf_c.c" />
<ClCompile Include="..\src\lzf_d.c" />
<ClCompile Include="..\src\memtest.c" />
<ClCompile Include="..\src\module.c" />
<ClCompile Include="..\src\multi.c" />
<ClCompile Include="..\src\networking.c" />
<ClCompile Include="..\src\notify.c" />
@@ -221,16 +229,20 @@
<ClCompile Include="..\src\pubsub.c" />
<ClCompile Include="..\src\quicklist.c" />
<ClCompile Include="..\src\rand.c" />
<ClCompile Include="..\src\rax.c" />
<ClCompile Include="..\src\rdb.c" />
<ClCompile Include="..\src\redis-check-aof.c" />
<ClCompile Include="..\src\redis-check-rdb.c" />
<ClCompile Include="..\src\server.c" />
<ClCompile Include="..\src\release.c" />
<ClCompile Include="..\src\replication.c" />
<ClCompile Include="..\src\rio.c" />
<ClCompile Include="..\src\scripting.c" />
<ClCompile Include="..\src\sds.c" />
<ClCompile Include="..\src\sentinel.c" />
<ClCompile Include="..\src\server.c" />
<ClCompile Include="..\src\setproctitle.c" />
<ClCompile Include="..\src\sha1.c" />
<ClCompile Include="..\src\siphash.c" />
<ClCompile Include="..\src\slowlog.c" />
<ClCompile Include="..\src\sort.c" />
<ClCompile Include="..\src\sparkline.c" />
@@ -241,6 +253,7 @@
<ClCompile Include="..\src\t_string.c" />
<ClCompile Include="..\src\t_zset.c" />
<ClCompile Include="..\src\util.c" />
<ClCompile Include="..\src\Win32_Interop\dlfcn.c" />
<ClCompile Include="..\src\ziplist.c" />
<ClCompile Include="..\src\zipmap.c" />
<ClCompile Include="..\src\zmalloc.c" />
@@ -249,13 +262,19 @@
<ClInclude Include="..\src\adlist.h" />
<ClInclude Include="..\src\ae.h" />
<ClInclude Include="..\src\anet.h" />
<ClInclude Include="..\src\asciilogo.h" />
<ClInclude Include="..\src\atomicvar.h" />
<ClInclude Include="..\src\bio.h" />
<ClInclude Include="..\src\cluster.h" />
<ClInclude Include="..\src\config.h" />
<ClInclude Include="..\src\crc64.h" />
<ClInclude Include="..\src\debugmacro.h" />
<ClInclude Include="..\src\dict.h" />
<ClInclude Include="..\src\endianconv.h" />
<ClInclude Include="..\src\fmacros.h" />
<ClInclude Include="..\src\geo.h" />
<ClInclude Include="..\src\geohash.h" />
<ClInclude Include="..\src\geohash_helper.h" />
<ClInclude Include="..\src\help.h" />
<ClInclude Include="..\src\intset.h" />
<ClInclude Include="..\src\latency.h" />
@@ -264,12 +283,15 @@
<ClInclude Include="..\src\pqsort.h" />
<ClInclude Include="..\src\quicklist.h" />
<ClInclude Include="..\src\rand.h" />
<ClInclude Include="..\src\rax.h" />
<ClInclude Include="..\src\rax_malloc.h" />
<ClInclude Include="..\src\rdb.h" />
<ClInclude Include="..\src\sdsalloc.h" />
<ClInclude Include="..\src\server.h" />
<ClInclude Include="..\src\redisassert.h" />
<ClInclude Include="..\src\redismodule.h" />
<ClInclude Include="..\src\rio.h" />
<ClInclude Include="..\src\sds.h" />
<ClInclude Include="..\src\sdsalloc.h" />
<ClInclude Include="..\src\server.h" />
<ClInclude Include="..\src\sha1.h" />
<ClInclude Include="..\src\slowlog.h" />
<ClInclude Include="..\src\solarisfixes.h" />
@@ -277,6 +299,7 @@
<ClInclude Include="..\src\testhelp.h" />
<ClInclude Include="..\src\util.h" />
<ClInclude Include="..\src\version.h" />
<ClInclude Include="..\src\Win32_Interop\dlfcn.h" />
<ClInclude Include="..\src\ziplist.h" />
<ClInclude Include="..\src\zipmap.h" />
<ClInclude Include="..\src\zmalloc.h" />
msvs/hiredis/hiredis.vcxproj
+2
View File
@@ -153,6 +153,7 @@
<ClCompile Include="..\..\deps\hiredis\async.c" />
<ClCompile Include="..\..\deps\hiredis\hiredis.c" />
<ClCompile Include="..\..\deps\hiredis\net.c" />
<ClCompile Include="..\..\deps\hiredis\read.c" />
<ClCompile Include="..\..\deps\hiredis\sds.c" />
</ItemGroup>
<ItemGroup>
@@ -160,6 +161,7 @@
<ClInclude Include="..\..\deps\hiredis\fmacros.h" />
<ClInclude Include="..\..\deps\hiredis\hiredis.h" />
<ClInclude Include="..\..\deps\hiredis\net.h" />
<ClInclude Include="..\..\deps\hiredis\read.h" />
<ClInclude Include="..\..\deps\hiredis\sds.h" />
<ClInclude Include="..\..\deps\hiredis\win32_hiredis.h" />
</ItemGroup>
src/Win32_Interop/Win32_PThread.c
+6 -1
View File
@@ -206,4 +206,9 @@ int pthread_cond_signal(pthread_cond_t *cond) {
0 : GetLastError();
else
return 0;
}
}
int pthread_cond_broadcast(pthread_cond_t *cond) {
//TODO
return 0;
}
src/Win32_Interop/Win32_PThread.h
+6 -1
View File
@@ -1,4 +1,5 @@
#ifndef __WIN32_PTHREAD_H_
#define __WIN32_PTHREAD_H_
#include <windows.h>
#include <errno.h>
@@ -48,6 +49,10 @@ int pthread_cond_init(pthread_cond_t *cond, const void *unused);
int pthread_cond_destroy(pthread_cond_t *cond);
int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex);
int pthread_cond_signal(pthread_cond_t *cond);
//TODO
int pthread_cond_broadcast(pthread_cond_t *cond);
int pthread_detach(pthread_t thread);
int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset);
#endif
src/Win32_Interop/Win32_QFork_impl.c
+4 -2
View File
@@ -35,7 +35,9 @@ int do_rdbSave(char* filename)
{
#ifndef NO_QFORKIMPL
server.rdb_child_pid = GetCurrentProcessId();
if( rdbSave(filename) != C_OK ) {
rdbSaveInfo rsi, *rsiptr;
rsiptr = rdbPopulateSaveInfo(&rsi);
if( rdbSave(filename, rsiptr) != C_OK ) {
serverLog(LL_WARNING,"rdbSave failed in qfork: %s", strerror(errno));
return C_ERR;
}
@@ -90,7 +92,7 @@ int do_rdbSaveToSlavesSockets(int *fds, int numfds, uint64_t *clientids)
retval = C_ERR;
if (retval == C_OK) {
size_t private_dirty = zmalloc_get_private_dirty();
size_t private_dirty = zmalloc_get_private_dirty(-1);
if (private_dirty) {
serverLog(LL_NOTICE,
src/Win32_Interop/Win32_StackTrace.cpp
+2 -1
View File
@@ -127,7 +127,8 @@ void LogStackTrace() {
GetModuleFileNameA((HINSTANCE) moduleBase, modulePath, MAX_PATH);
}
serverLog(LL_WARNING | LL_RAW, "%s!%s(%s:%d)(0x%08LX, 0x%08LX, 0x%08LX, 0x%08LX)\n",
//serverLog(LL_WARNING | LL_RAW, "%s!%s(%s:%d)(0x%08LX, 0x%08LX, 0x%08LX, 0x%08LX)\n",
serverLog(LL_WARNING | LL_RAW, "%s!%s(%s:%d)(0x%08I64X, 0x%08I64X, 0x%08I64X, 0x%08I64X)\n",
&modulePath[GetFilenameStart(modulePath)],
pSymbol->Name,
line.FileName,
src/Win32_Interop/dlfcn.c
+488
View File
@@ -0,0 +1,488 @@
/*
* dlfcn-win32
* Copyright (c) 2007 Ramiro Polla
* Copyright (c) 2015 Tiancheng "Timothy" Gu
*
* dlfcn-win32 is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* dlfcn-win32 is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with dlfcn-win32; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifdef _DEBUG
#define _CRTDBG_MAP_ALLOC
#include <stdlib.h>
#include <crtdbg.h>
#endif
#define PSAPI_VERSION 1
#include <windows.h>
#include <psapi.h>
#include <stdio.h>
#ifdef SHARED
#define DLFCN_WIN32_EXPORTS
#endif
#include "dlfcn.h"
#if ((defined(_WIN32) || defined(WIN32)) && (defined(_MSC_VER)) )
#define snprintf sprintf_s
#endif
#ifdef UNICODE
#include <wchar.h>
#define CHAR wchar_t
#define UNICODE_L(s) L##s
#else
#define CHAR char
#define UNICODE_L(s) s
#endif
/* Note:
* MSDN says these functions are not thread-safe. We make no efforts to have
* any kind of thread safety.
*/
typedef struct global_object {
HMODULE hModule;
struct global_object *previous;
struct global_object *next;
} global_object;
static global_object first_object;
static global_object first_automatic_object;
static int auto_ref_count = 0;
/* These functions implement a double linked list for the global objects. */
static global_object *global_search( global_object *start, HMODULE hModule )
{
global_object *pobject;
if( hModule == NULL )
return NULL;
for( pobject = start; pobject; pobject = pobject->next )
if( pobject->hModule == hModule )
return pobject;
return NULL;
}
static void global_add( global_object *start, HMODULE hModule )
{
global_object *pobject;
global_object *nobject;
if( hModule == NULL )
return;
pobject = global_search( start, hModule );
/* Do not add object again if it's already on the list */
if( pobject )
return;
if( start == &first_automatic_object )
{
pobject = global_search( &first_object, hModule );
if( pobject )
return;
}
for( pobject = start; pobject->next; pobject = pobject->next );
nobject = (global_object*) malloc( sizeof( global_object ) );
/* Should this be enough to fail global_add, and therefore also fail
* dlopen?
*/
if( !nobject )
return;
pobject->next = nobject;
nobject->next = NULL;
nobject->previous = pobject;
nobject->hModule = hModule;
}
static void global_rem( global_object *start, HMODULE hModule )
{
global_object *pobject;
if( hModule == NULL )
return;
pobject = global_search( start, hModule );
if( !pobject )
return;
if( pobject->next )
pobject->next->previous = pobject->previous;
if( pobject->previous )
pobject->previous->next = pobject->next;
free( pobject );
}
/* POSIX says dlerror( ) doesn't have to be thread-safe, so we use one
* static buffer.
* MSDN says the buffer cannot be larger than 64K bytes, so we set it to
* the limit.
*/
static CHAR error_buffer[65535];
static CHAR *current_error;
static char dlerror_buffer[65536];
static int copy_string( CHAR *dest, int dest_size, const CHAR *src )
{
int i = 0;
/* gcc should optimize this out */
if( !src || !dest )
return 0;
for( i = 0 ; i < dest_size-1 ; i++ )
{
if( !src[i] )
break;
else
dest[i] = src[i];
}
dest[i] = '\0';
return i;
}
static void save_err_str( const CHAR *str )
{
DWORD dwMessageId;
DWORD pos;
dwMessageId = GetLastError( );
if( dwMessageId == 0 )
return;
/* Format error message to:
* "<argument to function that failed>": <Windows localized error message>
*/
pos = copy_string( error_buffer, sizeof(error_buffer), UNICODE_L("\"") );
pos += copy_string( error_buffer+pos, sizeof(error_buffer)-pos, str );
pos += copy_string( error_buffer+pos, sizeof(error_buffer)-pos, UNICODE_L("\": ") );
pos += FormatMessage( FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwMessageId,
MAKELANGID( LANG_NEUTRAL, SUBLANG_DEFAULT ),
error_buffer+pos, sizeof(error_buffer)-pos, NULL );
if( pos > 1 )
{
/* POSIX says the string must not have trailing <newline> */
if( error_buffer[pos-2] == '\r' && error_buffer[pos-1] == '\n' )
error_buffer[pos-2] = '\0';
}
current_error = error_buffer;
}
static void save_err_ptr_str( const void *ptr )
{
CHAR ptr_buf[19]; /* 0x<pointer> up to 64 bits. */
#ifdef UNICODE
# if ((defined(_WIN32) || defined(WIN32)) && (defined(_MSC_VER)) )
swprintf_s( ptr_buf, 19, UNICODE_L("0x%p"), ptr );
# else
swprintf(ptr_buf, 19, UNICODE_L("0x%p"), ptr);
# endif
#else
snprintf( ptr_buf, 19, "0x%p", ptr );
#endif
save_err_str( ptr_buf );
}
void *dlopen( const char *file, int mode )
{
HMODULE hModule;
UINT uMode;
current_error = NULL;
/* Do not let Windows display the critical-error-handler message box */
uMode = SetErrorMode( SEM_FAILCRITICALERRORS );
if( file == 0 )
{
HMODULE hAddtnlMods[1024]; // Already loaded modules
HANDLE hCurrentProc = GetCurrentProcess( );
DWORD cbNeeded;
/* POSIX says that if the value of file is 0, a handle on a global
* symbol object must be provided. That object must be able to access
* all symbols from the original program file, and any objects loaded
* with the RTLD_GLOBAL flag.
* The return value from GetModuleHandle( ) allows us to retrieve
* symbols only from the original program file. For objects loaded with
* the RTLD_GLOBAL flag, we create our own list later on. For objects
* outside of the program file but already loaded (e.g. linked DLLs)
* they are added below.
*/
hModule = GetModuleHandle( NULL );
if( !hModule )
save_err_ptr_str( file );
/* GetModuleHandle( NULL ) only returns the current program file. So
* if we want to get ALL loaded module including those in linked DLLs,
* we have to use EnumProcessModules( ).
*/
if( EnumProcessModules( hCurrentProc, hAddtnlMods,
sizeof( hAddtnlMods ), &cbNeeded ) != 0 )
{
DWORD i;
for( i = 0; i < cbNeeded / sizeof( HMODULE ); i++ )
{
global_add( &first_automatic_object, hAddtnlMods[i] );
}
}
auto_ref_count++;
}
else
{
CHAR lpFileName[MAX_PATH];
int i;
/* MSDN says backslashes *must* be used instead of forward slashes. */
for( i = 0 ; i < sizeof(lpFileName) - 1 ; i ++ )
{
if( !file[i] )
break;
else if( file[i] == '/' )
lpFileName[i] = '\\';
else
lpFileName[i] = file[i];
}
lpFileName[i] = '\0';
/* POSIX says the search path is implementation-defined.
* LOAD_WITH_ALTERED_SEARCH_PATH is used to make it behave more closely
* to UNIX's search paths (start with system folders instead of current
* folder).
*/
hModule = LoadLibraryEx(lpFileName, NULL,
LOAD_WITH_ALTERED_SEARCH_PATH );
/* If the object was loaded with RTLD_GLOBAL, add it to list of global
* objects, so that its symbols may be retrieved even if the handle for
* the original program file is passed. POSIX says that if the same
* file is specified in multiple invocations, and any of them are
* RTLD_GLOBAL, even if any further invocations use RTLD_LOCAL, the
* symbols will remain global.
*/
if( !hModule )
save_err_str( lpFileName );
else if( (mode & RTLD_GLOBAL) )
global_add( &first_object, hModule );
}
/* Return to previous state of the error-mode bit flags. */
SetErrorMode( uMode );
return (void *) hModule;
}
static void free_auto( )
{
global_object *pobject = first_automatic_object.next;
if( pobject )
{
global_object *next;
for ( ; pobject; pobject = next )
{
next = pobject->next;
free( pobject );
}
first_automatic_object.next = NULL;
}
}
int dlclose( void *handle )
{
HMODULE hModule = (HMODULE) handle;
BOOL ret;
current_error = NULL;
ret = FreeLibrary( hModule );
/* If the object was loaded with RTLD_GLOBAL, remove it from list of global
* objects.
*/
if( ret )
{
HMODULE cur = GetModuleHandle( NULL );
global_rem( &first_object, hModule );
if( hModule == cur )
{
auto_ref_count--;
if( auto_ref_count < 0 )
auto_ref_count = 0;
if( !auto_ref_count )
free_auto( );
}
}
else
save_err_ptr_str( handle );
/* dlclose's return value in inverted in relation to FreeLibrary's. */
ret = !ret;
return (int) ret;
}
void *dlsym( void *handle, const char *name )
{
FARPROC symbol;
HMODULE hModule;
#ifdef UNICODE
wchar_t namew[MAX_PATH];
wmemset(namew, 0, MAX_PATH);
#endif
current_error = NULL;
symbol = GetProcAddress( (HMODULE) handle, name );
if( symbol != NULL )
goto end;
/* If the handle for the original program file is passed, also search
* in all globally loaded objects.
*/
hModule = GetModuleHandle( NULL );
if( hModule == handle )
{
global_object *pobject;
for( pobject = &first_object; pobject; pobject = pobject->next )
{
if( pobject->hModule )
{
symbol = GetProcAddress( pobject->hModule, name );
if( symbol != NULL )
goto end;
}
}
for( pobject = &first_automatic_object; pobject; pobject = pobject->next )
{
if( pobject->hModule )
{
symbol = GetProcAddress( pobject->hModule, name );
if( symbol != NULL )
goto end;
}
}
}
end:
if( symbol == NULL )
{
#ifdef UNICODE
size_t converted_chars;
size_t str_len = strlen(name) + 1;
#if ((defined(_WIN32) || defined(WIN32)) && (defined(_MSC_VER)) )
errno_t err = mbstowcs_s(&converted_chars, namew, str_len, name, str_len);
if (err != 0)
return NULL;
#else
mbstowcs(namew, name, str_len);
#endif
save_err_str( namew );
#else
save_err_str( name );
#endif
}
// warning C4054: 'type cast' : from function pointer 'FARPROC' to data pointer 'void *'
#ifdef _MSC_VER
#pragma warning( suppress: 4054 )
#endif
return (void*) symbol;
}
char *dlerror( void )
{
char *error_pointer = dlerror_buffer;
/* If this is the second consecutive call to dlerror, return NULL */
if (current_error == NULL)
{
return NULL;
}
#ifdef UNICODE
errno_t err = 0;
size_t converted_chars = 0;
size_t str_len = wcslen(current_error) + 1;
memset(error_pointer, 0, 65535);
# if ((defined(_WIN32) || defined(WIN32)) && (defined(_MSC_VER)) )
err = wcstombs_s(&converted_chars,
error_pointer, str_len * sizeof(char),
current_error, str_len * sizeof(wchar_t));
if (err != 0)
return NULL;
# else
wcstombs(error_pointer, current_error, str_len);
# endif
#else
memcpy(error_pointer, current_error, strlen(current_error) + 1);
#endif
/* POSIX says that invoking dlerror( ) a second time, immediately following
* a prior invocation, shall result in NULL being returned.
*/
current_error = NULL;
return error_pointer;
}
#ifdef SHARED
BOOL WINAPI DllMain( HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved )
{
(void) hinstDLL;
/*
* https://msdn.microsoft.com/en-us/library/windows/desktop/ms682583(v=vs.85).aspx
*
* When handling DLL_PROCESS_DETACH, a DLL should free resources such as heap
* memory only if the DLL is being unloaded dynamically (the lpReserved
* parameter is NULL).
*/
if( fdwReason == DLL_PROCESS_DETACH && !lpvReserved )
{
auto_ref_count = 0;
free_auto( );
}
return TRUE;
}
#endif
src/Win32_Interop/dlfcn.h
+59
View File
@@ -0,0 +1,59 @@
/*
* dlfcn-win32
* Copyright (c) 2007 Ramiro Polla
*
* dlfcn-win32 is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* dlfcn-win32 is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with dlfcn-win32; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef DLFCN_H
#define DLFCN_H
#ifdef __cplusplus
extern "C" {
#endif
#if defined(DLFCN_WIN32_EXPORTS)
# define DLFCN_EXPORT __declspec(dllexport)
#else
# define DLFCN_EXPORT
#endif
/* POSIX says these are implementation-defined.
* To simplify use with Windows API, we treat them the same way.
*/
#define RTLD_LAZY 0
#define RTLD_NOW 0
#define RTLD_GLOBAL (1 << 1)
#define RTLD_LOCAL (1 << 2)
/* These two were added in The Open Group Base Specifications Issue 6.
* Note: All other RTLD_* flags in any dlfcn.h are not standard compliant.
*/
#define RTLD_DEFAULT 0
#define RTLD_NEXT 0
DLFCN_EXPORT void *dlopen ( const char *file, int mode );
DLFCN_EXPORT int dlclose(void *handle);
DLFCN_EXPORT void *dlsym(void *handle, const char *name);
DLFCN_EXPORT char *dlerror(void);
#ifdef __cplusplus
}
#endif
#endif /* DLFCN_H */
src/adlist.c
+31 -4
View File
@@ -55,10 +55,8 @@ list *listCreate(void)
return list;
}
/* Free the whole list.
*
* This function can't fail. */
void listRelease(list *list)
/* Remove all the elements from the list without destroying the list itself. */
void listEmpty(list *list)
{
PORT_ULONG len;
listNode *current, *next;
@@ -71,6 +69,16 @@ void listRelease(list *list)
zfree(current);
current = next;
}
list->head = list->tail = NULL;
list->len = 0;
}
/* Free the whole list.
*
* This function can't fail. */
void listRelease(list *list)
{
listEmpty(list);
zfree(list);
}
@@ -336,3 +344,22 @@ void listRotate(list *list) {
tail->next = list->head;
list->head = tail;
}
/* Add all the elements of the list 'o' at the end of the
* list 'l'. The list 'other' remains empty but otherwise valid. */
void listJoin(list *l, list *o) {
if (o->head)
o->head->prev = l->tail;
if (l->tail)
l->tail->next = o->head;
else
l->head = o->head;
l->tail = o->tail;
l->len += o->len;
/* Setup other as an empty list. */
o->head = o->tail = NULL;
o->len = 0;
}
src/adlist.h
+2
View File
@@ -75,6 +75,7 @@ typedef struct list {
/* Prototypes */
list *listCreate(void);
void listRelease(list *list);
void listEmpty(list *list);
list *listAddNodeHead(list *list, void *value);
list *listAddNodeTail(list *list, void *value);
list *listInsertNode(list *list, listNode *old_node, void *value, int after);
@@ -88,6 +89,7 @@ listNode *listIndex(list *list, PORT_LONG index);
void listRewind(list *list, listIter *li);
void listRewindTail(list *list, listIter *li);
void listRotate(list *list);
void listJoin(list *l, list *o);
/* Directions for iterators */
#define AL_START_HEAD 0
src/ae.c
+14 -1
View File
@@ -86,6 +86,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) {
eventLoop->stop = 0;
eventLoop->maxfd = -1;
eventLoop->beforesleep = NULL;
eventLoop->aftersleep = NULL;
if (aeApiCreate(eventLoop) == -1) goto err;
/* Events with mask == AE_NONE are not set. So let's initialize the
* vector with it. */
@@ -363,6 +364,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
* if flags has AE_FILE_EVENTS set, file events are processed.
* if flags has AE_TIME_EVENTS set, time events are processed.
* if flags has AE_DONT_WAIT set the function returns ASAP until all
* if flags has AE_CALL_AFTER_SLEEP set, the aftersleep callback is called.
* the events that's possible to process without to wait are processed.
*
* The function returns the number of events processed. */
@@ -423,7 +425,14 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
}
}
/* Call the multiplexing API, will return only on timeout or when
* some event fires. */
numevents = aeApiPoll(eventLoop, tvp);
/* After sleep callback. */
if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP)
eventLoop->aftersleep(eventLoop);
for (j = 0; j < numevents; j++) {
aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd];
int mask = eventLoop->fired[j].mask;
@@ -478,7 +487,7 @@ void aeMain(aeEventLoop *eventLoop) {
while (!eventLoop->stop) {
if (eventLoop->beforesleep != NULL)
eventLoop->beforesleep(eventLoop);
aeProcessEvents(eventLoop, AE_ALL_EVENTS);
aeProcessEvents(eventLoop, AE_ALL_EVENTS|AE_CALL_AFTER_SLEEP);
}
}
@@ -489,3 +498,7 @@ char *aeGetApiName(void) {
void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep) {
eventLoop->beforesleep = beforesleep;
}
void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep) {
eventLoop->aftersleep = aftersleep;
}
src/ae.h
+3
View File
@@ -46,6 +46,7 @@
#define AE_TIME_EVENTS 2
#define AE_ALL_EVENTS (AE_FILE_EVENTS|AE_TIME_EVENTS)
#define AE_DONT_WAIT 4
#define AE_CALL_AFTER_SLEEP 8
#define AE_NOMORE -1
#define AE_DELETED_EVENT_ID -1
@@ -98,6 +99,7 @@ typedef struct aeEventLoop {
int stop;
void *apidata; /* This is used for polling API specific data */
aeBeforeSleepProc *beforesleep;
aeBeforeSleepProc *aftersleep;
} aeEventLoop;
/* Prototypes */
@@ -117,6 +119,7 @@ int aeWait(int fd, int mask, PORT_LONGLONG milliseconds);
void aeMain(aeEventLoop *eventLoop);
char *aeGetApiName(void);
void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep);
void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep);
int aeGetSetSize(aeEventLoop *eventLoop);
int aeResizeSetSize(aeEventLoop *eventLoop, int setsize);
src/anet.c
+22 -18
View File
@@ -33,6 +33,7 @@
#include "Win32_Interop/win32_types.h"
#include "Win32_Interop/win32fixes.h"
#include "Win32_Interop/win32_wsiocp2.h"
#include "Win32_Interop/Win32_Error.h"
#define ANET_NOTUSED(V) V
#include <Mstcpip.h>
#endif
@@ -76,7 +77,7 @@ int anetSetBlock(char *err, int fd, int non_block) {
* Note that fcntl(2) for F_GETFL and F_SETFL can't be
* interrupted by a signal. */
if ((flags = fcntl(fd, F_GETFL, 0)) == -1) { WIN_PORT_FIX /* fcntl default value for the 'flags' parameter */
anetSetError(err, "fcntl(F_GETFL): %s", strerror(errno));
anetSetError(err, "fcntl(F_GETFL): %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
@@ -86,7 +87,7 @@ int anetSetBlock(char *err, int fd, int non_block) {
flags &= ~O_NONBLOCK;
if (fcntl(fd, F_SETFL, flags) == -1) {
anetSetError(err, "fcntl(F_SETFL,O_NONBLOCK): %s", strerror(errno));
anetSetError(err, "fcntl(F_SETFL,O_NONBLOCK): %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
return ANET_OK;
@@ -109,7 +110,7 @@ int anetKeepAlive(char *err, int fd, int interval)
#ifdef _WIN32
if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1) {
anetSetError(err, "setsockopt SO_KEEPALIVE: %s", strerror(errno));
anetSetError(err, "setsockopt SO_KEEPALIVE: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
@@ -130,7 +131,7 @@ int anetKeepAlive(char *err, int fd, int interval)
NULL, 0, &dwBytesRet, NULL, NULL) == SOCKET_ERROR) {
anetSetError(err,
"WSAIotcl(SIO_KEEPALIVE_VALS) failed with error code %d\n",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
#else
@@ -175,7 +176,7 @@ static int anetSetTcpNoDelay(char *err, int fd, int val)
{
if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)) == -1)
{
anetSetError(err, "setsockopt TCP_NODELAY: %s", strerror(errno));
anetSetError(err, "setsockopt TCP_NODELAY: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
return ANET_OK;
@@ -196,7 +197,7 @@ int anetSetSendBuffer(char *err, int fd, int buffsize)
{
if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffsize, sizeof(buffsize)) == -1)
{
anetSetError(err, "setsockopt SO_SNDBUF: %s", strerror(errno));
anetSetError(err, "setsockopt SO_SNDBUF: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
return ANET_OK;
@@ -206,7 +207,7 @@ int anetTcpKeepAlive(char *err, int fd)
{
int yes = 1;
if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)) == -1) {
anetSetError(err, "setsockopt SO_KEEPALIVE: %s", strerror(errno));
anetSetError(err, "setsockopt SO_KEEPALIVE: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
return ANET_OK;
@@ -220,7 +221,7 @@ int anetSendTimeout(char *err, int fd, PORT_LONGLONG ms) {
tv.tv_sec = (int) ms/1000; WIN_PORT_FIX /* cast (int) */
tv.tv_usec = (ms%1000)*1000;
if (setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)) == -1) {
anetSetError(err, "setsockopt SO_SNDTIMEO: %s", strerror(errno));
anetSetError(err, "setsockopt SO_SNDTIMEO: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
return ANET_OK;
@@ -273,7 +274,7 @@ static int anetSetReuseAddr(char *err, int fd) {
/* Make sure connection-intensive things like the redis benckmark
* will be able to close/open sockets a zillion of times */
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) == -1) {
anetSetError(err, "setsockopt SO_REUSEADDR: %s", strerror(errno));
anetSetError(err, "setsockopt SO_REUSEADDR: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
return ANET_OK;
@@ -282,7 +283,7 @@ static int anetSetReuseAddr(char *err, int fd) {
static int anetCreateSocket(char *err, int domain) {
int s;
if ((s = socket(domain, SOCK_STREAM, IF_WIN32(IPPROTO_TCP,0))) == -1) {
anetSetError(err, "creating socket: %s", strerror(errno));
anetSetError(err, "creating socket: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
@@ -454,8 +455,10 @@ int anetUnixGenericConnect(char *err, char *path, int flags)
sa.sun_family = AF_LOCAL;
strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1);
if (flags & ANET_CONNECT_NONBLOCK) {
if (anetNonBlock(err,s) != ANET_OK)
if (anetNonBlock(err,s) != ANET_OK) {
close(s);
return ANET_ERR;
}
}
if (connect(s,(struct sockaddr*)&sa,sizeof(sa)) == -1) {
if (errno == EINPROGRESS &&
@@ -512,7 +515,7 @@ int anetWrite(int fd, char *buf, int count)
static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len, int backlog) {
if (bind(s,sa,len) == -1) {
anetSetError(err, "bind: %s", strerror(errno));
anetSetError(err, "bind: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
close(s);
return ANET_ERR;
}
@@ -522,7 +525,7 @@ static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len, int
#else
if (listen(s, backlog) == -1) {
#endif
anetSetError(err, "listen: %s", strerror(errno));
anetSetError(err, "listen: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
close(s);
return ANET_ERR;
}
@@ -532,7 +535,7 @@ static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len, int
static int anetV6Only(char *err, int s) {
int yes = 1;
if (setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,&yes,sizeof(yes)) == -1) {
anetSetError(err, "setsockopt: %s", strerror(errno));
anetSetError(err, "setsockopt: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
close(s);
return ANET_ERR;
}
@@ -545,7 +548,7 @@ static int anetSetExclusiveAddr(char *err, int fd) {
/* Make sure connection-intensive things like the redis benchmark
* will be able to close/open sockets a zillion of times */
if (setsockopt(fd, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, &yes, sizeof(yes)) == -1) {
anetSetError(err, "setsockopt SO_EXCLUSIVEADDRUSE: %s", strerror(errno));
anetSetError(err, "setsockopt SO_EXCLUSIVEADDRUSE: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
return ANET_OK;
@@ -554,7 +557,7 @@ static int anetSetExclusiveAddr(char *err, int fd) {
static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backlog)
{
int s, rv;
int s = -1, rv;
char _port[6]; /* strlen("65535") */
struct addrinfo hints, *servinfo, *p;
@@ -578,11 +581,12 @@ static int _anetTcpServer(char *err, int port, char *bindaddr, int af, int backl
goto end;
}
if (p == NULL) {
anetSetError(err, "unable to bind socket");
anetSetError(err, "unable to bind socket, errno: %d", errno);
goto error;
}
error:
if (s != -1) close(s);
s = ANET_ERR;
end:
freeaddrinfo(servinfo);
@@ -632,7 +636,7 @@ static int anetGenericAccept(char *err, int s, struct sockaddr *sa, socklen_t *l
if (errno == EINTR)
continue;
else {
anetSetError(err, "accept: %s", strerror(errno));
anetSetError(err, "accept: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return ANET_ERR;
}
}
src/aof.c
+21 -20
View File
@@ -29,6 +29,7 @@
#ifdef _WIN32
#include "Win32_Interop/win32_types.h"
#include "Win32_Interop/Win32_Error.h"
#endif
#include "server.h"
@@ -266,7 +267,7 @@ int startAppendOnly(void) {
"append only file %s (in server root dir %s): %s",
server.aof_filename,
cwdp ? cwdp : "unknown",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
return C_ERR;
}
if (rewriteAppendOnlyFileBackground() == C_ERR) {
@@ -371,7 +372,7 @@ void flushAppendOnlyFile(int force) {
if (nwritten == -1) {
if (can_log) {
serverLog(LL_WARNING,"Error writing to the AOF file: %s",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
server.aof_last_write_errno = errno;
}
} else {
@@ -388,7 +389,7 @@ void flushAppendOnlyFile(int force) {
serverLog(LL_WARNING, "Could not remove short write "
"from the append-only file. Redis may refuse "
"to load the AOF the next time it starts. "
"ftruncate: %s", strerror(errno));
"ftruncate: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
}
} else {
/* If the ftruncate() succeeded we can set nwritten to
@@ -641,7 +642,7 @@ int loadAppendOnlyFile(char *filename) {
}
if (fp == NULL) {
serverLog(LL_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));
serverLog(LL_WARNING,"Fatal error: can't open the append log file for reading: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
exit(1);
}
@@ -741,7 +742,7 @@ loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */
readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */
if (!feof(fp)) {
if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */
serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno));
serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
exit(1);
}
@@ -755,14 +756,14 @@ uxeof: /* Unexpected AOF end of file. */
serverLog(LL_WARNING,"Last valid command offset is invalid");
} else {
serverLog(LL_WARNING,"Error truncating the AOF file: %s",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
}
} else {
/* Make sure the AOF file descriptor points to the end of the
* file after the truncate call. */
if (server.aof_fd != -1 && lseek(server.aof_fd,0,SEEK_END) == -1) {
serverLog(LL_WARNING,"Can't seek the end of the AOF file: %s",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
} else {
serverLog(LL_WARNING,
"AOF loaded anyway because aof-load-truncated is enabled");
@@ -1043,7 +1044,7 @@ int rewriteAppendOnlyFile(char *filename) {
snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) getpid());
fp = fopen(tmpfile,IF_WIN32("wb","w"));
if (!fp) {
serverLog(LL_WARNING, "Opening the temp file for AOF rewrite in rewriteAppendOnlyFile(): %s", strerror(errno));
serverLog(LL_WARNING, "Opening the temp file for AOF rewrite in rewriteAppendOnlyFile(): %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return C_ERR;
}
@@ -1172,7 +1173,7 @@ int rewriteAppendOnlyFile(char *filename) {
/* Use RENAME to make sure the DB file is changed atomically only
* if the generate DB file is ok. */
if (rename(tmpfile,filename) == -1) {
serverLog(LL_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno));
serverLog(LL_WARNING,"Error moving temp append only file on the final destination: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
unlink(tmpfile);
return C_ERR;
}
@@ -1180,7 +1181,7 @@ int rewriteAppendOnlyFile(char *filename) {
return C_OK;
werr:
serverLog(LL_WARNING,"Write error writing append only file on disk: %s", strerror(errno));
serverLog(LL_WARNING,"Write error writing append only file on disk: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
fclose(fp);
unlink(tmpfile);
if (di) dictReleaseIterator(di);
@@ -1209,7 +1210,7 @@ void aofChildPipeReadable(aeEventLoop *el, int fd, void *privdata, int mask) {
* kernel can't buffer our write, or, the children was
* terminated. */
serverLog(LL_WARNING,"Can't send ACK to AOF child: %s",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
}
}
#ifndef _WIN32
@@ -1253,7 +1254,7 @@ int aofCreatePipes(void) {
error:
serverLog(LL_WARNING,"Error opening /setting AOF rewrite IPC pipes: %s",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
for (j = 0; j < 6; j++) if(fds[j] != -1) close(fds[j]);
return C_ERR;
}
@@ -1334,7 +1335,7 @@ int rewriteAppendOnlyFileBackground(void) {
if (childpid == -1) {
serverLog(LL_WARNING,
"Can't rewrite append only file in background: fork: %s",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
return C_ERR;
}
serverLog(LL_NOTICE,
@@ -1394,7 +1395,7 @@ void aofUpdateCurrentSize(void) {
latencyStartMonitor(latency);
if (redis_fstat(server.aof_fd,&sb) == -1) {
serverLog(LL_WARNING,"Unable to obtain the AOF file length. stat: %s",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
} else {
server.aof_current_size = sb.st_size;
}
@@ -1428,13 +1429,13 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
#endif
if (newfd == -1) {
serverLog(LL_WARNING,
"Unable to open the temporary AOF produced by the child: %s", strerror(errno));
"Unable to open the temporary AOF produced by the child: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
goto cleanup;
}
if (aofRewriteBufferWrite(newfd) == -1) {
serverLog(LL_WARNING,
"Error trying to flush the parent diff to the rewritten AOF: %s", strerror(errno));
"Error trying to flush the parent diff to the rewritten AOF: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
close(newfd);
goto cleanup;
}
@@ -1487,7 +1488,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
// Now rename the existing AOF file to allow the new file to be renamed
if (rename(server.aof_filename, tmpfile_win_old) == -1) {
serverLog(LL_WARNING,
"Error trying to rename the existing AOF to old tempfile: %s", strerror(errno));
"Error trying to rename the existing AOF to old tempfile: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
// Let's clean the Windows-specific temp file here
unlink(tmpfile_win_old);
goto cleanup;
@@ -1501,16 +1502,16 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
"Error trying to rename the temporary AOF file %s into %s: %s",
tmpfile,
server.aof_filename,
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
if (server.aof_fd != -1) {
if (rename(tmpfile_win_old, server.aof_filename) == -1) {
serverLog(LL_WARNING,
"Error trying to rename the old tempfile %s into the existing AOF file %s: %s",
tmpfile,
server.aof_filename,
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
serverLog(LL_WARNING,
"Error trying to rename the existing AOF from old tempfile: %s", strerror(errno));
"Error trying to rename the existing AOF from old tempfile: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
// The Windows-specific temp file couldn't be renamed to
// the configured AOF file, that should never happen but
// if it happens we leave the file behind in case the user
src/asciilogo.h
+1 -1
View File
@@ -34,7 +34,7 @@ char *ascii_logo =
" .-`` .-```. ```\\/ _.,_ ''-._ \n"
" ( ' , .-` | `, ) Running in %s mode\n"
" |`-._`-...-` __...-.``-._|'` _.-'| Port: %d\n"
" | `-._ `._ / _.-' | PID: %ld\n"
" | `-._ `._ / _.-' | PID: %Id\n" WIN_PORT_FIX /* %ld -> %Id */
" `-._ `-._ `-./ _.-' _.-' \n"
" |`-._`-._ `-.__.-' _.-'_.-'| \n"
" | `-._`-._ _.-'_.-' | http://redis.io \n"
src/atomicvar.h
+137
View File
@@ -0,0 +1,137 @@
/* This file implements atomic counters using __atomic or __sync macros if
* available, otherwise synchronizing different threads using a mutex.
*
* The exported interaface is composed of three macros:
*
* atomicIncr(var,count) -- Increment the atomic counter
* atomicGetIncr(var,oldvalue_var,count) -- Get and increment the atomic counter
* atomicDecr(var,count) -- Decrement the atomic counter
* atomicGet(var,dstvar) -- Fetch the atomic counter value
* atomicSet(var,value) -- Set the atomic counter value
*
* The variable 'var' should also have a declared mutex with the same
* name and the "_mutex" postfix, for instance:
*
* long myvar;
* pthread_mutex_t myvar_mutex;
* atomicSet(myvar,12345);
*
* If atomic primitives are availble (tested in config.h) the mutex
* is not used.
*
* Never use return value from the macros, instead use the AtomicGetIncr()
* if you need to get the current value and increment it atomically, like
* in the followign example:
*
* long oldvalue;
* atomicGetIncr(myvar,oldvalue,1);
* doSomethingWith(oldvalue);
*
* ----------------------------------------------------------------------------
*
* Copyright (c) 2015, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _WIN32
#include <pthread.h>
#else
#include "Win32_Interop\Win32_PThread.h"
#endif
#ifndef __ATOMIC_VAR_H
#define __ATOMIC_VAR_H
/* To test Redis with Helgrind (a Valgrind tool) it is useful to define
* the following macro, so that __sync macros are used: those can be detected
* by Helgrind (even if they are less efficient) so that no false positive
* is reported. */
// #define __ATOMIC_VAR_FORCE_SYNC_MACROS
#if !defined(__ATOMIC_VAR_FORCE_SYNC_MACROS) && defined(__ATOMIC_RELAXED) && !defined(__sun) && (!defined(__clang__) || !defined(__APPLE__) || __apple_build_version__ > 4210057)
/* Implementation using __atomic macros. */
#define atomicIncr(var,count) __atomic_add_fetch(&var,(count),__ATOMIC_RELAXED)
#define atomicGetIncr(var,oldvalue_var,count) do { \
oldvalue_var = __atomic_fetch_add(&var,(count),__ATOMIC_RELAXED); \
} while(0)
#define atomicDecr(var,count) __atomic_sub_fetch(&var,(count),__ATOMIC_RELAXED)
#define atomicGet(var,dstvar) do { \
dstvar = __atomic_load_n(&var,__ATOMIC_RELAXED); \
} while(0)
#define atomicSet(var,value) __atomic_store_n(&var,value,__ATOMIC_RELAXED)
#define REDIS_ATOMIC_API "atomic-builtin"
#elif defined(HAVE_ATOMIC)
/* Implementation using __sync macros. */
#define atomicIncr(var,count) __sync_add_and_fetch(&var,(count))
#define atomicGetIncr(var,oldvalue_var,count) do { \
oldvalue_var = __sync_fetch_and_add(&var,(count)); \
} while(0)
#define atomicDecr(var,count) __sync_sub_and_fetch(&var,(count))
#define atomicGet(var,dstvar) do { \
dstvar = __sync_sub_and_fetch(&var,0); \
} while(0)
#define atomicSet(var,value) do { \
while(!__sync_bool_compare_and_swap(&var,var,value)); \
} while(0)
#define REDIS_ATOMIC_API "sync-builtin"
#else
/* Implementation using pthread mutex. */
#define atomicIncr(var,count) do { \
pthread_mutex_lock(&var ## _mutex); \
var += (count); \
pthread_mutex_unlock(&var ## _mutex); \
} while(0)
#define atomicGetIncr(var,oldvalue_var,count) do { \
pthread_mutex_lock(&var ## _mutex); \
oldvalue_var = var; \
var += (count); \
pthread_mutex_unlock(&var ## _mutex); \
} while(0)
#define atomicDecr(var,count) do { \
pthread_mutex_lock(&var ## _mutex); \
var -= (count); \
pthread_mutex_unlock(&var ## _mutex); \
} while(0)
#define atomicGet(var,dstvar) do { \
pthread_mutex_lock(&var ## _mutex); \
dstvar = var; \
pthread_mutex_unlock(&var ## _mutex); \
} while(0)
#define atomicSet(var,value) do { \
pthread_mutex_lock(&var ## _mutex); \
var = value; \
pthread_mutex_unlock(&var ## _mutex); \
} while(0)
#define REDIS_ATOMIC_API "pthread-mutex"
#endif
#endif /* __ATOMIC_VAR_H */
src/bio.c
+49 -8
View File
@@ -7,7 +7,7 @@
* file is slow, blocking the server.
*
* In the future we'll either continue implementing new things we need or
* we'll switch to libeio. However there are probably PORT_LONG term uses for this
* we'll switch to libeio. However there are probably long term uses for this
* file as we may want to put here Redis specific background tasks (for instance
* it is not impossible that we'll need a non blocking FLUSHDB/FLUSHALL
* implementation).
@@ -61,6 +61,7 @@
#include "Win32_Interop/win32fixes.h"
#include "Win32_Interop/Win32_PThread.h"
#include "Win32_Interop/Win32_ThreadControl.h"
#include "Win32_Interop/Win32_Error.h"
#endif
#include "server.h"
@@ -68,7 +69,8 @@
static pthread_t bio_threads[BIO_NUM_OPS];
static pthread_mutex_t bio_mutex[BIO_NUM_OPS];
static pthread_cond_t bio_condvar[BIO_NUM_OPS];
static pthread_cond_t bio_newjob_cond[BIO_NUM_OPS];
static pthread_cond_t bio_step_cond[BIO_NUM_OPS];
static list *bio_jobs[BIO_NUM_OPS];
/* The following array is used to hold the number of pending jobs for every
* OP type. This allows us to export the bioPendingJobsOfType() API that is
@@ -88,6 +90,9 @@ struct bio_job {
};
void *bioProcessBackgroundJobs(void *arg);
void lazyfreeFreeObjectFromBioThread(robj *o);
void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2);
void lazyfreeFreeSlotsMapFromBioThread(zskiplist *sl);
/* Make sure we have enough stack to perform all the things we do in the
* main thread. */
@@ -103,7 +108,8 @@ void bioInit(void) {
/* Initialization of state vars and objects */
for (j = 0; j < BIO_NUM_OPS; j++) {
pthread_mutex_init(&bio_mutex[j],NULL);
pthread_cond_init(&bio_condvar[j],NULL);
pthread_cond_init(&bio_newjob_cond[j],NULL);
pthread_cond_init(&bio_step_cond[j],NULL);
bio_jobs[j] = listCreate();
bio_pending[j] = 0;
}
@@ -138,7 +144,7 @@ void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) {
pthread_mutex_lock(&bio_mutex[type]);
listAddNodeTail(bio_jobs[type],job);
bio_pending[type]++;
pthread_cond_signal(&bio_condvar[type]);
pthread_cond_signal(&bio_newjob_cond[type]);
pthread_mutex_unlock(&bio_mutex[type]);
}
@@ -150,7 +156,7 @@ void *bioProcessBackgroundJobs(void *arg) {
/* Check that the type is within the right interval. */
if (type >= BIO_NUM_OPS) {
serverLog(LL_WARNING,
"Warning: bio thread started with wrong type %lu",type);
"Warning: bio thread started with wrong type %Iu",type); WIN_PORT_FIX /* %lu -> %Iu */
return NULL;
}
@@ -165,14 +171,13 @@ void *bioProcessBackgroundJobs(void *arg) {
#endif
pthread_mutex_lock(&bio_mutex[type]);
/* Block SIGALRM so we are sure that only the main thread will
* receive the watchdog signal. */
sigemptyset(&sigset);
sigaddset(&sigset, SIGALRM);
if (pthread_sigmask(SIG_BLOCK, &sigset, NULL))
serverLog(LL_WARNING,
"Warning: can't mask SIGALRM in bio.c thread: %s", strerror(errno));
"Warning: can't mask SIGALRM in bio.c thread: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
while(1) {
listNode *ln;
@@ -180,7 +185,7 @@ void *bioProcessBackgroundJobs(void *arg) {
/* The loop always starts with the lock hold. */
if (listLength(bio_jobs[type]) == 0) {
WIN32_ONLY(WorkerThread_EnterSafeMode());
pthread_cond_wait(&bio_condvar[type],&bio_mutex[type]);
pthread_cond_wait(&bio_newjob_cond[type],&bio_mutex[type]);
WIN32_ONLY(pthread_mutex_unlock(&bio_mutex[type]));
WIN32_ONLY(WorkerThread_ExitSafeMode());
WIN32_ONLY(pthread_mutex_lock(&bio_mutex[type]));
@@ -198,11 +203,25 @@ void *bioProcessBackgroundJobs(void *arg) {
close((PORT_LONG)job->arg1);
} else if (type == BIO_AOF_FSYNC) {
aof_fsync((PORT_LONG)job->arg1);
} else if (type == BIO_LAZY_FREE) {
/* What we free changes depending on what arguments are set:
* arg1 -> free the object at pointer.
* arg2 & arg3 -> free two dictionaries (a Redis DB).
* only arg3 -> free the skiplist. */
if (job->arg1)
lazyfreeFreeObjectFromBioThread(job->arg1);
else if (job->arg2 && job->arg3)
lazyfreeFreeDatabaseFromBioThread(job->arg2,job->arg3);
else if (job->arg3)
lazyfreeFreeSlotsMapFromBioThread(job->arg3);
} else {
serverPanic("Wrong job type in bioProcessBackgroundJobs().");
}
zfree(job);
/* Unblock threads blocked on bioWaitStepOfType() if any. */
pthread_cond_broadcast(&bio_step_cond[type]);
/* Lock again before reiterating the loop, if there are no longer
* jobs to process we'll block again in pthread_cond_wait(). */
pthread_mutex_lock(&bio_mutex[type]);
@@ -220,6 +239,28 @@ PORT_ULONGLONG bioPendingJobsOfType(int type) {
return val;
}
/* If there are pending jobs for the specified type, the function blocks
* and waits that the next job was processed. Otherwise the function
* does not block and returns ASAP.
*
* The function returns the number of jobs still to process of the
* requested type.
*
* This function is useful when from another thread, we want to wait
* a bio.c thread to do more work in a blocking way.
*/
PORT_ULONGLONG bioWaitStepOfType(int type) {
PORT_ULONGLONG val;
pthread_mutex_lock(&bio_mutex[type]);
val = bio_pending[type];
if (val != 0) {
pthread_cond_wait(&bio_step_cond[type],&bio_mutex[type]);
val = bio_pending[type];
}
pthread_mutex_unlock(&bio_mutex[type]);
return val;
}
/* Kill the running bio threads in an unclean way. This function should be
* used only when it's critical to stop the threads for some reason.
* Currently Redis does this only on crash (for instance on SIGSEGV) in order
src/bio.h
+3 -2
View File
@@ -31,11 +31,12 @@
void bioInit(void);
void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3);
PORT_ULONGLONG bioPendingJobsOfType(int type);
void bioWaitPendingJobsLE(int type, PORT_ULONGLONG num);
PORT_ULONGLONG bioWaitStepOfType(int type);
time_t bioOlderJobOfType(int type);
void bioKillThreads(void);
/* Background job opcodes */
#define BIO_CLOSE_FILE 0 /* Deferred close(2) syscall. */
#define BIO_AOF_FSYNC 1 /* Deferred AOF fsync. */
#define BIO_NUM_OPS 2
#define BIO_LAZY_FREE 2 /* Deferred objects freeing. */
#define BIO_NUM_OPS 3
src/bitops.c
+22 -10
View File
@@ -104,6 +104,7 @@ PORT_LONG redisBitpos(void *s, PORT_ULONG count, int bit) {
PORT_ULONG skipval, word = 0, one;
PORT_LONG pos = 0; /* Position of bit, to return to the caller. */
PORT_ULONG j;
int found;
/* Process whole words first, seeking for first word that is not
* all ones or all zeros respectively if we are lookig for zeros
@@ -117,21 +118,27 @@ PORT_LONG redisBitpos(void *s, PORT_ULONG count, int bit) {
/* Skip initial bits not aligned to sizeof(PORT_ULONG) byte by byte. */
skipval = bit ? 0 : UCHAR_MAX;
c = (unsigned char*) s;
found = 0;
while((PORT_ULONG)c & (sizeof(*l)-1) && count) {
if (*c != skipval) break;
if (*c != skipval) {
found = 1;
break;
}
c++;
count--;
pos += 8;
}
/* Skip bits with full word step. */
skipval = bit ? 0 : PORT_ULONG_MAX;
l = (PORT_ULONG*) c;
while (count >= sizeof(*l)) {
if (*l != skipval) break;
l++;
count -= sizeof(*l);
pos += sizeof(*l)*8;
if (!found) {
skipval = bit ? 0 : PORT_ULONG_MAX;
while (count >= sizeof(*l)) {
if (*l != skipval) break;
l++;
count -= sizeof(*l);
pos += sizeof(*l)*8;
}
}
/* Load bytes into "word" considering the first byte as the most significant
@@ -648,14 +655,17 @@ void bitopCommand(client *c) {
/* Compute the bit operation, if at least one string is not empty. */
if (maxlen) {
res = (unsigned char*) sdsnewlen(NULL, maxlen);
res = (unsigned char*) sdsnewlen(NULL,maxlen);
unsigned char output, byte;
PORT_ULONG i;
/* Fast path: as far as we have data for all the input bitmaps we
* can take a fast path that performs much better than the
* vanilla algorithm. */
* vanilla algorithm. On ARM we skip the fast path since it will
* result in GCC compiling the code using multiple-words load/store
* operations that are not supported even in ARM >= v6. */
j = 0;
#ifndef USE_ALIGNED_ACCESS
if (minlen >= sizeof(PORT_ULONG)*4 && numkeys <= 16) {
PORT_ULONG *lp[16];
PORT_ULONG *lres = (PORT_ULONG*) res;
@@ -716,6 +726,7 @@ void bitopCommand(client *c) {
}
}
}
#endif
/* j is set to the next byte to process by the previous loop. */
for (; j < maxlen; j++) {
@@ -957,7 +968,8 @@ void bitfieldCommand(client *c) {
if (opcode != BITFIELDOP_GET) {
readonly = 0;
higest_write_offset = bitoffset + bits - 1;
if (higest_write_offset < bitoffset + bits - 1)
higest_write_offset = bitoffset + bits - 1;
/* INCRBY and SET require another argument. */
if (getLongLongFromObjectOrReply(c,c->argv[j+3],&i64,NULL) != C_OK){
zfree(ops);
src/blocked.c
+6 -1
View File
@@ -136,6 +136,8 @@ void unblockClient(client *c) {
unblockClientWaitingData(c);
} else if (c->btype == BLOCKED_WAIT) {
unblockClientWaitingReplicas(c);
} else if (c->btype == BLOCKED_MODULE) {
unblockClientFromModule(c);
} else {
serverPanic("Unknown btype in unblockClient().");
}
@@ -153,12 +155,15 @@ void unblockClient(client *c) {
}
/* This function gets called when a blocked client timed out in order to
* send it a reply of some kind. */
* send it a reply of some kind. After this function is called,
* unblockClient() will be called with the same client as argument. */
void replyToBlockedClientTimedOut(client *c) {
if (c->btype == BLOCKED_LIST) {
addReply(c,shared.nullmultibulk);
} else if (c->btype == BLOCKED_WAIT) {
addReplyLongLong(c,replicationCountAcksByOffset(c->bpop.reploffset));
} else if (c->btype == BLOCKED_MODULE) {
moduleBlockedClientTimedOut(c);
} else {
serverPanic("Unknown btype in replyToBlockedClientTimedOut().");
}
src/childinfo.c
+85
View File
@@ -0,0 +1,85 @@
/*
* Copyright (c) 2016, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "server.h"
POSIX_ONLY(#include <unistd.h>)
/* Open a child-parent channel used in order to move information about the
* RDB / AOF saving process from the child to the parent (for instance
* the amount of copy on write memory used) */
void openChildInfoPipe(void) {
if (pipe(server.child_info_pipe) == -1) {
/* On error our two file descriptors should be still set to -1,
* but we call anyway cloesChildInfoPipe() since can't hurt. */
closeChildInfoPipe();
} else if (anetNonBlock(NULL,server.child_info_pipe[0]) != ANET_OK) {
closeChildInfoPipe();
} else {
memset(&server.child_info_data,0,sizeof(server.child_info_data));
}
}
/* Close the pipes opened with openChildInfoPipe(). */
void closeChildInfoPipe(void) {
if (server.child_info_pipe[0] != -1 ||
server.child_info_pipe[1] != -1)
{
close(server.child_info_pipe[0]);
close(server.child_info_pipe[1]);
server.child_info_pipe[0] = -1;
server.child_info_pipe[1] = -1;
}
}
/* Send COW data to parent. The child should call this function after populating
* the corresponding fields it want to sent (according to the process type). */
void sendChildInfo(int ptype) {
if (server.child_info_pipe[1] == -1) return;
server.child_info_data.magic = CHILD_INFO_MAGIC;
server.child_info_data.process_type = ptype;
ssize_t wlen = sizeof(server.child_info_data);
if (write(server.child_info_pipe[1],&server.child_info_data,wlen) != wlen) {
/* Nothing to do on error, this will be detected by the other side. */
}
}
/* Receive COW data from parent. */
void receiveChildInfo(void) {
if (server.child_info_pipe[0] == -1) return;
ssize_t wlen = sizeof(server.child_info_data);
if (read(server.child_info_pipe[0],&server.child_info_data,wlen) == wlen &&
server.child_info_data.magic == CHILD_INFO_MAGIC)
{
if (server.child_info_data.process_type == CHILD_INFO_TYPE_RDB) {
server.stat_rdb_cow_bytes = server.child_info_data.cow_size;
} else if (server.child_info_data.process_type == CHILD_INFO_TYPE_AOF) {
server.stat_aof_cow_bytes = server.child_info_data.cow_size;
}
}
}
src/cluster.c
+363 -118
View File
@@ -42,7 +42,10 @@ POSIX_ONLY(#include <sys/socket.h>)
POSIX_ONLY(#include <sys/file.h>)
#include <math.h>
WIN32_ONLY(extern int WSIOCP_QueueAccept(int listenfd);)
#ifdef _WIN32
extern int WSIOCP_QueueAccept(int listenfd);
#include "Win32_Interop/Win32_Error.h"
#endif
/* A global reference to myself is handy to make code more clear.
* Myself always points to server.cluster->myself, that is, the clusterNode
@@ -101,7 +104,7 @@ int clusterLoadConfig(char *filename) {
} else {
serverLog(LL_WARNING,
"Loading the cluster node config from %s: %s",
filename, strerror(errno));
filename, IF_WIN32(wsa_strerror(errno), strerror(errno)));
exit(1);
}
}
@@ -131,7 +134,7 @@ int clusterLoadConfig(char *filename) {
/* Skip blank lines, they can be created either by users manually
* editing nodes.conf or by the config writing process if stopped
* before the truncate() call. */
if (line[0] == '\n') continue;
if (line[0] == '\n' || line[0] == '\0') continue;
/* Split the line into arguments for processing. */
argv = sdssplitargs(line,&argc);
@@ -170,7 +173,17 @@ int clusterLoadConfig(char *filename) {
if ((p = strrchr(argv[1],':')) == NULL) goto fmterr;
*p = '\0';
memcpy(n->ip,argv[1],strlen(argv[1])+1);
n->port = atoi(p+1);
char *port = p+1;
char *busp = strchr(port,'@');
if (busp) {
*busp = '\0';
busp++;
}
n->port = atoi(port);
/* In older versions of nodes.conf the "@busport" part is missing.
* In this case we set it to the default offset of 10000 from the
* base port. */
n->cport = busp ? atoi(busp) : n->port + CLUSTER_PORT_INCR;
/* Parse flags */
p = s = argv[2];
@@ -372,7 +385,7 @@ int clusterLockConfig(char *filename) {
if (fd == -1) {
serverLog(LL_WARNING,
"Can't open %s in order to acquire a lock: %s",
filename, strerror(errno));
filename, IF_WIN32(wsa_strerror(errno), strerror(errno)));
return C_ERR;
}
@@ -392,20 +405,19 @@ int clusterLockConfig(char *filename) {
if (err == ERROR_LOCK_VIOLATION) {
#endif
serverLog(LL_WARNING,
"Sorry, the cluster configuration file %s is already used "
"by a different Redis Cluster node. Please make sure that "
"different nodes use different cluster configuration "
"files.", filename);
"Sorry, the cluster configuration file %s is already used "
"by a different Redis Cluster node. Please make sure that "
"different nodes use different cluster configuration "
"files.", filename);
} else {
serverLog(LL_WARNING,
"Impossible to lock %s: %d", filename, err);
"Impossible to lock %s: %s", filename, IF_WIN32(wsa_strerror(errno), strerror(errno)));
}
close(fd);
return C_ERR;
}
/* Lock acquired: leak the 'fd' by not closing it, so that we'll retain the
* lock to the file as PORT_LONG as the process exists. */
* lock to the file as long as the process exists. */
#endif /* __sun */
return C_OK;
@@ -429,8 +441,11 @@ void clusterInit(void) {
server.cluster->failover_auth_epoch = 0;
server.cluster->cant_failover_reason = CLUSTER_CANT_FAILOVER_NONE;
server.cluster->lastVoteEpoch = 0;
server.cluster->stats_bus_messages_sent = 0;
server.cluster->stats_bus_messages_received = 0;
for (int i = 0; i < CLUSTERMSG_TYPE_COUNT; i++) {
server.cluster->stats_bus_messages_sent[i] = 0;
server.cluster->stats_bus_messages_received[i] = 0;
}
server.cluster->stats_pfail_nodes = 0;
memset(server.cluster->slots,0, sizeof(server.cluster->slots));
clusterCloseAllSlots();
@@ -484,12 +499,19 @@ void clusterInit(void) {
}
}
/* The slots -> keys map is a sorted set. Init it. */
server.cluster->slots_to_keys = zslCreate();
/* The slots -> keys map is a radix tree. Initialize it here. */
server.cluster->slots_to_keys = raxNew();
memset(server.cluster->slots_keys_count,0,
sizeof(server.cluster->slots_keys_count));
/* Set myself->port to my listening port, we'll just need to discover
* the IP address via MEET messages. */
/* Set myself->port / cport to my listening ports, we'll just need to
* discover the IP address via MEET messages. */
myself->port = server.port;
myself->cport = server.port+CLUSTER_PORT_INCR;
if (server.cluster_announce_port)
myself->port = server.cluster_announce_port;
if (server.cluster_announce_bus_port)
myself->cport = server.cluster_announce_bus_port;
server.cluster->mf_end = 0;
resetManualFailover();
@@ -513,7 +535,7 @@ void clusterReset(int hard) {
if (nodeIsSlave(myself)) {
clusterSetNodeAsMaster(myself);
replicationUnsetMaster();
emptyDb(NULL);
emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
}
/* Close slots, reset manual failover state. */
@@ -697,6 +719,7 @@ clusterNode *createClusterNode(char *nodename, int flags) {
node->link = NULL;
memset(node->ip,0,sizeof(node->ip));
node->port = 0;
node->cport = 0;
node->fail_reports = listCreate();
node->voted_time = 0;
node->orphaned_time = 0;
@@ -1105,7 +1128,7 @@ void clusterHandleConfigEpochCollision(clusterNode *sender) {
* entries from the black list. This is an O(N) operation but it is not a
* problem since add / exists operations are called very infrequently and
* the hash table is supposed to contain very little elements at max.
* However without the cleanup during PORT_LONG uptimes and with some automated
* However without the cleanup during long uptimes and with some automated
* node add/removal procedures, entries could accumulate. */
void clusterBlacklistCleanup(void) {
dictIterator *di;
@@ -1239,7 +1262,7 @@ void clearNodeFailureIfNeeded(clusterNode *node) {
/* Return true if we already have a node in HANDSHAKE state matching the
* specified ip address and port number. This function is used in order to
* avoid adding a new handshake node for the same address multiple times. */
int clusterHandshakeInProgress(char *ip, int port) {
int clusterHandshakeInProgress(char *ip, int port, int cport) {
dictIterator *di;
dictEntry *de;
@@ -1248,7 +1271,9 @@ int clusterHandshakeInProgress(char *ip, int port) {
clusterNode *node = dictGetVal(de);
if (!nodeInHandshake(node)) continue;
if (!strcasecmp(node->ip,ip) && node->port == port) break;
if (!strcasecmp(node->ip,ip) &&
node->port == port &&
node->cport == cport) break;
}
dictReleaseIterator(di);
return de != NULL;
@@ -1261,7 +1286,7 @@ int clusterHandshakeInProgress(char *ip, int port) {
*
* EAGAIN - There is already an handshake in progress for this address.
* EINVAL - IP or port are not valid. */
int clusterStartHandshake(char *ip, int port) {
int clusterStartHandshake(char *ip, int port, int cport) {
clusterNode *n;
char norm_ip[NET_IP_STR_LEN];
struct sockaddr_storage sa;
@@ -1281,7 +1306,7 @@ int clusterStartHandshake(char *ip, int port) {
}
/* Port sanity check */
if (port <= 0 || port > (65535-CLUSTER_PORT_INCR)) {
if (port <= 0 || port > 65535 || cport <= 0 || cport > 65535) {
errno = EINVAL;
return 0;
}
@@ -1298,7 +1323,7 @@ int clusterStartHandshake(char *ip, int port) {
(void*)&(((struct sockaddr_in6 *)&sa)->sin6_addr),
norm_ip,NET_IP_STR_LEN);
if (clusterHandshakeInProgress(norm_ip,port)) {
if (clusterHandshakeInProgress(norm_ip,port,cport)) {
errno = EAGAIN;
return 0;
}
@@ -1309,6 +1334,7 @@ int clusterStartHandshake(char *ip, int port) {
n = createClusterNode(NULL,CLUSTER_NODE_HANDSHAKE|CLUSTER_NODE_MEET);
memcpy(n->ip,norm_ip,sizeof(n->ip));
n->port = port;
n->cport = cport;
clusterAddNode(n);
return 1;
}
@@ -1327,13 +1353,16 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
clusterNode *node;
sds ci;
ci = representClusterNodeFlags(sdsempty(), flags);
serverLog(LL_DEBUG,"GOSSIP %.40s %s:%d %s",
g->nodename,
g->ip,
ntohs(g->port),
ci);
sdsfree(ci);
if (server.verbosity == LL_DEBUG) {
ci = representClusterNodeFlags(sdsempty(), flags);
serverLog(LL_DEBUG,"GOSSIP %.40s %s:%d@%d %s",
g->nodename,
g->ip,
ntohs(g->port),
ntohs(g->cport),
ci);
sdsfree(ci);
}
/* Update our state accordingly to the gossip sections */
node = clusterLookupNode(g->nodename);
@@ -1357,6 +1386,28 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
}
}
/* If from our POV the node is up (no failure flags are set),
* we have no pending ping for the node, nor we have failure
* reports for this node, update the last pong time with the
* one we see from the other nodes. */
if (!(flags & (CLUSTER_NODE_FAIL|CLUSTER_NODE_PFAIL)) &&
node->ping_sent == 0 &&
clusterNodeFailureReportsCount(node) == 0)
{
mstime_t pongtime = ntohl(g->pong_received);
pongtime *= 1000; /* Convert back to milliseconds. */
/* Replace the pong time with the received one only if
* it's greater than our view but is not in the future
* (with 500 milliseconds tolerance) from the POV of our
* clock. */
if (pongtime <= (server.mstime+500) &&
pongtime > node->pong_received)
{
node->pong_received = pongtime;
}
}
/* If we already know this node, but it is not reachable, and
* we see a different address in the gossip section of a node that
* can talk with this other node, update the address, disconnect
@@ -1365,11 +1416,14 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
if (node->flags & (CLUSTER_NODE_FAIL|CLUSTER_NODE_PFAIL) &&
!(flags & CLUSTER_NODE_NOADDR) &&
!(flags & (CLUSTER_NODE_FAIL|CLUSTER_NODE_PFAIL)) &&
(strcasecmp(node->ip,g->ip) || node->port != ntohs(g->port)))
(strcasecmp(node->ip,g->ip) ||
node->port != ntohs(g->port) ||
node->cport != ntohs(g->cport)))
{
if (node->link) freeClusterLink(node->link);
memcpy(node->ip,g->ip,NET_IP_STR_LEN);
node->port = ntohs(g->port);
node->cport = ntohs(g->cport);
node->flags &= ~CLUSTER_NODE_NOADDR;
}
} else {
@@ -1383,7 +1437,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
!(flags & CLUSTER_NODE_NOADDR) &&
!clusterBlacklistExists(g->nodename))
{
clusterStartHandshake(g->ip,ntohs(g->port));
clusterStartHandshake(g->ip,ntohs(g->port),ntohs(g->cport));
}
}
@@ -1392,23 +1446,36 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
}
}
/* IP -> string conversion. 'buf' is supposed to at least be 46 bytes. */
void nodeIp2String(char *buf, clusterLink *link) {
anetPeerToString(link->fd, buf, NET_IP_STR_LEN, NULL);
/* IP -> string conversion. 'buf' is supposed to at least be 46 bytes.
* If 'announced_ip' length is non-zero, it is used instead of extracting
* the IP from the socket peer address. */
void nodeIp2String(char *buf, clusterLink *link, char *announced_ip) {
if (announced_ip[0] != '\0') {
memcpy(buf,announced_ip,NET_IP_STR_LEN);
buf[NET_IP_STR_LEN-1] = '\0'; /* We are not sure the input is sane. */
} else {
anetPeerToString(link->fd, buf, NET_IP_STR_LEN, NULL);
}
}
/* Update the node address to the IP address that can be extracted
* from link->fd, and at the specified port.
* Also disconnect the node link so that we'll connect again to the new
* address.
* from link->fd, or if hdr->myip is non empty, to the address the node
* is announcing us. The port is taken from the packet header as well.
*
* If the address or port changed, disconnect the node link so that we'll
* connect again to the new address.
*
* If the ip/port pair are already correct no operation is performed at
* all.
*
* The function returns 0 if the node address is still the same,
* otherwise 1 is returned. */
int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, int port) {
int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link,
clusterMsg *hdr)
{
char ip[NET_IP_STR_LEN] = {0};
int port = ntohs(hdr->port);
int cport = ntohs(hdr->cport);
/* We don't proceed if the link is the same as the sender link, as this
* function is designed to see if the node link is consistent with the
@@ -1418,12 +1485,14 @@ int nodeUpdateAddressIfNeeded(clusterNode *node, clusterLink *link, int port) {
* it is safe to call during packet processing. */
if (link == node->link) return 0;
nodeIp2String(ip,link);
if (node->port == port && strcmp(ip,node->ip) == 0) return 0;
nodeIp2String(ip,link,hdr->myip);
if (node->port == port && node->cport == cport &&
strcmp(ip,node->ip) == 0) return 0;
/* IP / port is different, update it. */
memcpy(node->ip,ip,sizeof(ip));
node->port = port;
node->cport = cport;
if (node->link) freeClusterLink(node->link);
node->flags &= ~CLUSTER_NODE_NOADDR;
serverLog(LL_WARNING,"Address updated for node %.40s, now %s:%d",
@@ -1570,8 +1639,9 @@ int clusterProcessPacket(clusterLink *link) {
uint32_t totlen = ntohl(hdr->totlen);
uint16_t type = ntohs(hdr->type);
server.cluster->stats_bus_messages_received++;
serverLog(LL_DEBUG,"--- Processing packet of type %d, %Iu bytes", WIN_PORT_FIX /* %lu -> %Iu */
if (type < CLUSTERMSG_TYPE_COUNT)
server.cluster->stats_bus_messages_received[type]++;
serverLog(LL_DEBUG,"--- Processing packet of type %d, %Iu bytes", WIN_PORT_FIX /* %lu -> %Iu */
type, (PORT_ULONG) totlen);
/* Perform sanity checks */
@@ -1662,7 +1732,7 @@ int clusterProcessPacket(clusterLink *link) {
/* We use incoming MEET messages in order to set the address
* for 'myself', since only other cluster nodes will send us
* MEET messagses on handshakes, when the cluster joins, or
* MEET messages on handshakes, when the cluster joins, or
* later if we changed address, and those nodes will use our
* official address to connect to us. So by obtaining this address
* from the socket is a simple way to discover / update our own
@@ -1671,7 +1741,9 @@ int clusterProcessPacket(clusterLink *link) {
* However if we don't have an address at all, we update the address
* even with a normal PING packet. If it's wrong it will be fixed
* by MEET later. */
if (type == CLUSTERMSG_TYPE_MEET || myself->ip[0] == '\0') {
if ((type == CLUSTERMSG_TYPE_MEET || myself->ip[0] == '\0') &&
server.cluster_announce_ip == NULL)
{
char ip[NET_IP_STR_LEN];
if (anetSockName(link->fd,ip,sizeof(ip),NULL) != -1 &&
@@ -1692,8 +1764,9 @@ int clusterProcessPacket(clusterLink *link) {
clusterNode *node;
node = createClusterNode(NULL,CLUSTER_NODE_HANDSHAKE);
nodeIp2String(node->ip,link);
nodeIp2String(node->ip,link,hdr->myip);
node->port = ntohs(hdr->port);
node->cport = ntohs(hdr->cport);
clusterAddNode(node);
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
}
@@ -1723,7 +1796,7 @@ int clusterProcessPacket(clusterLink *link) {
serverLog(LL_VERBOSE,
"Handshake: we already know node %.40s, "
"updating the address if needed.", sender->name);
if (nodeUpdateAddressIfNeeded(sender,link,ntohs(hdr->port)))
if (nodeUpdateAddressIfNeeded(sender,link,hdr))
{
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|
CLUSTER_TODO_UPDATE_STATE);
@@ -1755,6 +1828,7 @@ int clusterProcessPacket(clusterLink *link) {
link->node->flags |= CLUSTER_NODE_NOADDR;
link->node->ip[0] = '\0';
link->node->port = 0;
link->node->cport = 0;
freeClusterLink(link);
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
return 0;
@@ -1764,7 +1838,7 @@ int clusterProcessPacket(clusterLink *link) {
/* Update the node address if it changed. */
if (sender && type == CLUSTERMSG_TYPE_PING &&
!nodeInHandshake(sender) &&
nodeUpdateAddressIfNeeded(sender,link,ntohs(hdr->port)))
nodeUpdateAddressIfNeeded(sender,link,hdr))
{
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|
CLUSTER_TODO_UPDATE_STATE);
@@ -2117,7 +2191,7 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
if (nread <= 0) {
/* I/O error... */
serverLog(LL_DEBUG,"I/O error reading from node link: %s",
(nread == 0) ? "connection closed" : strerror(errno));
(nread == 0) ? "connection closed" : IF_WIN32(wsa_strerror(errno), strerror(errno)));
handleLinkIOError(link);
return;
} else {
@@ -2151,7 +2225,12 @@ void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) {
clusterWriteHandler,link);
link->sndbuf = sdscatlen(link->sndbuf, msg, msglen);
server.cluster->stats_bus_messages_sent++;
/* Populate sent messages stats. */
clusterMsg *hdr = (clusterMsg*) msg;
uint16_t type = ntohs(hdr->type);
if (type < CLUSTERMSG_TYPE_COUNT)
server.cluster->stats_bus_messages_sent[type]++;
}
/* Send a message to all the nodes that are part of the cluster having
@@ -2199,11 +2278,28 @@ void clusterBuildMessageHdr(clusterMsg *hdr, int type) {
hdr->type = htons(type);
memcpy(hdr->sender,myself->name,CLUSTER_NAMELEN);
/* If cluster-announce-ip option is enabled, force the receivers of our
* packets to use the specified address for this node. Otherwise if the
* first byte is zero, they'll do auto discovery. */
memset(hdr->myip,0,NET_IP_STR_LEN);
if (server.cluster_announce_ip) {
strncpy(hdr->myip,server.cluster_announce_ip,NET_IP_STR_LEN);
hdr->myip[NET_IP_STR_LEN-1] = '\0';
}
/* Handle cluster-announce-port as well. */
int announced_port = server.cluster_announce_port ?
server.cluster_announce_port : server.port;
int announced_cport = server.cluster_announce_bus_port ?
server.cluster_announce_bus_port :
(server.port + CLUSTER_PORT_INCR);
memcpy(hdr->myslots,master->slots,sizeof(hdr->myslots));
memset(hdr->slaveof,0,CLUSTER_NAMELEN);
if (myself->slaveof != NULL)
memcpy(hdr->slaveof,myself->slaveof->name, CLUSTER_NAMELEN);
hdr->port = htons(server.port);
hdr->port = htons(announced_port);
hdr->cport = htons(announced_cport);
hdr->flags = htons(myself->flags);
hdr->state = server.cluster->state;
@@ -2235,6 +2331,33 @@ void clusterBuildMessageHdr(clusterMsg *hdr, int type) {
/* For PING, PONG, and MEET, fixing the totlen field is up to the caller. */
}
/* Return non zero if the node is already present in the gossip section of the
* message pointed by 'hdr' and having 'count' gossip entries. Otherwise
* zero is returned. Helper for clusterSendPing(). */
int clusterNodeIsInGossipSection(clusterMsg *hdr, int count, clusterNode *n) {
int j;
for (j = 0; j < count; j++) {
if (memcmp(hdr->data.ping.gossip[j].nodename,n->name,
CLUSTER_NAMELEN) == 0) break;
}
return j != count;
}
/* Set the i-th entry of the gossip section in the message pointed by 'hdr'
* to the info of the specified node 'n'. */
void clusterSetGossipEntry(clusterMsg *hdr, int i, clusterNode *n) {
clusterMsgDataGossip *gossip;
gossip = &(hdr->data.ping.gossip[i]);
memcpy(gossip->nodename,n->name,CLUSTER_NAMELEN);
gossip->ping_sent = htonl(n->ping_sent/1000);
gossip->pong_received = htonl(n->pong_received/1000);
memcpy(gossip->ip,n->ip,sizeof(n->ip));
gossip->port = htons(n->port);
gossip->cport = htons(n->cport);
gossip->flags = htons(n->flags);
gossip->notused1 = 0;
}
/* Send a PING or PONG packet to the specified node, making sure to add enough
* gossip informations. */
void clusterSendPing(clusterLink *link, int type) {
@@ -2279,11 +2402,15 @@ void clusterSendPing(clusterLink *link, int type) {
if (wanted < 3) wanted = 3;
if (wanted > freshnodes) wanted = freshnodes;
/* Include all the nodes in PFAIL state, so that failure reports are
* faster to propagate to go from PFAIL to FAIL state. */
int pfail_wanted = server.cluster->stats_pfail_nodes;
/* Compute the maxium totlen to allocate our buffer. We'll fix the totlen
* later according to the number of gossip sections we really were able
* to put inside the packet. */
totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData);
totlen += (sizeof(clusterMsgDataGossip)*wanted);
totlen += (sizeof(clusterMsgDataGossip)*(wanted+pfail_wanted));
/* Note: clusterBuildMessageHdr() expects the buffer to be always at least
* sizeof(clusterMsg) or more. */
if (totlen < (int)sizeof(clusterMsg)) totlen = sizeof(clusterMsg);
@@ -2300,17 +2427,13 @@ void clusterSendPing(clusterLink *link, int type) {
while(freshnodes > 0 && gossipcount < wanted && maxiterations--) {
dictEntry *de = dictGetRandomKey(server.cluster->nodes);
clusterNode *this = dictGetVal(de);
clusterMsgDataGossip *gossip;
int j;
/* Don't include this node: the whole packet header is about us
* already, so we just gossip about other nodes. */
if (this == myself) continue;
/* Give a bias to FAIL/PFAIL nodes. */
if (maxiterations > wanted*2 &&
!(this->flags & (CLUSTER_NODE_PFAIL|CLUSTER_NODE_FAIL)))
continue;
/* PFAIL nodes will be added later. */
if (this->flags & CLUSTER_NODE_PFAIL) continue;
/* In the gossip section don't include:
* 1) Nodes in HANDSHAKE state.
@@ -2324,27 +2447,37 @@ void clusterSendPing(clusterLink *link, int type) {
continue;
}
/* Check if we already added this node */
for (j = 0; j < gossipcount; j++) {
if (memcmp(hdr->data.ping.gossip[j].nodename,this->name,
CLUSTER_NAMELEN) == 0) break;
}
if (j != gossipcount) continue;
/* Do not add a node we already have. */
if (clusterNodeIsInGossipSection(hdr,gossipcount,this)) continue;
/* Add it */
clusterSetGossipEntry(hdr,gossipcount,this);
freshnodes--;
gossip = &(hdr->data.ping.gossip[gossipcount]);
memcpy(gossip->nodename,this->name,CLUSTER_NAMELEN);
gossip->ping_sent = htonl((u_long)this->ping_sent); WIN_PORT_FIX /* cast (u_long) */
gossip->pong_received = htonl((u_long)this->pong_received); WIN_PORT_FIX /* cast (u_long) */
memcpy(gossip->ip,this->ip,sizeof(this->ip));
gossip->port = htons(this->port);
gossip->flags = htons(this->flags);
gossip->notused1 = 0;
gossip->notused2 = 0;
gossipcount++;
}
/* If there are PFAIL nodes, add them at the end. */
if (pfail_wanted) {
dictIterator *di;
dictEntry *de;
di = dictGetSafeIterator(server.cluster->nodes);
while((de = dictNext(di)) != NULL && pfail_wanted > 0) {
clusterNode *node = dictGetVal(de);
if (node->flags & CLUSTER_NODE_HANDSHAKE) continue;
if (node->flags & CLUSTER_NODE_NOADDR) continue;
if (!(node->flags & CLUSTER_NODE_PFAIL)) continue;
clusterSetGossipEntry(hdr,gossipcount,node);
freshnodes--;
gossipcount++;
/* We take the count of the slots we allocated, since the
* PFAIL stats may not match perfectly with the current number
* of PFAIL nodes. */
pfail_wanted--;
}
dictReleaseIterator(di);
}
/* Ready to send... fix the totlen fiend and queue the message in the
* output buffer. */
totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData);
@@ -2692,9 +2825,9 @@ void clusterLogCantFailover(int reason) {
server.cluster->cant_failover_reason = reason;
/* We also don't emit any log if the master failed no PORT_LONG ago, the
/* We also don't emit any log if the master failed no long ago, the
* goal of this function is to log slaves in a stalled condition for
* a PORT_LONG time. */
* a long time. */
if (myself->slaveof &&
nodeFailed(myself->slaveof) &&
(mstime() - myself->slaveof->fail_time) < nolog_fail_time) return;
@@ -2780,7 +2913,7 @@ void clusterHandleSlaveFailover(void) {
* and wait for replies), and the failover retry time (the time to wait
* before trying to get voted again).
*
* Timeout is MIN(NODE_TIMEOUT*2,2000) milliseconds.
* Timeout is MAX(NODE_TIMEOUT*2,2000) milliseconds.
* Retry is two times the Timeout.
*/
auth_timeout = server.cluster_node_timeout*2;
@@ -3138,6 +3271,31 @@ void clusterCron(void) {
iteration++; /* Number of times this function was called so far. */
/* We want to take myself->ip in sync with the cluster-announce-ip option.
* The option can be set at runtime via CONFIG SET, so we periodically check
* if the option changed to reflect this into myself->ip. */
{
static char *prev_ip = NULL;
char *curr_ip = server.cluster_announce_ip;
int changed = 0;
if (prev_ip == NULL && curr_ip != NULL) changed = 1;
if (prev_ip != NULL && curr_ip == NULL) changed = 1;
if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1;
if (changed) {
prev_ip = curr_ip;
if (prev_ip) prev_ip = zstrdup(prev_ip);
if (curr_ip) {
strncpy(myself->ip,server.cluster_announce_ip,NET_IP_STR_LEN);
myself->ip[NET_IP_STR_LEN-1] = '\0';
} else {
myself->ip[0] = '\0'; /* Force autodetection. */
}
}
}
/* The handshake timeout is the time after which a handshake node that was
* not turned into a normal node is removed from the nodes. Usually it is
* just the NODE_TIMEOUT value, but when NODE_TIMEOUT is too small we use
@@ -3145,13 +3303,21 @@ void clusterCron(void) {
handshake_timeout = server.cluster_node_timeout;
if (handshake_timeout < 1000) handshake_timeout = 1000;
/* Check if we have disconnected nodes and re-establish the connection. */
/* Check if we have disconnected nodes and re-establish the connection.
* Also update a few stats while we are here, that can be used to make
* better decisions in other part of the code. */
di = dictGetSafeIterator(server.cluster->nodes);
server.cluster->stats_pfail_nodes = 0;
while((de = dictNext(di)) != NULL) {
clusterNode *node = dictGetVal(de);
/* Not interested in reconnecting the link with myself or nodes
* for which we have no address. */
if (node->flags & (CLUSTER_NODE_MYSELF|CLUSTER_NODE_NOADDR)) continue;
if (node->flags & CLUSTER_NODE_PFAIL)
server.cluster->stats_pfail_nodes++;
/* A Node in HANDSHAKE state has a limited lifespan equal to the
* configured node timeout. */
if (nodeInHandshake(node) && now - node->ctime > handshake_timeout) {
@@ -3165,7 +3331,7 @@ void clusterCron(void) {
clusterLink *link;
fd = anetTcpNonBlockBindConnect(server.neterr, node->ip,
node->port+CLUSTER_PORT_INCR, NET_FIRST_BIND_ADDR);
node->cport, NET_FIRST_BIND_ADDR);
if (fd == -1) {
/* We got a synchronous error from connect before
* clusterSendPing() had a chance to be called.
@@ -3175,8 +3341,7 @@ void clusterCron(void) {
if (node->ping_sent == 0) node->ping_sent = mstime();
serverLog(LL_DEBUG, "Unable to connect to "
"Cluster Node [%s]:%d -> %s", node->ip,
node->port+CLUSTER_PORT_INCR,
server.neterr);
node->cport, server.neterr);
continue;
}
link = createClusterLink(node);
@@ -3207,7 +3372,7 @@ void clusterCron(void) {
node->flags &= ~CLUSTER_NODE_MEET;
serverLog(LL_DEBUG,"Connecting with Node %.40s at %s:%d",
node->name, node->ip, node->port+CLUSTER_PORT_INCR);
node->name, node->ip, node->cport);
}
}
dictReleaseIterator(di);
@@ -3505,8 +3670,10 @@ int clusterDelNodeSlots(clusterNode *node) {
int deleted = 0, j;
for (j = 0; j < CLUSTER_SLOTS; j++) {
if (clusterNodeGetSlotBit(node,j)) clusterDelSlot(j);
deleted++;
if (clusterNodeGetSlotBit(node,j)) {
clusterDelSlot(j);
deleted++;
}
}
return deleted;
}
@@ -3740,15 +3907,14 @@ static struct redisNodeFlags redisNodeFlagsTable[] = {
/* Concatenate the comma separated list of node flags to the given SDS
* string 'ci'. */
sds representClusterNodeFlags(sds ci, uint16_t flags) {
if (flags == 0) {
ci = sdscat(ci,"noflags,");
} else {
int i, size = sizeof(redisNodeFlagsTable)/sizeof(struct redisNodeFlags);
for (i = 0; i < size; i++) {
struct redisNodeFlags *nodeflag = redisNodeFlagsTable + i;
if (flags & nodeflag->flag) ci = sdscat(ci, nodeflag->name);
}
size_t orig_len = sdslen(ci);
int i, size = sizeof(redisNodeFlagsTable)/sizeof(struct redisNodeFlags);
for (i = 0; i < size; i++) {
struct redisNodeFlags *nodeflag = redisNodeFlagsTable + i;
if (flags & nodeflag->flag) ci = sdscat(ci, nodeflag->name);
}
/* If no flag was added, add the "noflags" special flag. */
if (sdslen(ci) == orig_len) ci = sdscat(ci,"noflags,");
sdsIncrLen(ci,-1); /* Remove trailing comma. */
return ci;
}
@@ -3762,10 +3928,11 @@ sds clusterGenNodeDescription(clusterNode *node) {
sds ci;
/* Node coordinates */
ci = sdscatprintf(sdsempty(),"%.40s %s:%d ",
ci = sdscatprintf(sdsempty(),"%.40s %s:%d@%d ",
node->name,
node->ip,
node->port);
node->port,
node->cport);
/* Flags */
ci = representClusterNodeFlags(ci, node->flags);
@@ -3856,6 +4023,21 @@ sds clusterGenNodesDescription(int filter) {
* CLUSTER command
* -------------------------------------------------------------------------- */
const char *clusterGetMessageTypeString(int type) {
switch(type) {
case CLUSTERMSG_TYPE_PING: return "ping";
case CLUSTERMSG_TYPE_PONG: return "pong";
case CLUSTERMSG_TYPE_MEET: return "meet";
case CLUSTERMSG_TYPE_FAIL: return "fail";
case CLUSTERMSG_TYPE_PUBLISH: return "publish";
case CLUSTERMSG_TYPE_FAILOVER_AUTH_REQUEST: return "auth-req";
case CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK: return "auth-ack";
case CLUSTERMSG_TYPE_UPDATE: return "update";
case CLUSTERMSG_TYPE_MFSTART: return "mfstart";
}
return "unknown";
}
int getSlotOrReply(client *c, robj *o) {
PORT_LONGLONG slot;
@@ -3948,16 +4130,27 @@ void clusterCommand(client *c) {
return;
}
if (!strcasecmp(c->argv[1]->ptr,"meet") && c->argc == 4) {
PORT_LONGLONG port;
if (!strcasecmp(c->argv[1]->ptr,"meet") && (c->argc == 4 || c->argc == 5)) {
/* CLUSTER MEET <ip> <port> [cport] */
PORT_LONGLONG port, cport;
if (getLongLongFromObject(c->argv[3], &port) != C_OK) {
addReplyErrorFormat(c,"Invalid TCP port specified: %s",
addReplyErrorFormat(c,"Invalid TCP base port specified: %s",
(char*)c->argv[3]->ptr);
return;
}
if (clusterStartHandshake(c->argv[2]->ptr,(int)port) == 0 && WIN_PORT_FIX /* cast (int) */
if (c->argc == 5) {
if (getLongLongFromObject(c->argv[4], &cport) != C_OK) {
addReplyErrorFormat(c,"Invalid TCP bus port specified: %s",
(char*)c->argv[4]->ptr);
return;
}
} else {
cport = port + CLUSTER_PORT_INCR;
}
if (clusterStartHandshake(c->argv[2]->ptr,(int)port,(int)cport) == 0 && WIN_PORT_FIX /* cast (int) */
errno == EINVAL)
{
addReplyErrorFormat(c,"Invalid node address specified: %s:%s",
@@ -4072,7 +4265,7 @@ void clusterCommand(client *c) {
}
if ((n = clusterLookupNode(c->argv[4]->ptr)) == NULL) {
addReplyErrorFormat(c,"I don't know about node %s",
(char*)c->argv[3]->ptr);
(char*)c->argv[4]->ptr);
return;
}
server.cluster->importing_slots_from[slot] = n;
@@ -4176,8 +4369,6 @@ void clusterCommand(client *c) {
"cluster_size:%d\r\n"
"cluster_current_epoch:%llu\r\n"
"cluster_my_epoch:%llu\r\n"
"cluster_stats_messages_sent:%lld\r\n"
"cluster_stats_messages_received:%lld\r\n"
, statestr[server.cluster->state],
slots_assigned,
slots_ok,
@@ -4186,10 +4377,36 @@ void clusterCommand(client *c) {
dictSize(server.cluster->nodes),
server.cluster->size,
(PORT_ULONGLONG) server.cluster->currentEpoch,
(PORT_ULONGLONG) myepoch,
server.cluster->stats_bus_messages_sent,
server.cluster->stats_bus_messages_received
(PORT_ULONGLONG) myepoch
);
/* Show stats about messages sent and received. */
PORT_LONGLONG tot_msg_sent = 0;
PORT_LONGLONG tot_msg_received = 0;
for (int i = 0; i < CLUSTERMSG_TYPE_COUNT; i++) {
if (server.cluster->stats_bus_messages_sent[i] == 0) continue;
tot_msg_sent += server.cluster->stats_bus_messages_sent[i];
info = sdscatprintf(info,
"cluster_stats_messages_%s_sent:%lld\r\n",
clusterGetMessageTypeString(i),
server.cluster->stats_bus_messages_sent[i]);
}
info = sdscatprintf(info,
"cluster_stats_messages_sent:%lld\r\n", tot_msg_sent);
for (int i = 0; i < CLUSTERMSG_TYPE_COUNT; i++) {
if (server.cluster->stats_bus_messages_received[i] == 0) continue;
tot_msg_received += server.cluster->stats_bus_messages_received[i];
info = sdscatprintf(info,
"cluster_stats_messages_%s_received:%lld\r\n",
clusterGetMessageTypeString(i),
server.cluster->stats_bus_messages_received[i]);
}
info = sdscatprintf(info,
"cluster_stats_messages_received:%lld\r\n", tot_msg_received);
/* Produce the reply protocol. */
addReplySds(c,sdscatprintf(sdsempty(),"$%Iu\r\n", WIN_PORT_FIX /* %lu -> %Iu */
(PORT_ULONG)sdslen(info)));
addReplySds(c,info);
@@ -4201,7 +4418,7 @@ void clusterCommand(client *c) {
addReply(c,shared.ok);
else
addReplyErrorFormat(c,"error saving the cluster node config: %s",
strerror(errno));
IF_WIN32(wsa_strerror(errno), strerror(errno)));
} else if (!strcasecmp(c->argv[1]->ptr,"keyslot") && c->argc == 3) {
/* CLUSTER KEYSLOT <key> */
sds key = c->argv[2]->ptr;
@@ -4234,10 +4451,18 @@ void clusterCommand(client *c) {
return;
}
/* Avoid allocating more than needed in case of large COUNT argument
* and smaller actual number of keys. */
unsigned int keys_in_slot = countKeysInSlot(slot);
if (maxkeys > keys_in_slot) maxkeys = keys_in_slot;
keys = zmalloc(sizeof(robj*)*maxkeys);
numkeys = getKeysInSlot((unsigned int)slot, keys, (unsigned int)maxkeys); WIN_PORT_FIX /* cast (unsigned int) */
addReplyMultiBulkLen(c,numkeys);
for (j = 0; j < numkeys; j++) addReplyBulk(c,keys[j]);
for (j = 0; j < numkeys; j++) {
addReplyBulk(c,keys[j]);
decrRefCount(keys[j]);
}
zfree(keys);
} else if (!strcasecmp(c->argv[1]->ptr,"forget") && c->argc == 3) {
/* CLUSTER FORGET <NODE ID> */
@@ -4584,7 +4809,7 @@ void restoreCommand(client *c) {
/* Create the key and set the TTL if any */
dbAdd(c->db,c->argv[1],obj);
if (ttl) setExpire(c->db,c->argv[1],mstime()+ttl);
if (ttl) setExpire(c,c->db,c->argv[1],mstime()+ttl);
signalModifiedKey(c->db,c->argv[1]);
addReply(c,shared.ok);
server.dirty++;
@@ -4724,6 +4949,7 @@ void migrateCommand(client *c) {
rio cmd, payload;
int may_retry = 1;
int write_error = 0;
int argv_rewritten = 0;
/* To support the KEYS option we need the following additional state. */
int first_key = 3; /* Argument index of the first key. */
@@ -4732,7 +4958,6 @@ void migrateCommand(client *c) {
/* Initialization */
copy = 0;
replace = 0;
ttl = 0;
/* Parse additional options */
for (j = 6; j < c->argc; j++) {
@@ -4808,7 +5033,9 @@ try_again:
/* Create RESTORE payload and generate the protocol to call the command. */
for (j = 0; j < num_keys; j++) {
expireat = getExpire(c->db,kv[j]);
PORT_LONGLONG ttl = 0;
PORT_LONGLONG expireat = getExpire(c->db,kv[j]);
if (expireat != -1) {
ttl = expireat-mstime();
if (ttl < 1) ttl = 1;
@@ -4928,12 +5155,20 @@ try_again:
goto socket_err; /* A retry is guaranteed because of tested conditions.*/
}
/* On socket errors, close the migration socket now that we still have
* the original host/port in the ARGV. Later the original command may be
* rewritten to DEL and will be too later. */
if (socket_error) migrateCloseSocket(c->argv[1],c->argv[2]);
if (!copy) {
/* Translate MIGRATE as DEL for replication/AOF. */
/* Translate MIGRATE as DEL for replication/AOF. Note that we do
* this only for the keys for which we received an acknowledgement
* from the receiving Redis server, by using the del_idx index. */
if (del_idx > 1) {
newargv[0] = createStringObject("DEL",3);
/* Note that the following call takes ownership of newargv. */
replaceClientCommandVector(c,del_idx,newargv);
argv_rewritten = 1;
} else {
/* No key transfer acknowledged, no need to rewrite as DEL. */
zfree(newargv);
@@ -4942,8 +5177,8 @@ try_again:
}
/* If we are here and a socket error happened, we don't want to retry.
* Just signal the problem to the client, but only do it if we don't
* already queued a different error reported by the destination server. */
* Just signal the problem to the client, but only do it if we did not
* already queue a different error reported by the destination server. */
if (!error_from_target && socket_error) {
may_retry = 0;
goto socket_err;
@@ -4951,7 +5186,11 @@ try_again:
if (!error_from_target) {
/* Success! Update the last_dbid in migrateCachedSocket, so that we can
* avoid SELECT the next time if the target DB is the same. Reply +OK. */
* avoid SELECT the next time if the target DB is the same. Reply +OK.
*
* Note: If we reached this point, even if socket_error is true
* still the SELECT command succeeded (otherwise the code jumps to
* socket_err label. */
cs->last_dbid = dbid;
addReply(c,shared.ok);
} else {
@@ -4961,7 +5200,6 @@ try_again:
sdsfree(cmd.io.buffer.ptr);
zfree(ov); zfree(kv); zfree(newargv);
if (socket_error) migrateCloseSocket(c->argv[1],c->argv[2]);
return;
/* On socket errors we try to close the cached socket and try again.
@@ -4974,7 +5212,12 @@ socket_err:
/* Cleanup we want to perform in both the retry and no retry case.
* Note: Closing the migrate socket will also force SELECT next time. */
sdsfree(cmd.io.buffer.ptr);
migrateCloseSocket(c->argv[1],c->argv[2]);
/* If the command was rewritten as DEL and there was a socket error,
* we already closed the socket earlier. While migrateCloseSocket()
* is idempotent, the host/port arguments are now gone, so don't do it
* again. */
if (!argv_rewritten) migrateCloseSocket(c->argv[1],c->argv[2]);
zfree(newargv);
newargv = NULL; /* This will get reallocated on retry. */
@@ -5012,7 +5255,7 @@ void askingCommand(client *c) {
}
/* The READONLY command is used by clients to enter the read-only mode.
* In this mode slaves will not redirect clients as PORT_LONG as clients access
* In this mode slaves will not redirect clients as long as clients access
* with read-only commands to keys that are served by the slave's master. */
void readonlyCommand(client *c) {
if (server.cluster_enabled == 0) {
@@ -5277,8 +5520,9 @@ int clusterRedirectBlockedClientIfNeeded(client *c) {
return 1;
}
/* All keys must belong to the same slot, so check first key only. */
di = dictGetIterator(c->bpop.keys);
while((de = dictNext(di)) != NULL) {
if ((de = dictNext(di)) != NULL) {
robj *key = dictGetKey(de);
int slot = keyHashSlot((char*)key->ptr, (int)sdslen(key->ptr)); WIN_PORT_FIX /* cast (int) */
clusterNode *node = server.cluster->slots[slot];
@@ -5296,6 +5540,7 @@ int clusterRedirectBlockedClientIfNeeded(client *c) {
clusterRedirectClient(c,node,slot,
CLUSTER_REDIR_MOVED);
}
dictReleaseIterator(di);
return 1;
}
}
src/cluster.h
+42 -33
View File
@@ -73,6 +73,29 @@ typedef struct clusterLink {
#define CLUSTER_CANT_FAILOVER_WAITING_VOTES 4
#define CLUSTER_CANT_FAILOVER_RELOG_PERIOD (60*5) /* seconds. */
/* clusterState todo_before_sleep flags. */
#define CLUSTER_TODO_HANDLE_FAILOVER (1<<0)
#define CLUSTER_TODO_UPDATE_STATE (1<<1)
#define CLUSTER_TODO_SAVE_CONFIG (1<<2)
#define CLUSTER_TODO_FSYNC_CONFIG (1<<3)
/* Message types.
*
* Note that the PING, PONG and MEET messages are actually the same exact
* kind of packet. PONG is the reply to ping, in the exact format as a PING,
* while MEET is a special PING that forces the receiver to add the sender
* as a node (if it is not already in the list). */
#define CLUSTERMSG_TYPE_PING 0 /* Ping */
#define CLUSTERMSG_TYPE_PONG 1 /* Pong (reply to Ping) */
#define CLUSTERMSG_TYPE_MEET 2 /* Meet "let's join" message */
#define CLUSTERMSG_TYPE_FAIL 3 /* Mark node xxx as failing */
#define CLUSTERMSG_TYPE_PUBLISH 4 /* Pub/Sub Publish propagation */
#define CLUSTERMSG_TYPE_FAILOVER_AUTH_REQUEST 5 /* May I failover? */
#define CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK 6 /* Yes, you have my vote */
#define CLUSTERMSG_TYPE_UPDATE 7 /* Another node slots configuration */
#define CLUSTERMSG_TYPE_MFSTART 8 /* Pause clients for manual failover */
#define CLUSTERMSG_TYPE_COUNT 9 /* Total number of message types. */
/* This structure represent elements of node->fail_reports. */
typedef struct clusterNodeFailReport {
struct clusterNode *node; /* Node reporting the failure condition. */
@@ -100,7 +123,8 @@ typedef struct clusterNode {
mstime_t orphaned_time; /* Starting time of orphaned master condition */
PORT_LONGLONG repl_offset; /* Last known repl offset for this node. */
char ip[NET_IP_STR_LEN]; /* Latest known IP address of this node */
int port; /* Latest known port of this node */
int port; /* Latest known clients port of this node */
int cport; /* Latest known cluster port of this node. */
clusterLink *link; /* TCP/IP link with this node */
list *fail_reports; /* List of nodes signaling this as failing */
} clusterNode;
@@ -115,7 +139,8 @@ typedef struct clusterState {
clusterNode *migrating_slots_to[CLUSTER_SLOTS];
clusterNode *importing_slots_from[CLUSTER_SLOTS];
clusterNode *slots[CLUSTER_SLOTS];
zskiplist *slots_to_keys;
uint64_t slots_keys_count[CLUSTER_SLOTS];
rax *slots_to_keys;
/* The following fields are used to take the slave state on elections. */
mstime_t failover_auth_time; /* Time of previous or next election. */
int failover_auth_count; /* Number of votes received so far. */
@@ -137,32 +162,15 @@ typedef struct clusterState {
/* The followign fields are used by masters to take state on elections. */
uint64_t lastVoteEpoch; /* Epoch of the last vote granted. */
int todo_before_sleep; /* Things to do in clusterBeforeSleep(). */
PORT_LONGLONG stats_bus_messages_sent; /* Num of msg sent via cluster bus. */
PORT_LONGLONG stats_bus_messages_received; /* Num of msg rcvd via cluster bus.*/
/* Messages received and sent by type. */
PORT_LONGLONG stats_bus_messages_sent[CLUSTERMSG_TYPE_COUNT];
PORT_LONGLONG stats_bus_messages_received[CLUSTERMSG_TYPE_COUNT];
PORT_LONGLONG stats_pfail_nodes; /* Number of nodes in PFAIL status,
excluding nodes without address. */
} clusterState;
/* clusterState todo_before_sleep flags. */
#define CLUSTER_TODO_HANDLE_FAILOVER (1<<0)
#define CLUSTER_TODO_UPDATE_STATE (1<<1)
#define CLUSTER_TODO_SAVE_CONFIG (1<<2)
#define CLUSTER_TODO_FSYNC_CONFIG (1<<3)
/* Redis cluster messages header */
/* Note that the PING, PONG and MEET messages are actually the same exact
* kind of packet. PONG is the reply to ping, in the exact format as a PING,
* while MEET is a special PING that forces the receiver to add the sender
* as a node (if it is not already in the list). */
#define CLUSTERMSG_TYPE_PING 0 /* Ping */
#define CLUSTERMSG_TYPE_PONG 1 /* Pong (reply to Ping) */
#define CLUSTERMSG_TYPE_MEET 2 /* Meet "let's join" message */
#define CLUSTERMSG_TYPE_FAIL 3 /* Mark node xxx as failing */
#define CLUSTERMSG_TYPE_PUBLISH 4 /* Pub/Sub Publish propagation */
#define CLUSTERMSG_TYPE_FAILOVER_AUTH_REQUEST 5 /* May I failover? */
#define CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK 6 /* Yes, you have my vote */
#define CLUSTERMSG_TYPE_UPDATE 7 /* Another node slots configuration */
#define CLUSTERMSG_TYPE_MFSTART 8 /* Pause clients for manual failover */
/* Initially we don't know our "name", but we'll find it once we connect
* to the first node, using the getsockname() function. Then we'll use this
* address for all the next messages. */
@@ -171,10 +179,10 @@ typedef struct {
uint32_t ping_sent;
uint32_t pong_received;
char ip[NET_IP_STR_LEN]; /* IP address last time it was seen */
uint16_t port; /* port last time it was seen */
uint16_t port; /* base port last time it was seen */
uint16_t cport; /* cluster port last time it was seen */
uint16_t flags; /* node->flags copy */
uint16_t notused1; /* Some room for future improvements. */
uint32_t notused2;
uint32_t notused1;
} clusterMsgDataGossip;
typedef struct {
@@ -219,13 +227,13 @@ union clusterMsgData {
} update;
};
#define CLUSTER_PROTO_VER 0 /* Cluster bus protocol version. */
#define CLUSTER_PROTO_VER 1 /* Cluster bus protocol version. */
typedef struct {
char sig[4]; /* Siganture "RCmb" (Redis Cluster message bus). */
uint32_t totlen; /* Total length of this message */
uint16_t ver; /* Protocol version, currently set to 0. */
uint16_t notused0; /* 2 bytes not used. */
uint16_t ver; /* Protocol version, currently set to 1. */
uint16_t port; /* TCP base port number. */
uint16_t type; /* Message type */
uint16_t count; /* Only used for some kind of messages. */
uint64_t currentEpoch; /* The epoch accordingly to the sending node. */
@@ -237,9 +245,10 @@ typedef struct {
char sender[CLUSTER_NAMELEN]; /* Name of the sender node */
unsigned char myslots[CLUSTER_SLOTS/8];
char slaveof[CLUSTER_NAMELEN];
char notused1[32]; /* 32 bytes reserved for future usage. */
uint16_t port; /* Sender TCP base port */
uint16_t flags; /* Sender node flags */
char myip[NET_IP_STR_LEN]; /* Sender IP, if not all zeroed. */
char notused1[34]; /* 34 bytes reserved for future usage. */
uint16_t cport; /* Sender TCP cluster bus port */
uint16_t flags; /* Sender node flags */
unsigned char state; /* Cluster state from the POV of the sender */
unsigned char mflags[3]; /* Message flags: CLUSTERMSG_FLAG[012]_... */
union clusterMsgData data;
src/config.c
+245 -38
View File
@@ -31,6 +31,7 @@
#ifdef _WIN32
#include "Win32_Interop/win32_types.h"
#include "Win32_Interop/Win32_EventLog.h"
#include "Win32_Interop/Win32_Error.h"
#include <direct.h>
#endif
@@ -51,9 +52,11 @@ typedef struct configEnum {
configEnum maxmemory_policy_enum[] = {
{"volatile-lru", MAXMEMORY_VOLATILE_LRU},
{"volatile-lfu", MAXMEMORY_VOLATILE_LFU},
{"volatile-random",MAXMEMORY_VOLATILE_RANDOM},
{"volatile-ttl",MAXMEMORY_VOLATILE_TTL},
{"allkeys-lru",MAXMEMORY_ALLKEYS_LRU},
{"allkeys-lfu",MAXMEMORY_ALLKEYS_LFU},
{"allkeys-random",MAXMEMORY_ALLKEYS_RANDOM},
{"noeviction",MAXMEMORY_NO_EVICTION},
{NULL, 0}
@@ -161,6 +164,20 @@ void resetServerSaveParams(void) {
server.saveparamslen = 0;
}
void queueLoadModule(sds path, sds *argv, int argc) {
int i;
struct moduleLoadQueueEntry *loadmod;
loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry));
loadmod->argv = zmalloc(sizeof(robj*)*argc);
loadmod->path = sdsnew(path);
loadmod->argc = argc;
for (i = 0; i < argc; i++) {
loadmod->argv[i] = createRawStringObject(argv[i],sdslen(argv[i]));
}
listAddNodeTail(server.loadmodule_queue,loadmod);
}
void loadServerConfigFromString(char *config) {
char *err = NULL;
int linenum = 0, totlines, i;
@@ -249,7 +266,7 @@ void loadServerConfigFromString(char *config) {
} else if (!strcasecmp(argv[0],"dir") && argc == 2) {
if (chdir(argv[1]) == -1) {
serverLog(LL_WARNING,"Can't chdir to '%s': %s",
argv[1], strerror(errno));
argv[1], IF_WIN32(wsa_strerror(errno), strerror(errno)));
exit(1);
}
} else if (!strcasecmp(argv[0],"loglevel") && argc == 2) {
@@ -290,7 +307,7 @@ void loadServerConfigFromString(char *config) {
logfp = fopen(server.logfile,"a");
if (logfp == NULL) {
err = sdscatprintf(sdsempty(),
"Can't open the log file: %s", strerror(errno));
"Can't open the log file: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
goto loaderr;
#ifdef _WIN32
} else {
@@ -300,6 +317,10 @@ void loadServerConfigFromString(char *config) {
fclose(logfp);
}
} else if (!strcasecmp(argv[0],"always-show-logo") && argc == 2) {
if ((server.always_show_logo = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"syslog-enabled") && argc == 2) {
if ((server.syslog_enabled = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
@@ -351,6 +372,18 @@ void loadServerConfigFromString(char *config) {
err = "maxmemory-samples must be 1 or greater";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"lfu-log-factor") && argc == 2) {
server.lfu_log_factor = atoi(argv[1]);
if (server.maxmemory_samples < 0) {
err = "lfu-log-factor must be 0 or greater";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"lfu-decay-time") && argc == 2) {
server.lfu_decay_time = atoi(argv[1]);
if (server.maxmemory_samples < 1) {
err = "lfu-decay-time must be 0 or greater";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"slaveof") && argc == 3) {
slaveof_linenum = linenum;
server.masterhost = sdsnew(argv[1]);
@@ -418,6 +451,26 @@ void loadServerConfigFromString(char *config) {
if ((server.activerehashing = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"lazyfree-lazy-eviction") && argc == 2) {
if ((server.lazyfree_lazy_eviction = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"lazyfree-lazy-expire") && argc == 2) {
if ((server.lazyfree_lazy_expire = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"lazyfree-lazy-server-del") && argc == 2){
if ((server.lazyfree_lazy_server_del = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"slave-lazy-flush") && argc == 2) {
if ((server.repl_slave_lazy_flush = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"activedefrag") && argc == 2) {
if ((server.active_defrag_enabled = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"daemonize") && argc == 2) {
if ((server.daemonize = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
@@ -474,6 +527,10 @@ void loadServerConfigFromString(char *config) {
if ((server.aof_load_truncated = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"aof-use-rdb-preamble") && argc == 2) {
if ((server.aof_use_rdb_preamble = yesnotoi(argv[1])) == -1) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"requirepass") && argc == 2) {
if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) {
err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN";
@@ -490,6 +547,36 @@ void loadServerConfigFromString(char *config) {
}
zfree(server.rdb_filename);
server.rdb_filename = zstrdup(argv[1]);
} else if (!strcasecmp(argv[0],"active-defrag-threshold-lower") && argc == 2) {
server.active_defrag_threshold_lower = atoi(argv[1]);
if (server.active_defrag_threshold_lower < 0) {
err = "active-defrag-threshold-lower must be 0 or greater";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"active-defrag-threshold-upper") && argc == 2) {
server.active_defrag_threshold_upper = atoi(argv[1]);
if (server.active_defrag_threshold_upper < 0) {
err = "active-defrag-threshold-upper must be 0 or greater";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"active-defrag-ignore-bytes") && argc == 2) {
server.active_defrag_ignore_bytes = memtoll(argv[1], NULL);
if (server.active_defrag_ignore_bytes <= 0) {
err = "active-defrag-ignore-bytes must above 0";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"active-defrag-cycle-min") && argc == 2) {
server.active_defrag_cycle_min = atoi(argv[1]);
if (server.active_defrag_cycle_min < 1 || server.active_defrag_cycle_min > 99) {
err = "active-defrag-cycle-min must be between 1 and 99";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"active-defrag-cycle-max") && argc == 2) {
server.active_defrag_cycle_max = atoi(argv[1]);
if (server.active_defrag_cycle_max < 1 || server.active_defrag_cycle_max > 99) {
err = "active-defrag-cycle-max must be between 1 and 99";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"hash-max-ziplist-entries") && argc == 2) {
server.hash_max_ziplist_entries = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"hash-max-ziplist-value") && argc == 2) {
@@ -541,6 +628,25 @@ void loadServerConfigFromString(char *config) {
} else if (!strcasecmp(argv[0],"cluster-config-file") && argc == 2) {
zfree(server.cluster_configfile);
server.cluster_configfile = zstrdup(argv[1]);
} else if (!strcasecmp(argv[0],"cluster-announce-ip") && argc == 2) {
zfree(server.cluster_announce_ip);
server.cluster_announce_ip = zstrdup(argv[1]);
} else if (!strcasecmp(argv[0],"cluster-announce-port") && argc == 2) {
server.cluster_announce_port = atoi(argv[1]);
if (server.cluster_announce_port < 0 ||
server.cluster_announce_port > 65535)
{
err = "Invalid port"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"cluster-announce-bus-port") &&
argc == 2)
{
server.cluster_announce_bus_port = atoi(argv[1]);
if (server.cluster_announce_bus_port < 0 ||
server.cluster_announce_bus_port > 65535)
{
err = "Invalid port"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"cluster-require-full-coverage") &&
argc == 2)
{
@@ -549,7 +655,7 @@ void loadServerConfigFromString(char *config) {
err = "argument must be 'yes' or 'no'"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"cluster-node-timeout") && argc == 2) {
server.cluster_node_timeout = strtol(argv[1],NULL,10);
server.cluster_node_timeout = strtoll(argv[1],NULL,10);
if (server.cluster_node_timeout <= 0) {
err = "cluster node timeout must be 1 or greater"; goto loaderr;
}
@@ -570,15 +676,15 @@ void loadServerConfigFromString(char *config) {
goto loaderr;
}
} else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) {
server.lua_time_limit = strtol(argv[1],NULL,10);
server.lua_time_limit = strtoll(argv[1],NULL,10);
} else if (!strcasecmp(argv[0],"slowlog-log-slower-than") &&
argc == 2)
{
server.slowlog_log_slower_than = strtol(argv[1],NULL,10);
server.slowlog_log_slower_than = strtoll(argv[1],NULL,10);
} else if (!strcasecmp(argv[0],"latency-monitor-threshold") &&
argc == 2)
{
server.latency_monitor_threshold = strtol(argv[1],NULL,10);
server.latency_monitor_threshold = strtoll(argv[1],NULL,10);
if (server.latency_monitor_threshold < 0) {
err = "The latency threshold can't be negative";
goto loaderr;
@@ -592,8 +698,9 @@ void loadServerConfigFromString(char *config) {
PORT_ULONGLONG hard, soft;
int soft_seconds;
if (class == -1) {
err = "Unrecognized client limit class";
if (class == -1 || class == CLIENT_TYPE_MASTER) {
err = "Unrecognized client limit class: the user specified "
"an invalid one, or 'master' which has no buffer limits.";
goto loaderr;
}
hard = memtoll(argv[2],NULL);
@@ -613,6 +720,16 @@ void loadServerConfigFromString(char *config) {
}
} else if (!strcasecmp(argv[0],"slave-priority") && argc == 2) {
server.slave_priority = atoi(argv[1]);
} else if (!strcasecmp(argv[0],"slave-announce-ip") && argc == 2) {
zfree(server.slave_announce_ip);
server.slave_announce_ip = zstrdup(argv[1]);
} else if (!strcasecmp(argv[0],"slave-announce-port") && argc == 2) {
server.slave_announce_port = atoi(argv[1]);
if (server.slave_announce_port < 0 ||
server.slave_announce_port > 65535)
{
err = "Invalid port"; goto loaderr;
}
} else if (!strcasecmp(argv[0],"min-slaves-to-write") && argc == 2) {
server.repl_min_slaves_to_write = atoi(argv[1]);
if (server.repl_min_slaves_to_write < 0) {
@@ -640,6 +757,8 @@ void loadServerConfigFromString(char *config) {
"Allowed values: 'upstart', 'systemd', 'auto', or 'no'";
goto loaderr;
}
} else if (!strcasecmp(argv[0],"loadmodule") && argc >= 2) {
queueLoadModule(argv[1],&argv[2],argc-2);
} else if (!strcasecmp(argv[0],"sentinel")) {
/* argc == 1 is handled by main() as we need to enter the sentinel
* mode ASAP. */
@@ -817,6 +936,9 @@ void configSetCommand(client *c) {
} config_set_special_field("masterauth") {
zfree(server.masterauth);
server.masterauth = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL;
} config_set_special_field("cluster-announce-ip") {
zfree(server.cluster_announce_ip);
server.cluster_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL;
} config_set_special_field("maxclients") {
int orig_value = server.maxclients;
@@ -871,7 +993,7 @@ void configSetCommand(client *c) {
char *eptr;
PORT_LONG val;
val = strtol(v[j], &eptr, 10);
val = strtoll(v[j], &eptr, 10);
if (eptr[0] != '\0' ||
((j & 1) == 0 && val < 1) ||
((j & 1) == 1 && val < 0)) {
@@ -885,14 +1007,14 @@ void configSetCommand(client *c) {
time_t seconds;
int changes;
seconds = strtol(v[j],NULL,10);
changes = strtol(v[j+1],NULL,10);
seconds = strtoll(v[j],NULL,10);
changes = strtoll(v[j+1],NULL,10);
appendServerSaveParams(seconds, changes);
}
sdsfreesplitres(v,vlen);
} config_set_special_field("dir") {
if (chdir((char*)o->ptr) == -1) {
addReplyErrorFormat(c,"Changing directory: %s", strerror(errno));
addReplyErrorFormat(c,"Changing directory: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
return;
}
} config_set_special_field("client-output-buffer-limit") {
@@ -912,7 +1034,8 @@ void configSetCommand(client *c) {
PORT_LONG val;
if ((j % 4) == 0) {
if (getClientTypeByName(v[j]) == -1) {
int class = getClientTypeByName(v[j]);
if (class == -1 || class == CLIENT_TYPE_MASTER) {
sdsfreesplitres(v,vlen);
goto badfmt;
}
@@ -931,9 +1054,9 @@ void configSetCommand(client *c) {
int soft_seconds;
class = getClientTypeByName(v[j]);
hard = strtol(v[j+1],NULL,10);
soft = strtol(v[j+2],NULL,10);
soft_seconds = strtol(v[j+3],NULL,10);
hard = strtoll(v[j+1],NULL,10);
soft = strtoll(v[j+2],NULL,10);
soft_seconds = strtoll(v[j+3],NULL,10);
server.client_obuf_limits[class].hard_limit_bytes = hard;
server.client_obuf_limits[class].soft_limit_bytes = soft;
@@ -945,6 +1068,9 @@ void configSetCommand(client *c) {
if (flags == -1) goto badfmt;
server.notify_keyspace_events = flags;
} config_set_special_field("slave-announce-ip") {
zfree(server.slave_announce_ip);
server.slave_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL;
/* Boolean fields.
* config_set_bool_field(name,var). */
@@ -960,16 +1086,38 @@ void configSetCommand(client *c) {
"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync) {
} config_set_bool_field(
"aof-load-truncated",server.aof_load_truncated) {
} config_set_bool_field(
"aof-use-rdb-preamble",server.aof_use_rdb_preamble) {
} config_set_bool_field(
"slave-serve-stale-data",server.repl_serve_stale_data) {
} config_set_bool_field(
"slave-read-only",server.repl_slave_ro) {
} config_set_bool_field(
"activerehashing",server.activerehashing) {
} config_set_bool_field(
"activedefrag",server.active_defrag_enabled) {
#ifndef HAVE_DEFRAG
if (server.active_defrag_enabled) {
server.active_defrag_enabled = 0;
addReplyError(c,
"Active defragmentation cannot be enabled: it requires a "
"Redis server compiled with a modified Jemalloc like the "
"one shipped by default with the Redis source distribution");
return;
}
#endif
} config_set_bool_field(
"protected-mode",server.protected_mode) {
} config_set_bool_field(
"stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err) {
} config_set_bool_field(
"lazyfree-lazy-eviction",server.lazyfree_lazy_eviction) {
} config_set_bool_field(
"lazyfree-lazy-expire",server.lazyfree_lazy_expire) {
} config_set_bool_field(
"lazyfree-lazy-server-del",server.lazyfree_lazy_server_del) {
} config_set_bool_field(
"slave-lazy-flush",server.repl_slave_lazy_flush) {
} config_set_bool_field(
"no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite) {
@@ -979,12 +1127,24 @@ void configSetCommand(client *c) {
"tcp-keepalive",server.tcpkeepalive,0,LLONG_MAX) {
} config_set_numerical_field(
"maxmemory-samples",server.maxmemory_samples,1,LLONG_MAX) {
} config_set_numerical_field(
"lfu-log-factor",server.lfu_log_factor,0,LLONG_MAX) {
} config_set_numerical_field(
"lfu-decay-time",server.lfu_decay_time,0,LLONG_MAX) {
} config_set_numerical_field(
"timeout",server.maxidletime,0,LONG_MAX) {
} config_set_numerical_field(
"auto-aof-rewrite-percentage",server.aof_rewrite_perc,0,LLONG_MAX){
"active-defrag-threshold-lower",server.active_defrag_threshold_lower,0,1000) {
} config_set_numerical_field(
"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,0,LLONG_MAX) {
"active-defrag-threshold-upper",server.active_defrag_threshold_upper,0,1000) {
} config_set_memory_field(
"active-defrag-ignore-bytes",server.active_defrag_ignore_bytes) {
} config_set_numerical_field(
"active-defrag-cycle-min",server.active_defrag_cycle_min,1,99) {
} config_set_numerical_field(
"active-defrag-cycle-max",server.active_defrag_cycle_max,1,99) {
} config_set_numerical_field(
"auto-aof-rewrite-percentage",server.aof_rewrite_perc,0,LLONG_MAX){
} config_set_numerical_field(
"hash-max-ziplist-entries",server.hash_max_ziplist_entries,0,LLONG_MAX) {
} config_set_numerical_field(
@@ -1021,6 +1181,8 @@ void configSetCommand(client *c) {
"repl-diskless-sync-delay",server.repl_diskless_sync_delay,0,LLONG_MAX) {
} config_set_numerical_field(
"slave-priority",server.slave_priority,0,LLONG_MAX) {
} config_set_numerical_field(
"slave-announce-port",server.slave_announce_port,0,65535) {
} config_set_numerical_field(
"min-slaves-to-write",server.repl_min_slaves_to_write,0,LLONG_MAX) {
refreshGoodSlavesCount();
@@ -1029,6 +1191,10 @@ void configSetCommand(client *c) {
refreshGoodSlavesCount();
} config_set_numerical_field(
"cluster-node-timeout",server.cluster_node_timeout,0,LLONG_MAX) {
} config_set_numerical_field(
"cluster-announce-port",server.cluster_announce_port,0,65535) {
} config_set_numerical_field(
"cluster-announce-bus-port",server.cluster_announce_bus_port,0,65535) {
} config_set_numerical_field(
"cluster-migration-barrier",server.cluster_migration_barrier,0,LLONG_MAX){
} config_set_numerical_field(
@@ -1057,6 +1223,8 @@ void configSetCommand(client *c) {
}
} config_set_memory_field("repl-backlog-size",ll) {
resizeReplicationBacklog(ll);
} config_set_memory_field("auto-aof-rewrite-min-size",ll) {
server.aof_rewrite_min_size = ll;
/* Enumeration fields.
* config_set_enum_field(name,var,enum_var) */
@@ -1092,7 +1260,7 @@ badfmt: /* Bad format errors */
*----------------------------------------------------------------------------*/
#define config_get_string_field(_name,_var) do { \
if (stringmatch(pattern,_name,0)) { \
if (stringmatch(pattern,_name,1)) { \
addReplyBulkCString(c,_name); \
addReplyBulkCString(c,_var ? _var : ""); \
matches++; \
@@ -1100,7 +1268,7 @@ badfmt: /* Bad format errors */
} while(0);
#define config_get_bool_field(_name,_var) do { \
if (stringmatch(pattern,_name,0)) { \
if (stringmatch(pattern,_name,1)) { \
addReplyBulkCString(c,_name); \
addReplyBulkCString(c,_var ? "yes" : "no"); \
matches++; \
@@ -1108,7 +1276,7 @@ badfmt: /* Bad format errors */
} while(0);
#define config_get_numerical_field(_name,_var) do { \
if (stringmatch(pattern,_name,0)) { \
if (stringmatch(pattern,_name,1)) { \
ll2string(buf,sizeof(buf),_var); \
addReplyBulkCString(c,_name); \
addReplyBulkCString(c,buf); \
@@ -1117,7 +1285,7 @@ badfmt: /* Bad format errors */
} while(0);
#define config_get_enum_field(_name,_var,_enumvar) do { \
if (stringmatch(pattern,_name,0)) { \
if (stringmatch(pattern,_name,1)) { \
addReplyBulkCString(c,_name); \
addReplyBulkCString(c,configEnumGetNameOrUnknown(_enumvar,_var)); \
matches++; \
@@ -1136,14 +1304,21 @@ void configGetCommand(client *c) {
config_get_string_field("dbfilename",server.rdb_filename);
config_get_string_field("requirepass",server.requirepass);
config_get_string_field("masterauth",server.masterauth);
config_get_string_field("cluster-announce-ip",server.cluster_announce_ip);
config_get_string_field("unixsocket",server.unixsocket);
config_get_string_field("logfile",server.logfile);
config_get_string_field("pidfile",server.pidfile);
config_get_string_field("slave-announce-ip",server.slave_announce_ip);
/* Numerical values */
config_get_numerical_field("maxmemory",server.maxmemory);
config_get_numerical_field("maxmemory-samples",server.maxmemory_samples);
config_get_numerical_field("timeout",server.maxidletime);
config_get_numerical_field("active-defrag-threshold-lower",server.active_defrag_threshold_lower);
config_get_numerical_field("active-defrag-threshold-upper",server.active_defrag_threshold_upper);
config_get_numerical_field("active-defrag-ignore-bytes",server.active_defrag_ignore_bytes);
config_get_numerical_field("active-defrag-cycle-min",server.active_defrag_cycle_min);
config_get_numerical_field("active-defrag-cycle-max",server.active_defrag_cycle_max);
config_get_numerical_field("auto-aof-rewrite-percentage",
server.aof_rewrite_perc);
config_get_numerical_field("auto-aof-rewrite-min-size",
@@ -1172,6 +1347,8 @@ void configGetCommand(client *c) {
config_get_numerical_field("slowlog-max-len",
server.slowlog_max_len);
config_get_numerical_field("port",server.port);
config_get_numerical_field("cluster-announce-port",server.cluster_announce_port);
config_get_numerical_field("cluster-announce-bus-port",server.cluster_announce_bus_port);
config_get_numerical_field("tcp-backlog",server.tcp_backlog);
config_get_numerical_field("databases",server.dbnum);
config_get_numerical_field("repl-ping-slave-period",server.repl_ping_slave_period);
@@ -1181,6 +1358,7 @@ void configGetCommand(client *c) {
config_get_numerical_field("maxclients",server.maxclients);
config_get_numerical_field("watchdog-period",server.watchdog_period);
config_get_numerical_field("slave-priority",server.slave_priority);
config_get_numerical_field("slave-announce-port",server.slave_announce_port);
config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write);
config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag);
config_get_numerical_field("hz",server.hz);
@@ -1205,6 +1383,7 @@ void configGetCommand(client *c) {
config_get_bool_field("rdbcompression", server.rdb_compression);
config_get_bool_field("rdbchecksum", server.rdb_checksum);
config_get_bool_field("activerehashing", server.activerehashing);
config_get_bool_field("activedefrag", server.active_defrag_enabled);
config_get_bool_field("protected-mode", server.protected_mode);
config_get_bool_field("repl-disable-tcp-nodelay",
server.repl_disable_tcp_nodelay);
@@ -1214,6 +1393,16 @@ void configGetCommand(client *c) {
server.aof_rewrite_incremental_fsync);
config_get_bool_field("aof-load-truncated",
server.aof_load_truncated);
config_get_bool_field("aof-use-rdb-preamble",
server.aof_use_rdb_preamble);
config_get_bool_field("lazyfree-lazy-eviction",
server.lazyfree_lazy_eviction);
config_get_bool_field("lazyfree-lazy-expire",
server.lazyfree_lazy_expire);
config_get_bool_field("lazyfree-lazy-server-del",
server.lazyfree_lazy_server_del);
config_get_bool_field("slave-lazy-flush",
server.repl_slave_lazy_flush);
/* Enum values */
config_get_enum_field("maxmemory-policy",
@@ -1224,17 +1413,19 @@ void configGetCommand(client *c) {
server.supervised_mode,supervised_mode_enum);
config_get_enum_field("appendfsync",
server.aof_fsync,aof_fsync_enum);
POSIX_ONLY(config_get_enum_field("syslog-facility",
server.syslog_facility,syslog_facility_enum);)
#ifndef _WIN32
config_get_enum_field("syslog-facility",
server.syslog_facility,syslog_facility_enum);
#endif
/* Everything we can't handle with macros follows. */
if (stringmatch(pattern,"appendonly",0)) {
if (stringmatch(pattern,"appendonly",1)) {
addReplyBulkCString(c,"appendonly");
addReplyBulkCString(c,server.aof_state == AOF_OFF ? "no" : "yes");
matches++;
}
if (stringmatch(pattern,"dir",0)) {
if (stringmatch(pattern,"dir",1)) {
char buf[1024];
if (getcwd(buf,sizeof(buf)) == NULL)
@@ -1244,7 +1435,7 @@ void configGetCommand(client *c) {
addReplyBulkCString(c,buf);
matches++;
}
if (stringmatch(pattern,"save",0)) {
if (stringmatch(pattern,"save",1)) {
sds buf = sdsempty();
int j;
@@ -1260,12 +1451,12 @@ void configGetCommand(client *c) {
sdsfree(buf);
matches++;
}
if (stringmatch(pattern,"client-output-buffer-limit",0)) {
if (stringmatch(pattern,"client-output-buffer-limit",1)) {
sds buf = sdsempty();
int j;
for (j = 0; j < CLIENT_TYPE_OBUF_COUNT; j++) {
buf = sdscatprintf(buf,"%s %llu %llu %ld",
buf = sdscatprintf(buf,"%s %llu %llu %Id", WIN_PORT_FIX /* %ld -> %Id */
getClientTypeName(j),
server.client_obuf_limits[j].hard_limit_bytes,
server.client_obuf_limits[j].soft_limit_bytes,
@@ -1278,14 +1469,14 @@ void configGetCommand(client *c) {
sdsfree(buf);
matches++;
}
if (stringmatch(pattern,"unixsocketperm",0)) {
if (stringmatch(pattern,"unixsocketperm",1)) {
char buf[32];
snprintf(buf,sizeof(buf),"%o",server.unixsocketperm);
addReplyBulkCString(c,"unixsocketperm");
addReplyBulkCString(c,buf);
matches++;
}
if (stringmatch(pattern,"slaveof",0)) {
if (stringmatch(pattern,"slaveof",1)) {
char buf[256];
addReplyBulkCString(c,"slaveof");
@@ -1297,7 +1488,7 @@ void configGetCommand(client *c) {
addReplyBulkCString(c,buf);
matches++;
}
if (stringmatch(pattern,"notify-keyspace-events",0)) {
if (stringmatch(pattern,"notify-keyspace-events",1)) {
robj *flagsobj = createObject(OBJ_STRING,
keyspaceEventsFlagsToString(server.notify_keyspace_events));
@@ -1306,7 +1497,7 @@ void configGetCommand(client *c) {
decrRefCount(flagsobj);
matches++;
}
if (stringmatch(pattern,"bind",0)) {
if (stringmatch(pattern,"bind",1)) {
sds aux = sdsjoin(server.bindaddr,server.bindaddr_count," ");
addReplyBulkCString(c,"bind");
@@ -1326,7 +1517,7 @@ void configGetCommand(client *c) {
/* We use the following dictionary type to store where a configuration
* option is mentioned in the old configuration file, so it's
* like "maxmemory" -> list of line numbers (first line is zero). */
unsigned int dictSdsCaseHash(const void *key);
uint64_t dictSdsCaseHash(const void *key);
int dictSdsKeyCaseCompare(void *privdata, const void *key1, const void *key2);
void dictSdsDestructor(void *privdata, void *val);
void dictListDestructor(void *privdata, void *val);
@@ -1614,7 +1805,7 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) {
* config line with "save" will be detected as orphaned and deleted,
* resulting into no RDB persistence as expected. */
for (j = 0; j < server.saveparamslen; j++) {
line = sdscatprintf(sdsempty(),"save %ld %d",
line = sdscatprintf(sdsempty(),"save %Id %d", WIN_PORT_FIX /* %ld -> %Id */
(PORT_LONG) server.saveparams[j].seconds, server.saveparams[j].changes);
rewriteConfigRewriteLine(state,"save",line,1);
}
@@ -1684,7 +1875,7 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state
rewriteConfigFormatMemory(soft,sizeof(soft),
server.client_obuf_limits[j].soft_limit_bytes);
line = sdscatprintf(sdsempty(),"%s %s %s %s %ld",
line = sdscatprintf(sdsempty(),"%s %s %s %s %Id", WIN_PORT_FIX /* %ld -> %Id */
option, getClientTypeName(j), hard, soft,
(PORT_LONG) server.client_obuf_limits[j].soft_limit_seconds);
rewriteConfigRewriteLine(state,option,line,force);
@@ -1855,12 +2046,15 @@ int rewriteConfig(char *path) {
rewriteConfigYesNoOption(state,"daemonize",server.daemonize,0);
rewriteConfigStringOption(state,"pidfile",server.pidfile,CONFIG_DEFAULT_PID_FILE);
rewriteConfigNumericalOption(state,"port",server.port,CONFIG_DEFAULT_SERVER_PORT);
rewriteConfigNumericalOption(state,"cluster-announce-port",server.cluster_announce_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT);
rewriteConfigNumericalOption(state,"cluster-announce-bus-port",server.cluster_announce_bus_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT);
rewriteConfigNumericalOption(state,"tcp-backlog",server.tcp_backlog,CONFIG_DEFAULT_TCP_BACKLOG);
rewriteConfigBindOption(state);
rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL);
rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM);
rewriteConfigNumericalOption(state,"timeout",server.maxidletime,CONFIG_DEFAULT_CLIENT_TIMEOUT);
rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,CONFIG_DEFAULT_TCP_KEEPALIVE);
rewriteConfigNumericalOption(state,"slave-announce-port",server.slave_announce_port,CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT);
rewriteConfigEnumOption(state,"loglevel",server.verbosity,loglevel_enum,CONFIG_DEFAULT_VERBOSITY);
rewriteConfigStringOption(state,"logfile",server.logfile,CONFIG_DEFAULT_LOGFILE);
rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,CONFIG_DEFAULT_SYSLOG_ENABLED);
@@ -1876,7 +2070,9 @@ int rewriteConfig(char *path) {
rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,CONFIG_DEFAULT_RDB_FILENAME);
rewriteConfigDirOption(state);
rewriteConfigSlaveofOption(state);
rewriteConfigStringOption(state,"slave-announce-ip",server.slave_announce_ip,CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP);
rewriteConfigStringOption(state,"masterauth",server.masterauth,NULL);
rewriteConfigStringOption(state,"cluster-announce-ip",server.cluster_announce_ip,NULL);
rewriteConfigYesNoOption(state,"slave-serve-stale-data",server.repl_serve_stale_data,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA);
rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,CONFIG_DEFAULT_SLAVE_READ_ONLY);
rewriteConfigNumericalOption(state,"repl-ping-slave-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD);
@@ -1894,6 +2090,11 @@ int rewriteConfig(char *path) {
rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,CONFIG_DEFAULT_MAXMEMORY);
rewriteConfigEnumOption(state,"maxmemory-policy",server.maxmemory_policy,maxmemory_policy_enum,CONFIG_DEFAULT_MAXMEMORY_POLICY);
rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,CONFIG_DEFAULT_MAXMEMORY_SAMPLES);
rewriteConfigNumericalOption(state,"active-defrag-threshold-lower",server.active_defrag_threshold_lower,CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER);
rewriteConfigNumericalOption(state,"active-defrag-threshold-upper",server.active_defrag_threshold_upper,CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER);
rewriteConfigBytesOption(state,"active-defrag-ignore-bytes",server.active_defrag_ignore_bytes,CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES);
rewriteConfigNumericalOption(state,"active-defrag-cycle-min",server.active_defrag_cycle_min,CONFIG_DEFAULT_DEFRAG_CYCLE_MIN);
rewriteConfigNumericalOption(state,"active-defrag-cycle-max",server.active_defrag_cycle_max,CONFIG_DEFAULT_DEFRAG_CYCLE_MAX);
rewriteConfigYesNoOption(state,"appendonly",server.aof_state != AOF_OFF,0);
rewriteConfigStringOption(state,"appendfilename",server.aof_filename,CONFIG_DEFAULT_AOF_FILENAME);
rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync,aof_fsync_enum,CONFIG_DEFAULT_AOF_FSYNC);
@@ -1920,12 +2121,18 @@ int rewriteConfig(char *path) {
rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,OBJ_ZSET_MAX_ZIPLIST_VALUE);
rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",server.hll_sparse_max_bytes,CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES);
rewriteConfigYesNoOption(state,"activerehashing",server.activerehashing,CONFIG_DEFAULT_ACTIVE_REHASHING);
rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG);
rewriteConfigYesNoOption(state,"protected-mode",server.protected_mode,CONFIG_DEFAULT_PROTECTED_MODE);
rewriteConfigClientoutputbufferlimitOption(state);
rewriteConfigNumericalOption(state,"hz",server.hz,CONFIG_DEFAULT_HZ);
rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC);
rewriteConfigYesNoOption(state,"aof-load-truncated",server.aof_load_truncated,CONFIG_DEFAULT_AOF_LOAD_TRUNCATED);
rewriteConfigYesNoOption(state,"aof-use-rdb-preamble",server.aof_use_rdb_preamble,CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE);
rewriteConfigEnumOption(state,"supervised",server.supervised_mode,supervised_mode_enum,SUPERVISED_NONE);
rewriteConfigYesNoOption(state,"lazyfree-lazy-eviction",server.lazyfree_lazy_eviction,CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION);
rewriteConfigYesNoOption(state,"lazyfree-lazy-expire",server.lazyfree_lazy_expire,CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE);
rewriteConfigYesNoOption(state,"lazyfree-lazy-server-del",server.lazyfree_lazy_server_del,CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL);
rewriteConfigYesNoOption(state,"slave-lazy-flush",server.repl_slave_lazy_flush,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH);
/* Rewrite Sentinel config if in Sentinel mode. */
if (server.sentinel_mode) rewriteConfigSentinelOption(state);
@@ -1974,8 +2181,8 @@ void configCommand(client *c) {
return;
}
if (rewriteConfig(server.configfile) == -1) {
serverLog(LL_WARNING,"CONFIG REWRITE failed: %s", strerror(errno));
addReplyErrorFormat(c,"Rewriting config file: %s", strerror(errno));
serverLog(LL_WARNING,"CONFIG REWRITE failed: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
addReplyErrorFormat(c,"Rewriting config file: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
} else {
serverLog(LL_WARNING,"CONFIG REWRITE executed with success.");
addReply(c,shared.ok);
src/config.h
+18
View File
@@ -217,4 +217,22 @@ void setproctitle(const char *fmt, ...);
#endif
#endif
/* Make sure we can test for ARM just checking for __arm__, since sometimes
* __arm is defined but __arm__ is not. */
#if defined(__arm) && !defined(__arm__)
#define __arm__
#endif
#if defined (__aarch64__) && !defined(__arm64__)
#define __arm64__
#endif
/* Make sure we can test for SPARC just checking for __sparc__. */
#if defined(__sparc) && !defined(__sparc__)
#define __sparc__
#endif
#if defined(__sparc__) || defined(__arm__)
#define USE_ALIGNED_ACCESS
#endif
#endif
src/db.c
+344 -229
View File
@@ -32,14 +32,11 @@
#include "Win32_Interop/Win32_QFork.h"
#endif
#include "cluster.h"
#include "atomicvar.h"
#include <signal.h>
#include <ctype.h>
void slotToKeyAdd(robj *key);
void slotToKeyDel(robj *key);
void slotToKeyFlush(void);
/*-----------------------------------------------------------------------------
* C-level DB API
*----------------------------------------------------------------------------*/
@@ -59,7 +56,13 @@ robj *lookupKey(redisDb *db, robj *key, int flags) {
server.aof_child_pid == -1 &&
!(flags & LOOKUP_NOTOUCH))
{
val->lru = LRU_CLOCK();
if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) {
PORT_LONG ldt = val->lru >> 8;
PORT_LONG counter = LFULogIncr(val->lru & 255);
val->lru = (ldt << 8) | counter;
} else {
val->lru = LRU_CLOCK();
}
}
return val;
} else {
@@ -93,7 +96,7 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags) {
if (expireIfNeeded(db,key) == 1) {
/* Key expired. If we are in the context of a master, expireIfNeeded()
* returns 0 only when the key does not exist at all, so it's save
* returns 0 only when the key does not exist at all, so it's safe
* to return NULL ASAP. */
if (server.masterhost == NULL) return NULL;
@@ -175,7 +178,14 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) {
dictEntry *de = dictFind(db->dict,key->ptr);
serverAssertWithInfo(NULL,key,de != NULL);
dictReplace(db->dict, key->ptr, val);
if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) {
robj *old = dictGetVal(de);
int saved_lru = old->lru;
dictReplace(db->dict, key->ptr, val);
val->lru = saved_lru;
} else {
dictReplace(db->dict, key->ptr, val);
}
}
/* High level Set operation. This function can be used in order to set
@@ -183,7 +193,9 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) {
*
* 1) The ref count of the value object is incremented.
* 2) clients WATCHing for the destination key notified.
* 3) The expire time of the key is reset (the key is made persistent). */
* 3) The expire time of the key is reset (the key is made persistent).
*
* All the new keys in the database should be craeted via this interface. */
void setKey(redisDb *db, robj *key, robj *val) {
if (lookupKeyWrite(db,key) == NULL) {
dbAdd(db,key,val);
@@ -226,7 +238,7 @@ robj *dbRandomKey(redisDb *db) {
}
/* Delete a key, value, and associated expiration entry if any, from the DB */
int dbDelete(redisDb *db, robj *key) {
int dbSyncDelete(redisDb *db, robj *key) {
/* Deleting an entry from the expires dict will not free the sds of
* the key, because it is shared with the main dictionary. */
if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr);
@@ -238,6 +250,13 @@ int dbDelete(redisDb *db, robj *key) {
}
}
/* This is a wrapper whose behavior depends on the Redis lazy free
* configuration. Deletes the key synchronously or asynchronously. */
int dbDelete(redisDb *db, robj *key) {
return server.lazyfree_lazy_server_del ? dbAsyncDelete(db,key) :
dbSyncDelete(db,key);
}
/* Prepare the string object stored at 'key' to be modified destructively
* to implement commands like SETBIT or APPEND.
*
@@ -276,16 +295,47 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) {
return o;
}
PORT_LONGLONG emptyDb(void(callback)(void*)) {
int j;
/* Remove all keys from all the databases in a Redis server.
* If callback is given the function is called from time to time to
* signal that work is in progress.
*
* The dbnum can be -1 if all teh DBs should be flushed, or the specified
* DB number if we want to flush only a single Redis database number.
*
* Flags are be EMPTYDB_NO_FLAGS if no special flags are specified or
* EMPTYDB_ASYNC if we want the memory to be freed in a different thread
* and the function to return ASAP.
*
* On success the fuction returns the number of keys removed from the
* database(s). Otherwise -1 is returned in the specific case the
* DB number is out of range, and errno is set to EINVAL. */
PORT_LONGLONG emptyDb(int dbnum, int flags, void(callback)(void*)) {
int j, async = (flags & EMPTYDB_ASYNC);
PORT_LONGLONG removed = 0;
for (j = 0; j < server.dbnum; j++) {
removed += dictSize(server.db[j].dict);
dictEmpty(server.db[j].dict,callback);
dictEmpty(server.db[j].expires,callback);
if (dbnum < -1 || dbnum >= server.dbnum) {
errno = EINVAL;
return -1;
}
if (server.cluster_enabled) slotToKeyFlush();
for (j = 0; j < server.dbnum; j++) {
if (dbnum != -1 && dbnum != j) continue;
removed += dictSize(server.db[j].dict);
if (async) {
emptyDbAsync(&server.db[j]);
} else {
dictEmpty(server.db[j].dict,callback);
dictEmpty(server.db[j].expires,callback);
}
}
if (server.cluster_enabled) {
if (async) {
slotToKeyFlushAsync();
} else {
slotToKeyFlush();
}
}
if (dbnum == -1) flushSlaveKeysWithExpireList();
return removed;
}
@@ -317,18 +367,49 @@ void signalFlushedDb(int dbid) {
* Type agnostic commands operating on the key space
*----------------------------------------------------------------------------*/
/* Return the set of flags to use for the emptyDb() call for FLUSHALL
* and FLUSHDB commands.
*
* Currently the command just attempts to parse the "ASYNC" option. It
* also checks if the command arity is wrong.
*
* On success C_OK is returned and the flags are stored in *flags, otherwise
* C_ERR is returned and the function sends an error to the client. */
int getFlushCommandFlags(client *c, int *flags) {
/* Parse the optional ASYNC option. */
if (c->argc > 1) {
if (c->argc > 2 || strcasecmp(c->argv[1]->ptr,"async")) {
addReply(c,shared.syntaxerr);
return C_ERR;
}
*flags = EMPTYDB_ASYNC;
} else {
*flags = EMPTYDB_NO_FLAGS;
}
return C_OK;
}
/* FLUSHDB [ASYNC]
*
* Flushes the currently SELECTed Redis DB. */
void flushdbCommand(client *c) {
server.dirty += dictSize(c->db->dict);
int flags;
if (getFlushCommandFlags(c,&flags) == C_ERR) return;
signalFlushedDb(c->db->id);
dictEmpty(c->db->dict,NULL);
dictEmpty(c->db->expires,NULL);
if (server.cluster_enabled) slotToKeyFlush();
server.dirty += emptyDb(c->db->id,flags,NULL);
addReply(c,shared.ok);
}
/* FLUSHALL [ASYNC]
*
* Flushes the whole server data set. */
void flushallCommand(client *c) {
int flags;
if (getFlushCommandFlags(c,&flags) == C_ERR) return;
signalFlushedDb(-1);
server.dirty += emptyDb(NULL);
server.dirty += emptyDb(-1,flags,NULL);
addReply(c,shared.ok);
if (server.rdb_child_pid != -1) {
#ifdef _WIN32
@@ -341,27 +422,40 @@ void flushallCommand(client *c) {
if (server.saveparamslen > 0) {
/* Normally rdbSave() will reset dirty, but we don't want this here
* as otherwise FLUSHALL will not be replicated nor put into the AOF. */
PORT_LONGLONG saved_dirty = server.dirty; /* UPSTREAM_FIX: server.dirty is a PORT_LONGLONG not an int */
rdbSave(server.rdb_filename);
int saved_dirty = server.dirty;
rdbSaveInfo rsi, *rsiptr;
rsiptr = rdbPopulateSaveInfo(&rsi);
rdbSave(server.rdb_filename,rsiptr);
server.dirty = saved_dirty;
}
server.dirty++;
}
void delCommand(client *c) {
int deleted = 0, j;
/* This command implements DEL and LAZYDEL. */
void delGenericCommand(client *c, int lazy) {
int numdel = 0, j;
for (j = 1; j < c->argc; j++) {
expireIfNeeded(c->db,c->argv[j]);
if (dbDelete(c->db,c->argv[j])) {
int deleted = lazy ? dbAsyncDelete(c->db,c->argv[j]) :
dbSyncDelete(c->db,c->argv[j]);
if (deleted) {
signalModifiedKey(c->db,c->argv[j]);
notifyKeyspaceEvent(NOTIFY_GENERIC,
"del",c->argv[j],c->db->id);
server.dirty++;
deleted++;
numdel++;
}
}
addReplyLongLong(c,deleted);
addReplyLongLong(c,numdel);
}
void delCommand(client *c) {
delGenericCommand(c,0);
}
void unlinkCommand(client *c) {
delGenericCommand(c,1);
}
/* EXISTS key1 key2 ... key_N.
@@ -389,7 +483,7 @@ void selectCommand(client *c) {
return;
}
if (selectDb(c,(int)id) == C_ERR) { WIN_PORT_FIX /* cast (int) */
addReplyError(c,"invalid DB index");
addReplyError(c,"DB index is out of range");
} else {
addReply(c,shared.ok);
}
@@ -446,16 +540,16 @@ void scanCallback(void *privdata, const dictEntry *de) {
sds sdskey = dictGetKey(de);
key = createStringObject(sdskey, sdslen(sdskey));
} else if (o->type == OBJ_SET) {
key = dictGetKey(de);
incrRefCount(key);
sds keysds = dictGetKey(de);
key = createStringObject(keysds,sdslen(keysds));
} else if (o->type == OBJ_HASH) {
key = dictGetKey(de);
incrRefCount(key);
val = dictGetVal(de);
incrRefCount(val);
sds sdskey = dictGetKey(de);
sds sdsval = dictGetVal(de);
key = createStringObject(sdskey,sdslen(sdskey));
val = createStringObject(sdsval,sdslen(sdsval));
} else if (o->type == OBJ_ZSET) {
key = dictGetKey(de);
incrRefCount(key);
sds sdskey = dictGetKey(de);
key = createStringObject(sdskey,sdslen(sdskey));
val = createStringObjectFromLongDouble(*(double*)dictGetVal(de),0);
} else {
serverPanic("Type not handled in SCAN callback.");
@@ -580,7 +674,7 @@ void scanGenericCommand(client *c, robj *o, PORT_ULONG cursor) {
privdata[0] = keys;
privdata[1] = o;
do {
cursor = dictScan(ht, cursor, scanCallback, privdata);
cursor = dictScan(ht, cursor, scanCallback, NULL, privdata);
} while (cursor &&
maxiterations-- &&
listLength(keys) < (PORT_ULONG)count);
@@ -701,6 +795,10 @@ void typeCommand(client *c) {
case OBJ_SET: type = "set"; break;
case OBJ_ZSET: type = "zset"; break;
case OBJ_HASH: type = "hash"; break;
case OBJ_MODULE: {
moduleValue *mv = o->ptr;
type = mv->type->name;
}; break;
default: type = "unknown"; break;
}
}
@@ -765,7 +863,7 @@ void renameGenericCommand(client *c, int nx) {
dbDelete(c->db,c->argv[2]);
}
dbAdd(c->db,c->argv[2],o);
if (expire != -1) setExpire(c->db,c->argv[2],expire);
if (expire != -1) setExpire(c,c->db,c->argv[2],expire);
dbDelete(c->db,c->argv[1]);
signalModifiedKey(c->db,c->argv[1]);
signalModifiedKey(c->db,c->argv[2]);
@@ -831,7 +929,7 @@ void moveCommand(client *c) {
return;
}
dbAdd(dst,c->argv[1],o);
if (expire != -1) setExpire(dst,c->argv[1],expire);
if (expire != -1) setExpire(c,dst,c->argv[1],expire);
incrRefCount(o);
/* OK! key moved, free the entry in the source DB */
@@ -840,6 +938,91 @@ void moveCommand(client *c) {
addReply(c,shared.cone);
}
/* Helper function for dbSwapDatabases(): scans the list of keys that have
* one or more blocked clients for B[LR]POP or other list blocking commands
* and signal the keys are ready if they are lists. See the comment where
* the function is used for more info. */
void scanDatabaseForReadyLists(redisDb *db) {
dictEntry *de;
dictIterator *di = dictGetSafeIterator(db->blocking_keys);
while((de = dictNext(di)) != NULL) {
robj *key = dictGetKey(de);
robj *value = lookupKey(db,key,LOOKUP_NOTOUCH);
if (value && value->type == OBJ_LIST)
signalListAsReady(db, key);
}
dictReleaseIterator(di);
}
/* Swap two databases at runtime so that all clients will magically see
* the new database even if already connected. Note that the client
* structure c->db points to a given DB, so we need to be smarter and
* swap the underlying referenced structures, otherwise we would need
* to fix all the references to the Redis DB structure.
*
* Returns C_ERR if at least one of the DB ids are out of range, otherwise
* C_OK is returned. */
int dbSwapDatabases(int id1, int id2) {
if (id1 < 0 || id1 >= server.dbnum ||
id2 < 0 || id2 >= server.dbnum) return C_ERR;
if (id1 == id2) return C_OK;
redisDb aux = server.db[id1];
redisDb *db1 = &server.db[id1], *db2 = &server.db[id2];
/* Swap hash tables. Note that we don't swap blocking_keys,
* ready_keys and watched_keys, since we want clients to
* remain in the same DB they were. */
db1->dict = db2->dict;
db1->expires = db2->expires;
db1->avg_ttl = db2->avg_ttl;
db2->dict = aux.dict;
db2->expires = aux.expires;
db2->avg_ttl = aux.avg_ttl;
/* Now we need to handle clients blocked on lists: as an effect
* of swapping the two DBs, a client that was waiting for list
* X in a given DB, may now actually be unblocked if X happens
* to exist in the new version of the DB, after the swap.
*
* However normally we only do this check for efficiency reasons
* in dbAdd() when a list is created. So here we need to rescan
* the list of clients blocked on lists and signal lists as ready
* if needed. */
scanDatabaseForReadyLists(db1);
scanDatabaseForReadyLists(db2);
return C_OK;
}
/* SWAPDB db1 db2 */
void swapdbCommand(client *c) {
PORT_LONG id1, id2;
/* Not allowed in cluster mode: we have just DB 0 there. */
if (server.cluster_enabled) {
addReplyError(c,"SWAPDB is not allowed in cluster mode");
return;
}
/* Get the two DBs indexes. */
if (getLongFromObjectOrReply(c, c->argv[1], &id1,
"invalid first DB index") != C_OK)
return;
if (getLongFromObjectOrReply(c, c->argv[2], &id2,
"invalid second DB index") != C_OK)
return;
/* Swap... */
if (dbSwapDatabases(id1,id2) == C_ERR) {
addReplyError(c,"DB index is out of range");
return;
} else {
server.dirty++;
addReply(c,shared.ok);
}
}
/*-----------------------------------------------------------------------------
* Expires API
*----------------------------------------------------------------------------*/
@@ -851,14 +1034,22 @@ int removeExpire(redisDb *db, robj *key) {
return dictDelete(db->expires,key->ptr) == DICT_OK;
}
void setExpire(redisDb *db, robj *key, PORT_LONGLONG when) {
/* Set an expire to the specified key. If the expire is set in the context
* of an user calling a command 'c' is the client, otherwise 'c' is set
* to NULL. The 'when' parameter is the absolute unix time in milliseconds
* after which the key will no longer be considered valid. */
void setExpire(client *c, redisDb *db, robj *key, PORT_LONGLONG when) {
dictEntry *kde, *de;
/* Reuse the sds from the main dict in the expire dict */
kde = dictFind(db->dict,key->ptr);
serverAssertWithInfo(NULL,key,kde != NULL);
de = dictReplaceRaw(db->expires,dictGetKey(kde));
de = dictAddOrFind(db->expires,dictGetKey(kde));
dictSetSignedIntegerVal(de,when);
int writable_slave = server.masterhost && server.repl_slave_ro == 0;
if (c && writable_slave && !(c->flags & CLIENT_MASTER))
rememberSlaveKeyWithExpire(db,key);
}
/* Return the expire time of the specified key, or -1 if no expire
@@ -884,10 +1075,10 @@ PORT_LONGLONG getExpire(redisDb *db, robj *key) {
* AOF and the master->slave link guarantee operation ordering, everything
* will be consistent even if we allow write operations against expiring
* keys. */
void propagateExpire(redisDb *db, robj *key) {
void propagateExpire(redisDb *db, robj *key, int lazy) {
robj *argv[2];
argv[0] = shared.del;
argv[0] = lazy ? shared.unlink : shared.del;
argv[1] = key;
incrRefCount(argv[0]);
incrRefCount(argv[1]);
@@ -930,137 +1121,11 @@ int expireIfNeeded(redisDb *db, robj *key) {
/* Delete the key */
server.stat_expiredkeys++;
propagateExpire(db,key);
propagateExpire(db,key,server.lazyfree_lazy_expire);
notifyKeyspaceEvent(NOTIFY_EXPIRED,
"expired",key,db->id);
return dbDelete(db,key);
}
/*-----------------------------------------------------------------------------
* Expires Commands
*----------------------------------------------------------------------------*/
/* This is the generic command implementation for EXPIRE, PEXPIRE, EXPIREAT
* and PEXPIREAT. Because the commad second argument may be relative or absolute
* the "basetime" argument is used to signal what the base time is (either 0
* for *AT variants of the command, or the current time for relative expires).
*
* unit is either UNIT_SECONDS or UNIT_MILLISECONDS, and is only used for
* the argv[2] parameter. The basetime is always specified in milliseconds. */
void expireGenericCommand(client *c, PORT_LONGLONG basetime, int unit) {
robj *key = c->argv[1], *param = c->argv[2];
PORT_LONGLONG when; /* unix time in milliseconds when the key will expire. */
if (getLongLongFromObjectOrReply(c, param, &when, NULL) != C_OK)
return;
if (unit == UNIT_SECONDS) when *= 1000;
when += basetime;
/* No key, return zero. */
if (lookupKeyWrite(c->db,key) == NULL) {
addReply(c,shared.czero);
return;
}
/* EXPIRE with negative TTL, or EXPIREAT with a timestamp into the past
* should never be executed as a DEL when load the AOF or in the context
* of a slave instance.
*
* Instead we take the other branch of the IF statement setting an expire
* (possibly in the past) and wait for an explicit DEL from the master. */
if (when <= mstime() && !server.loading && !server.masterhost) {
robj *aux;
serverAssertWithInfo(c,key,dbDelete(c->db,key));
server.dirty++;
/* Replicate/AOF this as an explicit DEL. */
aux = createStringObject("DEL",3);
rewriteClientCommandVector(c,2,aux,key);
decrRefCount(aux);
signalModifiedKey(c->db,key);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
addReply(c, shared.cone);
return;
} else {
setExpire(c->db,key,when);
addReply(c,shared.cone);
signalModifiedKey(c->db,key);
notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id);
server.dirty++;
return;
}
}
void expireCommand(client *c) {
expireGenericCommand(c,mstime(),UNIT_SECONDS);
}
void expireatCommand(client *c) {
expireGenericCommand(c,0,UNIT_SECONDS);
}
void pexpireCommand(client *c) {
expireGenericCommand(c,mstime(),UNIT_MILLISECONDS);
}
void pexpireatCommand(client *c) {
expireGenericCommand(c,0,UNIT_MILLISECONDS);
}
void ttlGenericCommand(client *c, int output_ms) {
PORT_LONGLONG expire, ttl = -1;
/* If the key does not exist at all, return -2 */
if (lookupKeyReadWithFlags(c->db,c->argv[1],LOOKUP_NOTOUCH) == NULL) {
addReplyLongLong(c,-2);
return;
}
/* The key exists. Return -1 if it has no expire, or the actual
* TTL value otherwise. */
expire = getExpire(c->db,c->argv[1]);
if (expire != -1) {
ttl = expire-mstime();
if (ttl < 0) ttl = 0;
}
if (ttl == -1) {
addReplyLongLong(c,-1);
} else {
addReplyLongLong(c,output_ms ? ttl : ((ttl+500)/1000));
}
}
void ttlCommand(client *c) {
ttlGenericCommand(c, 0);
}
void pttlCommand(client *c) {
ttlGenericCommand(c, 1);
}
void persistCommand(client *c) {
dictEntry *de;
de = dictFind(c->db->dict,c->argv[1]->ptr);
if (de == NULL) {
addReply(c,shared.czero);
} else {
if (removeExpire(c->db,c->argv[1])) {
addReply(c,shared.cone);
server.dirty++;
} else {
addReply(c,shared.czero);
}
}
}
/* TOUCH key1 [key2 key3 ... keyN] */
void touchCommand(client *c) {
int touched = 0;
for (int j = 1; j < c->argc; j++)
if (lookupKeyRead(c->db,c->argv[j]) != NULL) touched++;
addReplyLongLong(c,touched);
return server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) :
dbSyncDelete(db,key);
}
/* -----------------------------------------------------------------------------
@@ -1077,11 +1142,24 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in
*numkeys = 0;
return NULL;
}
last = cmd->lastkey;
if (last < 0) last = argc+last;
keys = zmalloc(sizeof(int)*((last - cmd->firstkey)+1));
for (j = cmd->firstkey; j <= last; j += cmd->keystep) {
serverAssert(j < argc);
if (j >= argc) {
/* Modules command do not have dispatch time arity checks, so
* we need to handle the case where the user passed an invalid
* number of arguments here. In this case we return no keys
* and expect the module command to report an arity error. */
if (cmd->flags & CMD_MODULE) {
zfree(keys);
*numkeys = 0;
return NULL;
} else {
serverPanic("Redis built-in command declared keys positions not matching the arity requirements.");
}
}
keys[i++] = j;
}
*numkeys = i;
@@ -1100,7 +1178,9 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in
* This function uses the command table if a command-specific helper function
* is not required, otherwise it calls the command-specific function. */
int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
if (cmd->getkeys_proc) {
if (cmd->flags & CMD_MODULE_GETKEYS) {
return moduleGetCommandKeysViaAPI(cmd,argv,argc,numkeys);
} else if (!(cmd->flags & CMD_MODULE) && cmd->getkeys_proc) {
return cmd->getkeys_proc(cmd,argv,argc,numkeys);
} else {
return getKeysUsingCommandTable(cmd,argv,argc,numkeys);
@@ -1241,90 +1321,125 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey
return keys;
}
/* Helper function to extract keys from following commands:
* GEORADIUS key x y radius unit [WITHDIST] [WITHHASH] [WITHCOORD] [ASC|DESC]
* [COUNT count] [STORE key] [STOREDIST key]
* GEORADIUSBYMEMBER key member radius unit ... options ... */
int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) {
int i, num, *keys;
UNUSED(cmd);
/* Check for the presence of the stored key in the command */
int stored_key = -1;
for (i = 5; i < argc; i++) {
char *arg = argv[i]->ptr;
/* For the case when user specifies both "store" and "storedist" options, the
* second key specified would override the first key. This behavior is kept
* the same as in georadiusCommand method.
*/
if ((!strcasecmp(arg, "store") || !strcasecmp(arg, "storedist")) && ((i+1) < argc)) {
stored_key = i+1;
i++;
}
}
num = 1 + (stored_key == -1 ? 0 : 1);
/* Keys in the command come from two places:
* argv[1] = key,
* argv[5...n] = stored key if present
*/
keys = zmalloc(sizeof(int) * num);
/* Add all key positions to keys[] */
keys[0] = 1;
if(num > 1) {
keys[1] = stored_key;
}
*numkeys = num;
return keys;
}
/* Slot to Key API. This is used by Redis Cluster in order to obtain in
* a fast way a key that belongs to a specified hash slot. This is useful
* while rehashing the cluster. */
void slotToKeyAdd(robj *key) {
unsigned int hashslot = keyHashSlot(key->ptr,(int)sdslen(key->ptr)); WIN_PORT_FIX /* cast (int) */
* while rehashing the cluster and in other conditions when we need to
* understand if we have keys for a given hash slot. */
void slotToKeyUpdateKey(robj *key, int add) {
unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr));
unsigned char buf[64];
unsigned char *indexed = buf;
size_t keylen = sdslen(key->ptr);
zslInsert(server.cluster->slots_to_keys,hashslot,key);
incrRefCount(key);
server.cluster->slots_keys_count[hashslot] += add ? 1 : -1;
if (keylen+2 > 64) indexed = zmalloc(keylen+2);
indexed[0] = (hashslot >> 8) & 0xff;
indexed[1] = hashslot & 0xff;
memcpy(indexed+2,key->ptr,keylen);
if (add) {
raxInsert(server.cluster->slots_to_keys,indexed,keylen+2,NULL,NULL);
} else {
raxRemove(server.cluster->slots_to_keys,indexed,keylen+2,NULL);
}
if (indexed != buf) zfree(indexed);
}
void slotToKeyAdd(robj *key) {
slotToKeyUpdateKey(key,1);
}
void slotToKeyDel(robj *key) {
unsigned int hashslot = keyHashSlot(key->ptr,(int)sdslen(key->ptr)); WIN_PORT_FIX /* cast (int) */
zslDelete(server.cluster->slots_to_keys,hashslot,key);
slotToKeyUpdateKey(key,0);
}
void slotToKeyFlush(void) {
zslFree(server.cluster->slots_to_keys);
server.cluster->slots_to_keys = zslCreate();
raxFree(server.cluster->slots_to_keys);
server.cluster->slots_to_keys = raxNew();
memset(server.cluster->slots_keys_count,0,
sizeof(server.cluster->slots_keys_count));
}
/* Pupulate the specified array of objects with keys in the specified slot.
* New objects are returned to represent keys, it's up to the caller to
* decrement the reference count to release the keys names. */
unsigned int getKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count) {
zskiplistNode *n;
zrangespec range;
raxIterator iter;
int j = 0;
unsigned char indexed[2];
range.min = range.max = hashslot;
range.minex = range.maxex = 0;
n = zslFirstInRange(server.cluster->slots_to_keys, &range);
while(n && n->score == hashslot && count--) {
keys[j++] = n->obj;
n = n->level[0].forward;
indexed[0] = (hashslot >> 8) & 0xff;
indexed[1] = hashslot & 0xff;
raxStart(&iter,server.cluster->slots_to_keys);
raxSeek(&iter,">=",indexed,2);
while(count-- && raxNext(&iter)) {
if (iter.key[0] != indexed[0] || iter.key[1] != indexed[1]) break;
keys[j++] = createStringObject((char*)iter.key+2,iter.key_len-2);
}
raxStop(&iter);
return j;
}
/* Remove all the keys in the specified hash slot.
* The number of removed items is returned. */
unsigned int delKeysInSlot(unsigned int hashslot) {
zskiplistNode *n;
zrangespec range;
raxIterator iter;
int j = 0;
unsigned char indexed[2];
range.min = range.max = hashslot;
range.minex = range.maxex = 0;
indexed[0] = (hashslot >> 8) & 0xff;
indexed[1] = hashslot & 0xff;
raxStart(&iter,server.cluster->slots_to_keys);
while(server.cluster->slots_keys_count[hashslot]) {
raxSeek(&iter,">=",indexed,2);
raxNext(&iter);
n = zslFirstInRange(server.cluster->slots_to_keys, &range);
while(n && n->score == hashslot) {
robj *key = n->obj;
n = n->level[0].forward; /* Go to the next item before freeing it. */
incrRefCount(key); /* Protect the object while freeing it. */
robj *key = createStringObject((char*)iter.key+2,iter.key_len-2);
dbDelete(&server.db[0],key);
decrRefCount(key);
j++;
}
raxStop(&iter);
return j;
}
unsigned int countKeysInSlot(unsigned int hashslot) {
zskiplist *zsl = server.cluster->slots_to_keys;
zskiplistNode *zn;
zrangespec range;
int rank, count = 0;
range.min = range.max = hashslot;
range.minex = range.maxex = 0;
/* Find first element in range */
zn = zslFirstInRange(zsl, &range);
/* Use rank of first element, if any, to determine preliminary count */
if (zn != NULL) {
rank = (int) zslGetRank(zsl, zn->score, zn->obj); WIN_PORT_FIX /* cast (int) */
count = (int) (zsl->length - (rank - 1)); WIN_PORT_FIX /* cast (int) */
/* Find last element in range */
zn = zslLastInRange(zsl, &range);
/* Use rank of last element, if any, to determine the actual count */
if (zn != NULL) {
rank = (int) zslGetRank(zsl, zn->score, zn->obj); WIN_PORT_FIX /* cast (int) */
count -= (int) (zsl->length - rank); WIN_PORT_FIX /* cast (int) */
}
}
return count;
return server.cluster->slots_keys_count[hashslot];
}
src/debug.c
+146 -78
View File
@@ -33,8 +33,10 @@
#ifndef _WIN32
#include <arpa/inet.h>
#include <dlfcn.h>
#else
#include "Win32_Interop/Win32_Portability.h"
#include "Win32_Interop/dlfcn.h"
#endif
#include <signal.h>
@@ -43,6 +45,7 @@
#include <ucontext.h>
#include <fcntl.h>
#include "bio.h"
#include <unistd.h>
#endif /* HAVE_BACKTRACE */
#ifdef __CYGWIN__
@@ -128,7 +131,7 @@ void computeDatasetDigest(unsigned char *final) {
redisDb *db = server.db+j;
if (dictSize(db->dict) == 0) continue;
di = dictGetIterator(db->dict);
di = dictGetSafeIterator(db->dict);
/* hash the DB id, so the same dataset moved in a different
* DB will lead to a different digest */
@@ -167,10 +170,10 @@ void computeDatasetDigest(unsigned char *final) {
listTypeReleaseIterator(li);
} else if (o->type == OBJ_SET) {
setTypeIterator *si = setTypeInitIterator(o);
robj *ele;
while((ele = setTypeNextObject(si)) != NULL) {
xorObjectDigest(digest,ele);
decrRefCount(ele);
sds sdsele;
while((sdsele = setTypeNextObject(si)) != NULL) {
xorDigest(digest,sdsele,sdslen(sdsele));
sdsfree(sdsele);
}
setTypeReleaseIterator(si);
} else if (o->type == OBJ_ZSET) {
@@ -212,12 +215,12 @@ void computeDatasetDigest(unsigned char *final) {
dictEntry *de;
while((de = dictNext(di)) != NULL) {
robj *eleobj = dictGetKey(de);
sds sdsele = dictGetKey(de);
double *score = dictGetVal(de);
snprintf(buf,sizeof(buf),"%.17g",*score);
memset(eledigest,0,20);
mixObjectDigest(eledigest,eleobj);
mixDigest(eledigest,sdsele,sdslen(sdsele));
mixDigest(eledigest,buf,strlen(buf));
xorDigest(digest,eledigest,20);
}
@@ -226,23 +229,30 @@ void computeDatasetDigest(unsigned char *final) {
serverPanic("Unknown sorted set encoding");
}
} else if (o->type == OBJ_HASH) {
hashTypeIterator *hi;
robj *obj;
hi = hashTypeInitIterator(o);
hashTypeIterator *hi = hashTypeInitIterator(o);
while (hashTypeNext(hi) != C_ERR) {
unsigned char eledigest[20];
sds sdsele;
memset(eledigest,0,20);
obj = hashTypeCurrentObject(hi,OBJ_HASH_KEY);
mixObjectDigest(eledigest,obj);
decrRefCount(obj);
obj = hashTypeCurrentObject(hi,OBJ_HASH_VALUE);
mixObjectDigest(eledigest,obj);
decrRefCount(obj);
sdsele = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_KEY);
mixDigest(eledigest,sdsele,sdslen(sdsele));
sdsfree(sdsele);
sdsele = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE);
mixDigest(eledigest,sdsele,sdslen(sdsele));
sdsfree(sdsele);
xorDigest(digest,eledigest,20);
}
hashTypeReleaseIterator(hi);
} else if (o->type == OBJ_MODULE) {
RedisModuleDigest md;
moduleValue *mv = o->ptr;
moduleType *mt = mv->type;
moduleInitDigestContext(md);
if (mt->digest) {
mt->digest(&md,mv->value);
xorDigest(digest,md.x,sizeof(md.x));
}
} else {
serverPanic("Unknown object type");
}
@@ -256,14 +266,6 @@ void computeDatasetDigest(unsigned char *final) {
}
}
#if defined(USE_JEMALLOC)
void inputCatSds(void *result, const char *str) {
/* result is actually a (sds *), so re-cast it here */
sds *info = (sds *)result;
*info = sdscat(*info, str);
}
#endif
void debugCommand(client *c) {
if (c->argc == 1) {
addReplyError(c,"You must specify a subcommand for DEBUG. Try DEBUG HELP for info.");
@@ -278,6 +280,8 @@ void debugCommand(client *c) {
blen++; addReplyStatus(c,
"segfault -- Crash the server with sigsegv.");
blen++; addReplyStatus(c,
"panic -- Crash the server simulating a panic.");
blen++; addReplyStatus(c,
"restart -- Graceful restart: save config, db, restart.");
blen++; addReplyStatus(c,
"crash-and-recovery <milliseconds> -- Hard crash and restart after <milliseconds> delay.");
@@ -292,7 +296,9 @@ void debugCommand(client *c) {
blen++; addReplyStatus(c,
"sdslen <key> -- Show low level SDS string info representing key and value.");
blen++; addReplyStatus(c,
"populate <count> [prefix] -- Create <count> string keys named key:<num>. If a prefix is specified is used instead of the 'key' prefix.");
"ziplist <key> -- Show low level info about the ziplist encoding.");
blen++; addReplyStatus(c,
"populate <count> [prefix] [size] -- Create <count> string keys named key:<num>. If a prefix is specified is used instead of the 'key' prefix.");
blen++; addReplyStatus(c,
"digest -- Outputs an hex signature representing the current DB content.");
blen++; addReplyStatus(c,
@@ -307,13 +313,11 @@ void debugCommand(client *c) {
"structsize -- Return the size of different Redis core C structures.");
blen++; addReplyStatus(c,
"htstats <dbid> -- Return hash table statistics of the specified Redis database.");
blen++; addReplyStatus(c,
"jemalloc info -- Show internal jemalloc statistics.");
blen++; addReplyStatus(c,
"jemalloc purge -- Force jemalloc to release unused memory.");
setDeferredMultiBulkLength(c,blenp,blen);
} else if (!strcasecmp(c->argv[1]->ptr,"segfault")) {
*((char*)-1) = 'x';
} else if (!strcasecmp(c->argv[1]->ptr,"panic")) {
serverPanic("DEBUG PANIC called at Unix time %Id", time(NULL)); WIN_PORT_FIX /* %ld -> %Id */
} else if (!strcasecmp(c->argv[1]->ptr,"restart") ||
!strcasecmp(c->argv[1]->ptr,"crash-and-recover"))
{
@@ -336,12 +340,14 @@ void debugCommand(client *c) {
if (c->argc >= 3) c->argv[2] = tryObjectEncoding(c->argv[2]);
serverAssertWithInfo(c,c->argv[0],1 == 2);
} else if (!strcasecmp(c->argv[1]->ptr,"reload")) {
if (rdbSave(server.rdb_filename) != C_OK) {
rdbSaveInfo rsi, *rsiptr;
rsiptr = rdbPopulateSaveInfo(&rsi);
if (rdbSave(server.rdb_filename,rsiptr) != C_OK) {
addReply(c,shared.err);
return;
}
emptyDb(NULL);
if (rdbLoad(server.rdb_filename) != C_OK) {
emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
if (rdbLoad(server.rdb_filename,NULL) != C_OK) {
addReplyError(c,"Error trying to load the RDB dump");
return;
}
@@ -349,7 +355,7 @@ void debugCommand(client *c) {
addReply(c,shared.ok);
} else if (!strcasecmp(c->argv[1]->ptr,"loadaof")) {
if (server.aof_state == AOF_ON) flushAppendOnlyFile(1);
emptyDb(NULL);
emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
if (loadAppendOnlyFile(server.aof_filename) != C_OK) {
addReply(c,shared.err);
return;
@@ -397,7 +403,7 @@ void debugCommand(client *c) {
for (quicklistNode *node = ql->head; node; node = node->next) {
sz += node->sz;
}
used = snprintf(nextra, remaining, " ql_uncompressed_size:%lu", sz);
used = snprintf(nextra, remaining, " ql_uncompressed_size:%Iu", sz); WIN_PORT_FIX /* %lu -> %Iu */
nextra += used;
remaining -= used;
}
@@ -425,15 +431,29 @@ void debugCommand(client *c) {
addReplyError(c,"Not an sds encoded string.");
} else {
addReplyStatusFormat(c,
"key_sds_len:%lld, key_sds_avail:%lld, "
"val_sds_len:%lld, val_sds_avail:%lld",
"key_sds_len:%lld, key_sds_avail:%lld, key_zmalloc: %lld, "
"val_sds_len:%lld, val_sds_avail:%lld, val_zmalloc: %lld",
(PORT_LONGLONG) sdslen(key),
(PORT_LONGLONG) sdsavail(key),
(PORT_LONGLONG) sdsZmallocSize(key),
(PORT_LONGLONG) sdslen(val->ptr),
(PORT_LONGLONG) sdsavail(val->ptr));
(PORT_LONGLONG) sdsavail(val->ptr),
(PORT_LONGLONG) getStringObjectSdsUsedMemory(val));
}
} else if (!strcasecmp(c->argv[1]->ptr,"ziplist") && c->argc == 3) {
robj *o;
if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nokeyerr))
== NULL) return;
if (o->encoding != OBJ_ENCODING_ZIPLIST) {
addReplyError(c,"Not an sds encoded string.");
} else {
ziplistRepr(o->ptr);
addReplyStatus(c,"Ziplist structure printed on stdout");
}
} else if (!strcasecmp(c->argv[1]->ptr,"populate") &&
(c->argc == 3 || c->argc == 4)) {
c->argc >= 3 && c->argc <= 5) {
PORT_LONG keys, j;
robj *key, *val;
char buf[128];
@@ -442,15 +462,25 @@ void debugCommand(client *c) {
return;
dictExpand(c->db->dict,keys);
for (j = 0; j < keys; j++) {
PORT_LONG valsize = 0;
snprintf(buf,sizeof(buf),"%s:%Iu", WIN_PORT_FIX /* %lu -> %Iu */
(c->argc == 3) ? "key" : (char*)c->argv[3]->ptr, j);
key = createStringObject(buf,strlen(buf));
if (c->argc == 5)
if (getLongFromObjectOrReply(c, c->argv[4], &valsize, NULL) != C_OK)
return;
if (lookupKeyWrite(c->db,key) != NULL) {
decrRefCount(key);
continue;
}
snprintf(buf,sizeof(buf),"value:%Iu",j); WIN_PORT_FIX /* %lu -> %Iu */
val = createStringObject(buf,strlen(buf));
if (valsize==0)
val = createStringObject(buf,strlen(buf));
else {
int buflen = strlen(buf);
val = createStringObject(NULL,valsize);
memcpy(val->ptr, buf, valsize<=buflen? valsize: buflen);
}
dbAdd(c->db,key,val);
signalModifiedKey(c->db,key);
decrRefCount(key);
@@ -528,30 +558,6 @@ void debugCommand(client *c) {
stats = sdscat(stats,buf);
addReplyBulkSds(c,stats);
} else if (!strcasecmp(c->argv[1]->ptr,"jemalloc") && c->argc == 3) {
#if defined(USE_JEMALLOC)
if (!strcasecmp(c->argv[2]->ptr, "info")) {
sds info = sdsempty();
je_malloc_stats_print(inputCatSds, &info, NULL);
addReplyBulkSds(c, info);
} else if (!strcasecmp(c->argv[2]->ptr, "purge")) {
char tmp[32];
unsigned narenas = 0;
size_t sz = sizeof(unsigned);
if (!je_mallctl("arenas.narenas", &narenas, &sz, NULL, 0)) {
sprintf(tmp, "arena.%d.purge", narenas);
if (!je_mallctl(tmp, NULL, 0, NULL, 0)) {
addReply(c, shared.ok);
return;
}
}
addReplyError(c, "Error purging dirty pages");
} else {
addReplyErrorFormat(c, "Valid jemalloc debug fields: info, purge");
}
#else
addReplyErrorFormat(c, "jemalloc support not available");
#endif
} else {
addReplyErrorFormat(c, "Unknown DEBUG subcommand or wrong number of arguments for '%s'",
(char*)c->argv[1]->ptr);
@@ -560,7 +566,7 @@ void debugCommand(client *c) {
/* =========================== Crash handling ============================== */
void _serverAssert(char *estr, char *file, int line) {
void _serverAssert(const char *estr, const char *file, int line) {
bugReportStart();
serverLog(LL_WARNING,"=== ASSERTION FAILED ===");
serverLog(LL_WARNING,"==> %s:%d '%s' is not true",file,line,estr);
@@ -573,7 +579,7 @@ void _serverAssert(char *estr, char *file, int line) {
*((char*)-1) = 'x';
}
void _serverAssertPrintClientInfo(client *c) {
void _serverAssertPrintClientInfo(const client *c) {
int j;
bugReportStart();
@@ -597,7 +603,7 @@ void _serverAssertPrintClientInfo(client *c) {
}
}
void serverLogObjectDebugInfo(robj *o) {
void serverLogObjectDebugInfo(const robj *o) {
serverLog(LL_WARNING,"Object type: %d", o->type);
serverLog(LL_WARNING,"Object encoding: %d", o->encoding);
serverLog(LL_WARNING,"Object refcount: %d", o->refcount);
@@ -617,30 +623,36 @@ void serverLogObjectDebugInfo(robj *o) {
} else if (o->type == OBJ_ZSET) {
serverLog(LL_WARNING,"Sorted set size: %d", (int) zsetLength(o));
if (o->encoding == OBJ_ENCODING_SKIPLIST)
serverLog(LL_WARNING,"Skiplist level: %d", (int) ((zset*)o->ptr)->zsl->level);
serverLog(LL_WARNING,"Skiplist level: %d", (int) ((const zset*)o->ptr)->zsl->level);
}
}
void _serverAssertPrintObject(robj *o) {
void _serverAssertPrintObject(const robj *o) {
bugReportStart();
serverLog(LL_WARNING,"=== ASSERTION FAILED OBJECT CONTEXT ===");
serverLogObjectDebugInfo(o);
}
void _serverAssertWithInfo(client *c, robj *o, char *estr, char *file, int line) {
void _serverAssertWithInfo(const client *c, const robj *o, const char *estr, const char *file, int line) {
if (c) _serverAssertPrintClientInfo(c);
if (o) _serverAssertPrintObject(o);
_serverAssert(estr,file,line);
}
void _serverPanic(char *msg, char *file, int line) {
void _serverPanic(const char *file, int line, const char *msg, ...) {
va_list ap;
va_start(ap,msg);
char fmtmsg[256];
vsnprintf(fmtmsg,sizeof(fmtmsg),msg,ap);
va_end(ap);
bugReportStart();
serverLog(LL_WARNING,"------------------------------------------------");
#ifdef _WIN32
serverLog(LL_WARNING, "Fatal Error: %s #%s:%d", msg, file, line);
serverLog(LL_WARNING, "Fatal Error: %s #%s:%d", fmtmsg, file, line);
#else
serverLog(LL_WARNING,"!!! Software Failure. Press left mouse button to continue");
serverLog(LL_WARNING,"Guru Meditation: %s #%s:%d",msg,file,line);
serverLog(LL_WARNING,"Guru Meditation: %s #%s:%d",fmtmsg,file,line);
#endif
#ifdef HAVE_BACKTRACE
serverLog(LL_WARNING,"(forcing SIGSEGV in order to print the stack trace)");
@@ -684,6 +696,8 @@ static void *getMcontextEip(ucontext_t *uc) {
return (void*) uc->uc_mcontext.gregs[16]; /* Linux 64 */
#elif defined(__ia64__) /* Linux IA64 */
return (void*) uc->uc_mcontext.sc_ip;
#elif defined(__arm__) /* Linux ARM */
return (void*) uc->uc_mcontext.arm_pc;
#endif
#else
return NULL;
@@ -985,6 +999,32 @@ int memtest_test_linux_anonymous_maps(void) {
}
#endif
/* Scans the (assumed) x86 code starting at addr, for a max of `len`
* bytes, searching for E8 (callq) opcodes, and dumping the symbols
* and the call offset if they appear to be valid. */
void dumpX86Calls(void *addr, size_t len) {
size_t j;
unsigned char *p = addr;
Dl_info info;
/* Hash table to best-effort avoid printing the same symbol
* multiple times. */
PORT_ULONG ht[256] = {0};
if (len < 5) return;
for (j = 0; j < len-4; j++) {
if (p[j] != 0xE8) continue; /* Not an E8 CALL opcode. */
PORT_ULONG target = (PORT_ULONG)addr+j+5;
target += *((int32_t*)(p+j+1));
if (dladdr((void*)target, &info) != 0 && info.dli_sname != NULL) {
if (ht[target&0xff] != target) {
printf("Function at 0x%lx is %s\n",target,info.dli_sname);
ht[target&0xff] = target;
}
j += 4; /* Skip the 32 bit immediate. */
}
}
}
void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
ucontext_t *uc = (ucontext_t*) secret;
void *eip = getMcontextEip(uc);
@@ -1014,8 +1054,6 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
/* Log INFO and CLIENT LIST */
serverLogRaw(LL_WARNING|LL_RAW, "\n------ INFO OUTPUT ------\n");
infostring = genRedisInfoString("all");
infostring = sdscatprintf(infostring, "hash_init_value: %u\n",
dictGetHashFunctionSeed());
serverLogRaw(LL_WARNING|LL_RAW, infostring);
serverLogRaw(LL_WARNING|LL_RAW, "\n------ CLIENT LIST OUTPUT ------\n");
clients = getAllClientsInfoString();
@@ -1035,19 +1073,49 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
bioKillThreads();
if (memtest_test_linux_anonymous_maps()) {
serverLogRaw(LL_WARNING|LL_RAW,
"!!! MEMORY ERROR DETECTED! Check your memory ASAP !!!");
"!!! MEMORY ERROR DETECTED! Check your memory ASAP !!!\n");
} else {
serverLogRaw(LL_WARNING|LL_RAW,
"Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.");
"Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.\n");
}
#endif
if (eip != NULL) {
Dl_info info;
if (dladdr(eip, &info) != 0) {
serverLog(LL_WARNING|LL_RAW,
"\n------ DUMPING CODE AROUND EIP ------\n"
"Symbol: %s (base: %p)\n"
"Module: %s (base %p)\n"
"$ xxd -r -p /tmp/dump.hex /tmp/dump.bin\n"
"$ objdump --adjust-vma=%p -D -b binary -m i386:x86-64 /tmp/dump.bin\n"
"------\n",
info.dli_sname, info.dli_saddr, info.dli_fname, info.dli_fbase,
info.dli_saddr);
size_t len = (PORT_LONG)eip - (PORT_LONG)info.dli_saddr;
PORT_ULONG sz = sysconf(_SC_PAGESIZE);
if (len < 1<<13) { /* we don't have functions over 8k (verified) */
/* Find the address of the next page, which is our "safety"
* limit when dumping. Then try to dump just 128 bytes more
* than EIP if there is room, or stop sooner. */
PORT_ULONG next = ((PORT_ULONG)eip + sz) & ~(sz-1);
PORT_ULONG end = (PORT_ULONG)eip + 128;
if (end > next) end = next;
len = end - (PORT_ULONG)info.dli_saddr;
serverLogHexDump(LL_WARNING, "dump of function",
info.dli_saddr ,len);
dumpX86Calls(info.dli_saddr,len);
}
}
}
serverLogRaw(LL_WARNING|LL_RAW,
"\n=== REDIS BUG REPORT END. Make sure to include from START to END. ===\n\n"
" Please report the crash by opening an issue on github:\n\n"
" http://github.com/antirez/redis/issues\n\n"
" Suspect RAM error? Use redis-server --test-memory to verify it.\n\n"
);
/* free(messages); Don't call free() with possibly corrupted memory. */
if (server.daemonize && server.supervised == 0) unlink(server.pidfile);
@@ -1068,7 +1136,7 @@ void serverLogHexDump(int level, char *descr, void *value, size_t len) {
unsigned char *v = value;
char charset[] = "0123456789abcdef";
serverLog(level,"%s (hexdump):", descr);
serverLog(level,"%s (hexdump of %Iu bytes):", descr, len);
b = buf;
while(len) {
b[0] = charset[(*v)>>4];
src/debugmacro.h
+41
View File
@@ -0,0 +1,41 @@
/* This file contains debugging macros to be used when investigating issues.
*
* -----------------------------------------------------------------------------
*
* Copyright (c) 2016, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#define D(...) \
do { \
FILE *fp = fopen("/tmp/log.txt","a"); \
fprintf(fp,"%s:%s:%d:\t", __FILE__, __func__, __LINE__); \
fprintf(fp,__VA_ARGS__); \
fprintf(fp,"\n"); \
fclose(fp); \
} while (0);
src/defrag.c
+579
View File
@@ -0,0 +1,579 @@
/*
* Active memory defragmentation
* Try to find key / value allocations that need to be re-allocated in order
* to reduce external fragmentation.
* We do that by scanning the keyspace and for each pointer we have, we can try to
* ask the allocator if moving it to a new address will help reduce fragmentation.
*
* Copyright (c) 2017, Oran Agra
* Copyright (c) 2017, Redis Labs, Inc
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "server.h"
#include <time.h>
#include <assert.h>
#include <stddef.h>
#ifdef HAVE_DEFRAG
/* this method was added to jemalloc in order to help us understand which
* pointers are worthwhile moving and which aren't */
int je_get_defrag_hint(void* ptr, int *bin_util, int *run_util);
/* Defrag helper for generic allocations.
*
* returns NULL in case the allocatoin wasn't moved.
* when it returns a non-null value, the old pointer was already released
* and should NOT be accessed. */
void* activeDefragAlloc(void *ptr) {
int bin_util, run_util;
size_t size;
void *newptr;
if(!je_get_defrag_hint(ptr, &bin_util, &run_util)) {
server.stat_active_defrag_misses++;
return NULL;
}
/* if this run is more utilized than the average utilization in this bin
* (or it is full), skip it. This will eventually move all the allocations
* from relatively empty runs into relatively full runs. */
if (run_util > bin_util || run_util == 1<<16) {
server.stat_active_defrag_misses++;
return NULL;
}
/* move this allocation to a new allocation.
* make sure not to use the thread cache. so that we don't get back the same
* pointers we try to free */
size = zmalloc_size(ptr);
newptr = zmalloc_no_tcache(size);
memcpy(newptr, ptr, size);
zfree_no_tcache(ptr);
return newptr;
}
/*Defrag helper for sds strings
*
* returns NULL in case the allocatoin wasn't moved.
* when it returns a non-null value, the old pointer was already released
* and should NOT be accessed. */
sds activeDefragSds(sds sdsptr) {
void* ptr = sdsAllocPtr(sdsptr);
void* newptr = activeDefragAlloc(ptr);
if (newptr) {
size_t offset = sdsptr - (char*)ptr;
sdsptr = (char*)newptr + offset;
return sdsptr;
}
return NULL;
}
/* Defrag helper for robj and/or string objects
*
* returns NULL in case the allocatoin wasn't moved.
* when it returns a non-null value, the old pointer was already released
* and should NOT be accessed. */
robj *activeDefragStringOb(robj* ob, int *defragged) {
robj *ret = NULL;
if (ob->refcount!=1)
return NULL;
/* try to defrag robj (only if not an EMBSTR type (handled below). */
if (ob->type!=OBJ_STRING || ob->encoding!=OBJ_ENCODING_EMBSTR) {
if ((ret = activeDefragAlloc(ob))) {
ob = ret;
(*defragged)++;
}
}
/* try to defrag string object */
if (ob->type == OBJ_STRING) {
if(ob->encoding==OBJ_ENCODING_RAW) {
sds newsds = activeDefragSds((sds)ob->ptr);
if (newsds) {
ob->ptr = newsds;
(*defragged)++;
}
} else if (ob->encoding==OBJ_ENCODING_EMBSTR) {
/* The sds is embedded in the object allocation, calculate the
* offset and update the pointer in the new allocation. */
PORT_LONG ofs = (intptr_t)ob->ptr - (intptr_t)ob;
if ((ret = activeDefragAlloc(ob))) {
ret->ptr = (void*)((intptr_t)ret + ofs);
(*defragged)++;
}
} else if (ob->encoding!=OBJ_ENCODING_INT) {
serverPanic("Unknown string encoding");
}
}
return ret;
}
/* Defrag helper for dictEntries to be used during dict iteration (called on
* each step). Teturns a stat of how many pointers were moved. */
int dictIterDefragEntry(dictIterator *iter) {
/* This function is a little bit dirty since it messes with the internals
* of the dict and it's iterator, but the benefit is that it is very easy
* to use, and require no other chagnes in the dict. */
int defragged = 0;
dictht *ht;
/* Handle the next entry (if there is one), and update the pointer in the
* current entry. */
if (iter->nextEntry) {
dictEntry *newde = activeDefragAlloc(iter->nextEntry);
if (newde) {
defragged++;
iter->nextEntry = newde;
iter->entry->next = newde;
}
}
/* handle the case of the first entry in the hash bucket. */
ht = &iter->d->ht[iter->table];
if (ht->table[iter->index] == iter->entry) {
dictEntry *newde = activeDefragAlloc(iter->entry);
if (newde) {
iter->entry = newde;
ht->table[iter->index] = newde;
defragged++;
}
}
return defragged;
}
/* Defrag helper for dict main allocations (dict struct, and hash tables).
* receives a pointer to the dict* and implicitly updates it when the dict
* struct itself was moved. Returns a stat of how many pointers were moved. */
int dictDefragTables(dict** dictRef) {
dict *d = *dictRef;
dictEntry **newtable;
int defragged = 0;
/* handle the dict struct */
dict *newd = activeDefragAlloc(d);
if (newd)
defragged++, *dictRef = d = newd;
/* handle the first hash table */
newtable = activeDefragAlloc(d->ht[0].table);
if (newtable)
defragged++, d->ht[0].table = newtable;
/* handle the second hash table */
if (d->ht[1].table) {
newtable = activeDefragAlloc(d->ht[1].table);
if (newtable)
defragged++, d->ht[1].table = newtable;
}
return defragged;
}
/* Internal function used by zslDefrag */
void zslUpdateNode(zskiplist *zsl, zskiplistNode *oldnode, zskiplistNode *newnode, zskiplistNode **update) {
int i;
for (i = 0; i < zsl->level; i++) {
if (update[i]->level[i].forward == oldnode)
update[i]->level[i].forward = newnode;
}
serverAssert(zsl->header!=oldnode);
if (newnode->level[0].forward) {
serverAssert(newnode->level[0].forward->backward==oldnode);
newnode->level[0].forward->backward = newnode;
} else {
serverAssert(zsl->tail==oldnode);
zsl->tail = newnode;
}
}
/* Defrag helper for sorted set.
* Update the robj pointer, defrag the skiplist struct and return the new score
* reference. We may not access oldele pointer (not even the pointer stored in
* the skiplist), as it was already freed. Newele may be null, in which case we
* only need to defrag the skiplist, but not update the obj pointer.
* When return value is non-NULL, it is the score reference that must be updated
* in the dict record. */
double *zslDefrag(zskiplist *zsl, double score, sds oldele, sds newele) {
zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x, *newx;
int i;
sds ele = newele? newele: oldele;
/* find the skiplist node referring to the object that was moved,
* and all pointers that need to be updated if we'll end up moving the skiplist node. */
x = zsl->header;
for (i = zsl->level-1; i >= 0; i--) {
while (x->level[i].forward &&
x->level[i].forward->ele != oldele && /* make sure not to access the
->obj pointer if it matches
oldele */
(x->level[i].forward->score < score ||
(x->level[i].forward->score == score &&
sdscmp(x->level[i].forward->ele,ele) < 0)))
x = x->level[i].forward;
update[i] = x;
}
/* update the robj pointer inside the skip list record. */
x = x->level[0].forward;
serverAssert(x && score == x->score && x->ele==oldele);
if (newele)
x->ele = newele;
/* try to defrag the skiplist record itself */
newx = activeDefragAlloc(x);
if (newx) {
zslUpdateNode(zsl, x, newx, update);
return &newx->score;
}
return NULL;
}
/* Utility function that replaces an old key pointer in the dictionary with a
* new pointer. Additionally, we try to defrag the dictEntry in that dict.
* Oldkey mey be a dead pointer and should not be accessed (we get a
* pre-calculated hash value). Newkey may be null if the key pointer wasn't
* moved. Return value is the the dictEntry if found, or NULL if not found.
* NOTE: this is very ugly code, but it let's us avoid the complication of
* doing a scan on another dict. */
dictEntry* replaceSateliteDictKeyPtrAndOrDefragDictEntry(dict *d, sds oldkey, sds newkey, unsigned int hash, int *defragged) {
dictEntry **deref = dictFindEntryRefByPtrAndHash(d, oldkey, hash);
if (deref) {
dictEntry *de = *deref;
dictEntry *newde = activeDefragAlloc(de);
if (newde) {
de = *deref = newde;
(*defragged)++;
}
if (newkey)
de->key = newkey;
return de;
}
return NULL;
}
/* for each key we scan in the main dict, this function will attempt to defrag
* all the various pointers it has. Returns a stat of how many pointers were
* moved. */
int defragKey(redisDb *db, dictEntry *de) {
sds keysds = dictGetKey(de);
robj *newob, *ob;
unsigned char *newzl;
dict *d;
dictIterator *di;
int defragged = 0;
sds newsds;
/* Try to defrag the key name. */
newsds = activeDefragSds(keysds);
if (newsds)
defragged++, de->key = newsds;
if (dictSize(db->expires)) {
/* Dirty code:
* I can't search in db->expires for that key after i already released
* the pointer it holds it won't be able to do the string compare */
unsigned int hash = dictGetHash(db->dict, de->key);
replaceSateliteDictKeyPtrAndOrDefragDictEntry(db->expires, keysds, newsds, hash, &defragged);
}
/* Try to defrag robj and / or string value. */
ob = dictGetVal(de);
if ((newob = activeDefragStringOb(ob, &defragged))) {
de->v.val = newob;
ob = newob;
}
if (ob->type == OBJ_STRING) {
/* Already handled in activeDefragStringOb. */
} else if (ob->type == OBJ_LIST) {
if (ob->encoding == OBJ_ENCODING_QUICKLIST) {
quicklist *ql = ob->ptr, *newql;
quicklistNode *node = ql->head, *newnode;
if ((newql = activeDefragAlloc(ql)))
defragged++, ob->ptr = ql = newql;
while (node) {
if ((newnode = activeDefragAlloc(node))) {
if (newnode->prev)
newnode->prev->next = newnode;
else
ql->head = newnode;
if (newnode->next)
newnode->next->prev = newnode;
else
ql->tail = newnode;
node = newnode;
defragged++;
}
if ((newzl = activeDefragAlloc(node->zl)))
defragged++, node->zl = newzl;
node = node->next;
}
} else if (ob->encoding == OBJ_ENCODING_ZIPLIST) {
if ((newzl = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newzl;
} else {
serverPanic("Unknown list encoding");
}
} else if (ob->type == OBJ_SET) {
if (ob->encoding == OBJ_ENCODING_HT) {
d = ob->ptr;
di = dictGetIterator(d);
while((de = dictNext(di)) != NULL) {
sds sdsele = dictGetKey(de);
if ((newsds = activeDefragSds(sdsele)))
defragged++, de->key = newsds;
defragged += dictIterDefragEntry(di);
}
dictReleaseIterator(di);
dictDefragTables((dict**)&ob->ptr);
} else if (ob->encoding == OBJ_ENCODING_INTSET) {
intset *is = ob->ptr;
intset *newis = activeDefragAlloc(is);
if (newis)
defragged++, ob->ptr = newis;
} else {
serverPanic("Unknown set encoding");
}
} else if (ob->type == OBJ_ZSET) {
if (ob->encoding == OBJ_ENCODING_ZIPLIST) {
if ((newzl = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newzl;
} else if (ob->encoding == OBJ_ENCODING_SKIPLIST) {
zset *zs = (zset*)ob->ptr;
zset *newzs;
zskiplist *newzsl;
struct zskiplistNode *newheader;
if ((newzs = activeDefragAlloc(zs)))
defragged++, ob->ptr = zs = newzs;
if ((newzsl = activeDefragAlloc(zs->zsl)))
defragged++, zs->zsl = newzsl;
if ((newheader = activeDefragAlloc(zs->zsl->header)))
defragged++, zs->zsl->header = newheader;
d = zs->dict;
di = dictGetIterator(d);
while((de = dictNext(di)) != NULL) {
double* newscore;
sds sdsele = dictGetKey(de);
if ((newsds = activeDefragSds(sdsele)))
defragged++, de->key = newsds;
newscore = zslDefrag(zs->zsl, *(double*)dictGetVal(de), sdsele, newsds);
if (newscore) {
dictSetVal(d, de, newscore);
defragged++;
}
defragged += dictIterDefragEntry(di);
}
dictReleaseIterator(di);
dictDefragTables(&zs->dict);
} else {
serverPanic("Unknown sorted set encoding");
}
} else if (ob->type == OBJ_HASH) {
if (ob->encoding == OBJ_ENCODING_ZIPLIST) {
if ((newzl = activeDefragAlloc(ob->ptr)))
defragged++, ob->ptr = newzl;
} else if (ob->encoding == OBJ_ENCODING_HT) {
d = ob->ptr;
di = dictGetIterator(d);
while((de = dictNext(di)) != NULL) {
sds sdsele = dictGetKey(de);
if ((newsds = activeDefragSds(sdsele)))
defragged++, de->key = newsds;
sdsele = dictGetVal(de);
if ((newsds = activeDefragSds(sdsele)))
defragged++, de->v.val = newsds;
defragged += dictIterDefragEntry(di);
}
dictReleaseIterator(di);
dictDefragTables((dict**)&ob->ptr);
} else {
serverPanic("Unknown hash encoding");
}
} else if (ob->type == OBJ_MODULE) {
/* Currently defragmenting modules private data types
* is not supported. */
} else {
serverPanic("Unknown object type");
}
return defragged;
}
/* Defrag scan callback for the main db dictionary. */
void defragScanCallback(void *privdata, const dictEntry *de) {
int defragged = defragKey((redisDb*)privdata, (dictEntry*)de);
server.stat_active_defrag_hits += defragged;
if(defragged)
server.stat_active_defrag_key_hits++;
else
server.stat_active_defrag_key_misses++;
}
/* Defrag scan callback for for each hash table bicket,
* used in order to defrag the dictEntry allocations. */
void defragDictBucketCallback(void *privdata, dictEntry **bucketref) {
UNUSED(privdata);
while(*bucketref) {
dictEntry *de = *bucketref, *newde;
if ((newde = activeDefragAlloc(de))) {
*bucketref = newde;
}
bucketref = &(*bucketref)->next;
}
}
/* Utility function to get the fragmentation ratio from jemalloc.
* It is critical to do that by comparing only heap maps that belown to
* jemalloc, and skip ones the jemalloc keeps as spare. Since we use this
* fragmentation ratio in order to decide if a defrag action should be taken
* or not, a false detection can cause the defragmenter to waste a lot of CPU
* without the possibility of getting any results. */
float getAllocatorFragmentation(size_t *out_frag_bytes) {
size_t epoch = 1, allocated = 0, resident = 0, active = 0, sz = sizeof(size_t);
/* Update the statistics cached by mallctl. */
je_mallctl("epoch", &epoch, &sz, &epoch, sz);
/* Unlike RSS, this does not include RSS from shared libraries and other non
* heap mappings. */
je_mallctl("stats.resident", &resident, &sz, NULL, 0);
/* Unlike resident, this doesn't not include the pages jemalloc reserves
* for re-use (purge will clean that). */
je_mallctl("stats.active", &active, &sz, NULL, 0);
/* Unlike zmalloc_used_memory, this matches the stats.resident by taking
* into account all allocations done by this process (not only zmalloc). */
je_mallctl("stats.allocated", &allocated, &sz, NULL, 0);
float frag_pct = ((float)active / allocated)*100 - 100;
size_t frag_bytes = active - allocated;
float rss_pct = ((float)resident / allocated)*100 - 100;
size_t rss_bytes = resident - allocated;
if(out_frag_bytes)
*out_frag_bytes = frag_bytes;
serverLog(LL_DEBUG,
"allocated=%zu, active=%zu, resident=%zu, frag=%.0f%% (%.0f%% rss), frag_bytes=%zu (%zu%% rss)",
allocated, active, resident, frag_pct, rss_pct, frag_bytes, rss_bytes);
return frag_pct;
}
#define INTERPOLATE(x, x1, x2, y1, y2) ( (y1) + ((x)-(x1)) * ((y2)-(y1)) / ((x2)-(x1)) )
#define LIMIT(y, min, max) ((y)<(min)? min: ((y)>(max)? max: (y)))
/* Perform incremental defragmentation work from the serverCron.
* This works in a similar way to activeExpireCycle, in the sense that
* we do incremental work across calls. */
void activeDefragCycle(void) {
static int current_db = -1;
static PORT_ULONG cursor = 0;
static redisDb *db = NULL;
static PORT_LONGLONG start_scan, start_stat;
unsigned int iterations = 0;
PORT_ULONGLONG defragged = server.stat_active_defrag_hits;
PORT_LONGLONG start, timelimit;
if (server.aof_child_pid!=-1 || server.rdb_child_pid!=-1)
return; /* Defragging memory while there's a fork will just do damage. */
/* Once a second, check if we the fragmentation justfies starting a scan
* or making it more aggressive. */
run_with_period(1000) {
size_t frag_bytes;
float frag_pct = getAllocatorFragmentation(&frag_bytes);
/* If we're not already running, and below the threshold, exit. */
if (!server.active_defrag_running) {
if(frag_pct < server.active_defrag_threshold_lower || frag_bytes < server.active_defrag_ignore_bytes)
return;
}
/* Calculate the adaptive aggressiveness of the defrag */
int cpu_pct = INTERPOLATE(frag_pct,
server.active_defrag_threshold_lower,
server.active_defrag_threshold_upper,
server.active_defrag_cycle_min,
server.active_defrag_cycle_max);
cpu_pct = LIMIT(cpu_pct,
server.active_defrag_cycle_min,
server.active_defrag_cycle_max);
/* We allow increasing the aggressiveness during a scan, but don't
* reduce it. */
if (!server.active_defrag_running ||
cpu_pct > server.active_defrag_running)
{
server.active_defrag_running = cpu_pct;
serverLog(LL_VERBOSE,
"Starting active defrag, frag=%.0f%%, frag_bytes=%zu, cpu=%d%%",
frag_pct, frag_bytes, cpu_pct);
}
}
if (!server.active_defrag_running)
return;
/* See activeExpireCycle for how timelimit is handled. */
start = ustime();
timelimit = 1000000*server.active_defrag_running/server.hz/100;
if (timelimit <= 0) timelimit = 1;
do {
if (!cursor) {
/* Move on to next database, and stop if we reached the last one. */
if (++current_db >= server.dbnum) {
PORT_LONGLONG now = ustime();
size_t frag_bytes;
float frag_pct = getAllocatorFragmentation(&frag_bytes);
serverLog(LL_VERBOSE,
"Active defrag done in %dms, reallocated=%d, frag=%.0f%%, frag_bytes=%zu",
(int)((now - start_scan)/1000), (int)(server.stat_active_defrag_hits - start_stat), frag_pct, frag_bytes);
start_scan = now;
current_db = -1;
cursor = 0;
db = NULL;
server.active_defrag_running = 0;
return;
}
else if (current_db==0) {
/* Start a scan from the first database. */
start_scan = ustime();
start_stat = server.stat_active_defrag_hits;
}
db = &server.db[current_db];
cursor = 0;
}
do {
cursor = dictScan(db->dict, cursor, defragScanCallback, defragDictBucketCallback, db);
/* Once in 16 scan iterations, or 1000 pointer reallocations
* (if we have a lot of pointers in one hash bucket), check if we
* reached the tiem limit. */
if (cursor && (++iterations > 16 || server.stat_active_defrag_hits - defragged > 1000)) {
if ((ustime() - start) > timelimit) {
return;
}
iterations = 0;
defragged = server.stat_active_defrag_hits;
}
} while(cursor);
} while(1);
}
#else /* HAVE_DEFRAG */
void activeDefragCycle(void) {
/* Not implemented yet. */
}
#endif
src/dict.c
+260 -121
View File
@@ -43,17 +43,21 @@ extern BOOL g_IsForkedProcess;
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <stdarg.h>
#include <limits.h>
#ifndef _WIN32
#include <sys/time.h>
#endif
#include <ctype.h>
#include "dict.h"
#include "zmalloc.h"
#ifndef DICT_BENCHMARK_MAIN
#include "redisassert.h"
#else
#include <assert.h>
#endif
/* Using dictEnableResize() / dictDisableResize() we make possible to
* enable/disable resizing of the hash table as needed. This is very important
@@ -70,94 +74,33 @@ static unsigned int dict_force_resize_ratio = 5;
static int _dictExpandIfNeeded(dict *ht);
static PORT_ULONG _dictNextPower(PORT_ULONG size);
static int _dictKeyIndex(dict *ht, const void *key);
static int _dictKeyIndex(dict *ht, const void *key, unsigned int hash, dictEntry **existing);
static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
/* -------------------------- hash functions -------------------------------- */
/* Thomas Wang's 32 bit Mix Function */
unsigned int dictIntHashFunction(unsigned int key)
{
key += ~(key << 15);
key ^= (key >> 10);
key += (key << 3);
key ^= (key >> 6);
key += ~(key << 11);
key ^= (key >> 16);
return key;
static uint8_t dict_hash_function_seed[16];
void dictSetHashFunctionSeed(uint8_t *seed) {
memcpy(dict_hash_function_seed,seed,sizeof(dict_hash_function_seed));
}
static uint32_t dict_hash_function_seed = 5381;
void dictSetHashFunctionSeed(uint32_t seed) {
dict_hash_function_seed = seed;
}
uint32_t dictGetHashFunctionSeed(void) {
uint8_t *dictGetHashFunctionSeed(void) {
return dict_hash_function_seed;
}
/* MurmurHash2, by Austin Appleby
* Note - This code makes a few assumptions about how your machine behaves -
* 1. We can read a 4-byte value from any address without crashing
* 2. sizeof(int) == 4
*
* And it has a few limitations -
*
* 1. It will not work incrementally.
* 2. It will not produce the same results on little-endian and big-endian
* machines.
*/
unsigned int dictGenHashFunction(const void *key, int len) {
/* 'm' and 'r' are mixing constants generated offline.
They're not really 'magic', they just happen to work well. */
uint32_t seed = dict_hash_function_seed;
const uint32_t m = 0x5bd1e995;
const int r = 24;
/* The default hashing function uses SipHash implementation
* in siphash.c. */
/* Initialize the hash to a 'random' value */
uint32_t h = seed ^ len;
uint64_t siphash(const uint8_t *in, const size_t inlen, const uint8_t *k);
uint64_t siphash_nocase(const uint8_t *in, const size_t inlen, const uint8_t *k);
/* Mix 4 bytes at a time into the hash */
const unsigned char *data = (const unsigned char *)key;
while(len >= 4) {
uint32_t k = *(uint32_t*)data;
k *= m;
k ^= k >> r;
k *= m;
h *= m;
h ^= k;
data += 4;
len -= 4;
}
/* Handle the last few bytes of the input array */
switch(len) {
case 3: h ^= data[2] << 16;
case 2: h ^= data[1] << 8;
case 1: h ^= data[0]; h *= m;
};
/* Do a few final mixes of the hash to ensure the last few
* bytes are well-incorporated. */
h ^= h >> 13;
h *= m;
h ^= h >> 15;
return (unsigned int)h;
uint64_t dictGenHashFunction(const void *key, int len) {
return siphash(key,len,dict_hash_function_seed);
}
/* And a case insensitive hash function (based on djb hash) */
unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len) {
unsigned int hash = (unsigned int)dict_hash_function_seed;
while (len--)
hash = ((hash << 5) + hash) + (tolower(*buf++)); /* hash * 33 + c */
return hash;
uint64_t dictGenCaseHashFunction(const unsigned char *buf, int len) {
return siphash_nocase(buf,len,dict_hash_function_seed);
}
/* ----------------------------- API implementation ------------------------- */
@@ -338,29 +281,32 @@ static void _dictRehashStep(dict *d) {
/* Add an element to the target hash table */
int dictAdd(dict *d, void *key, void *val)
{
dictEntry *entry = dictAddRaw(d,key);
dictEntry *entry = dictAddRaw(d,key,NULL);
if (!entry) return DICT_ERR;
dictSetVal(d, entry, val);
return DICT_OK;
}
/* Low level add. This function adds the entry but instead of setting
* a value returns the dictEntry structure to the user, that will make
* sure to fill the value field as he wishes.
/* Low level add or find:
* This function adds the entry but instead of setting a value returns the
* dictEntry structure to the user, that will make sure to fill the value
* field as he wishes.
*
* This function is also directly exposed to the user API to be called
* mainly in order to store non-pointers inside the hash value, example:
*
* entry = dictAddRaw(dict,mykey);
* entry = dictAddRaw(dict,mykey,NULL);
* if (entry != NULL) dictSetSignedIntegerVal(entry,1000);
*
* Return values:
*
* If key already exists NULL is returned.
* If key already exists NULL is returned, and "*existing" is populated
* with the existing entry if existing is not NULL.
*
* If key was added, the hash entry is returned to be manipulated by the caller.
*/
dictEntry *dictAddRaw(dict *d, void *key)
dictEntry *dictAddRaw(dict *d, void *key, dictEntry **existing)
{
int index;
dictEntry *entry;
@@ -370,7 +316,7 @@ dictEntry *dictAddRaw(dict *d, void *key)
/* Get the index of the new element, or -1 if
* the element already exists. */
if ((index = _dictKeyIndex(d, key)) == -1)
if ((index = _dictKeyIndex(d, key, dictHashKey(d,key), existing)) == -1)
return NULL;
/* Allocate the memory and store the new entry.
@@ -388,51 +334,57 @@ dictEntry *dictAddRaw(dict *d, void *key)
return entry;
}
/* Add an element, discarding the old if the key already exists.
/* Add or Overwrite:
* Add an element, discarding the old value if the key already exists.
* Return 1 if the key was added from scratch, 0 if there was already an
* element with such key and dictReplace() just performed a value update
* operation. */
int dictReplace(dict *d, void *key, void *val)
{
dictEntry *entry, auxentry;
dictEntry *entry, *existing, auxentry;
/* Try to add the element. If the key
* does not exists dictAdd will suceed. */
if (dictAdd(d, key, val) == DICT_OK)
entry = dictAddRaw(d,key,&existing);
if (entry) {
dictSetVal(d, entry, val);
return 1;
/* It already exists, get the entry */
entry = dictFind(d, key);
}
/* Set the new value and free the old one. Note that it is important
* to do that in this order, as the value may just be exactly the same
* as the previous one. In this context, think to reference counting,
* you want to increment (set), and then decrement (free), and not the
* reverse. */
auxentry = *entry;
dictSetVal(d, entry, val);
auxentry = *existing;
dictSetVal(d, existing, val);
dictFreeVal(d, &auxentry);
return 0;
}
/* dictReplaceRaw() is simply a version of dictAddRaw() that always
/* Add or Find:
* dictAddOrFind() is simply a version of dictAddRaw() that always
* returns the hash entry of the specified key, even if the key already
* exists and can't be added (in that case the entry of the already
* existing key is returned.)
*
* See dictAddRaw() for more information. */
dictEntry *dictReplaceRaw(dict *d, void *key) {
dictEntry *entry = dictFind(d,key);
return entry ? entry : dictAddRaw(d,key);
dictEntry *dictAddOrFind(dict *d, void *key) {
dictEntry *entry, *existing;
entry = dictAddRaw(d,key,&existing);
return entry ? entry : existing;
}
/* Search and remove an element */
static int dictGenericDelete(dict *d, const void *key, int nofree)
{
/* Search and remove an element. This is an helper function for
* dictDelete() and dictUnlink(), please check the top comment
* of those functions. */
static dictEntry *dictGenericDelete(dict *d, const void *key, int nofree) {
unsigned int h, idx;
dictEntry *he, *prevHe;
int table;
if (d->ht[0].size == 0) return DICT_ERR; /* d->ht[0].table is NULL */
if (d->ht[0].used == 0 && d->ht[1].used == 0) return NULL;
if (dictIsRehashing(d)) _dictRehashStep(d);
h = dictHashKey(d, key);
@@ -450,27 +402,59 @@ static int dictGenericDelete(dict *d, const void *key, int nofree)
if (!nofree) {
dictFreeKey(d, he);
dictFreeVal(d, he);
zfree(he);
}
zfree(he);
d->ht[table].used--;
return DICT_OK;
return he;
}
prevHe = he;
he = he->next;
}
if (!dictIsRehashing(d)) break;
}
return DICT_ERR; /* not found */
return NULL; /* not found */
}
/* Remove an element, returning DICT_OK on success or DICT_ERR if the
* element was not found. */
int dictDelete(dict *ht, const void *key) {
return dictGenericDelete(ht,key,0);
return dictGenericDelete(ht,key,0) ? DICT_OK : DICT_ERR;
}
int dictDeleteNoFree(dict *ht, const void *key) {
/* Remove an element from the table, but without actually releasing
* the key, value and dictionary entry. The dictionary entry is returned
* if the element was found (and unlinked from the table), and the user
* should later call `dictFreeUnlinkedEntry()` with it in order to release it.
* Otherwise if the key is not found, NULL is returned.
*
* This function is useful when we want to remove something from the hash
* table but want to use its value before actually deleting the entry.
* Without this function the pattern would require two lookups:
*
* entry = dictFind(...);
* // Do something with entry
* dictDelete(dictionary,entry);
*
* Thanks to this function it is possible to avoid this, and use
* instead:
*
* entry = dictUnlink(dictionary,entry);
* // Do something with entry
* dictFreeUnlinkedEntry(entry); // <- This does not need to lookup again.
*/
dictEntry *dictUnlink(dict *ht, const void *key) {
return dictGenericDelete(ht,key,1);
}
/* You need to call this function to really free the entry after a call
* to dictUnlink(). It's safe to call this function with 'he' = NULL. */
void dictFreeUnlinkedEntry(dict *d, dictEntry *he) {
if (he == NULL) return;
dictFreeKey(d, he);
dictFreeVal(d, he);
zfree(he);
}
/* Destroy an entire dictionary */
int _dictClear(dict *d, dictht *ht, void(callback)(void *)) {
PORT_ULONG i;
@@ -869,10 +853,11 @@ static PORT_ULONG rev(PORT_ULONG v) {
PORT_ULONG dictScan(dict *d,
PORT_ULONG v,
dictScanFunction *fn,
dictScanBucketFunction* bucketfn,
void *privdata)
{
dictht *t0, *t1;
const dictEntry *de;
const dictEntry *de, *next;
PORT_ULONG m0, m1;
if (dictSize(d) == 0) return 0;
@@ -882,10 +867,12 @@ PORT_ULONG dictScan(dict *d,
m0 = (PORT_ULONG)t0->sizemask; WIN_PORT_FIX /* cast (PORT_ULONG) */
/* Emit entries at cursor */
if (bucketfn) bucketfn(privdata, &t0->table[v & m0]);
de = t0->table[v & m0];
while (de) {
next = de->next;
fn(privdata, de);
de = de->next;
de = next;
}
} else {
@@ -902,20 +889,24 @@ PORT_ULONG dictScan(dict *d,
m1 = (PORT_ULONG)t1->sizemask; WIN_PORT_FIX /* cast (PORT_ULONG) */
/* Emit entries at cursor */
if (bucketfn) bucketfn(privdata, &t0->table[v & m0]);
de = t0->table[v & m0];
while (de) {
next = de->next;
fn(privdata, de);
de = de->next;
de = next;
}
/* Iterate over indices in larger table that are the expansion
* of the index pointed to by the cursor in the smaller table */
do {
/* Emit entries at cursor */
if (bucketfn) bucketfn(privdata, &t1->table[v & m1]);
de = t1->table[v & m1];
while (de) {
next = de->next;
fn(privdata, de);
de = de->next;
de = next;
}
/* Increment bits not covered by the smaller mask */
@@ -976,27 +967,29 @@ static PORT_ULONG _dictNextPower(PORT_ULONG size)
/* Returns the index of a free slot that can be populated with
* a hash entry for the given 'key'.
* If the key already exists, -1 is returned.
* If the key already exists, -1 is returned
* and the optional output parameter may be filled.
*
* Note that if we are in the process of rehashing the hash table, the
* index is always returned in the context of the second (new) hash table. */
static int _dictKeyIndex(dict *d, const void *key)
static int _dictKeyIndex(dict *d, const void *key, unsigned int hash, dictEntry **existing)
{
unsigned int h, idx, table;
unsigned int idx, table;
dictEntry *he;
if (existing) *existing = NULL;
/* Expand the hash table if needed */
if (_dictExpandIfNeeded(d) == DICT_ERR)
return -1;
/* Compute the key hash value */
h = dictHashKey(d, key);
for (table = 0; table <= 1; table++) {
idx = h & d->ht[table].sizemask;
idx = hash & d->ht[table].sizemask;
/* Search if this slot does not already contain the given key */
he = d->ht[table].table[idx];
while(he) {
if (key==he->key || dictCompareKeys(d, key, he->key))
if (key==he->key || dictCompareKeys(d, key, he->key)) {
if (existing) *existing = he;
return -1;
}
he = he->next;
}
if (!dictIsRehashing(d)) break;
@@ -1019,6 +1012,35 @@ void dictDisableResize(void) {
dict_can_resize = 0;
}
unsigned int dictGetHash(dict *d, const void *key) {
return dictHashKey(d, key);
}
/* Finds the dictEntry reference by using pointer and pre-calculated hash.
* oldkey is a dead pointer and should not be accessed.
* the hash value should be provided using dictGetHash.
* no string / key comparison is performed.
* return value is the reference to the dictEntry if found, or NULL if not found. */
dictEntry **dictFindEntryRefByPtrAndHash(dict *d, const void *oldptr, unsigned int hash) {
dictEntry *he, **heref;
unsigned int idx, table;
if (d->ht[0].used + d->ht[1].used == 0) return NULL; /* dict is empty */
for (table = 0; table <= 1; table++) {
idx = hash & d->ht[table].sizemask;
heref = &d->ht[table].table[idx];
he = *heref;
while(he) {
if (oldptr==he->key)
return heref;
heref = &he->next;
he = *heref;
}
if (!dictIsRehashing(d)) return NULL;
}
return NULL;
}
/* ------------------------------- Debugging ---------------------------------*/
#define DICT_STATS_VECTLEN 50
@@ -1058,10 +1080,10 @@ size_t _dictGetStatsHt(char *buf, size_t bufsize, dictht *ht, int tableid) {
/* Generate human readable stats. */
l += snprintf(buf+l,bufsize-l,
"Hash table %d stats (%s):\n"
" table size: %ld\n"
" number of elements: %ld\n"
" different slots: %ld\n"
" max chain length: %ld\n"
" table size: %Id\n" WIN_PORT_FIX /* %ld -> %Id */
" number of elements: %Id\n" WIN_PORT_FIX /* %ld -> %Id */
" different slots: %Id\n" WIN_PORT_FIX /* %ld -> %Id */
" max chain length: %Id\n" WIN_PORT_FIX /* %ld -> %Id */
" avg chain length (counted): %.02f\n"
" avg chain length (computed): %.02f\n"
" Chain length distribution:\n",
@@ -1073,7 +1095,7 @@ size_t _dictGetStatsHt(char *buf, size_t bufsize, dictht *ht, int tableid) {
if (clvector[i] == 0) continue;
if (l >= bufsize) break;
l += snprintf(buf+l,bufsize-l,
" %s%ld: %ld (%.02f%%)\n",
" %s%Id: %Id (%.02f%%)\n", WIN_PORT_FIX /* %ld -> %Id */
(i == DICT_STATS_VECTLEN-1)?">= ":"",
i, clvector[i], ((float)clvector[i]/ht->size)*100);
}
@@ -1097,3 +1119,120 @@ void dictGetStats(char *buf, size_t bufsize, dict *d) {
/* Make sure there is a NULL term at the end. */
if (orig_bufsize) orig_buf[orig_bufsize-1] = '\0';
}
/* ------------------------------- Benchmark ---------------------------------*/
#ifdef DICT_BENCHMARK_MAIN
#include "sds.h"
uint64_t hashCallback(const void *key) {
return dictGenHashFunction((unsigned char*)key, sdslen((char*)key));
}
int compareCallback(void *privdata, const void *key1, const void *key2) {
int l1,l2;
DICT_NOTUSED(privdata);
l1 = sdslen((sds)key1);
l2 = sdslen((sds)key2);
if (l1 != l2) return 0;
return memcmp(key1, key2, l1) == 0;
}
void freeCallback(void *privdata, void *val) {
DICT_NOTUSED(privdata);
sdsfree(val);
}
dictType BenchmarkDictType = {
hashCallback,
NULL,
NULL,
compareCallback,
freeCallback,
NULL
};
#define start_benchmark() start = timeInMilliseconds()
#define end_benchmark(msg) do { \
elapsed = timeInMilliseconds()-start; \
printf(msg ": %ld items in %lld ms\n", count, elapsed); \
} while(0);
/* dict-benchmark [count] */
int main(int argc, char **argv) {
PORT_LONG j;
PORT_LONGLONG start, elapsed;
dict *dict = dictCreate(&BenchmarkDictType,NULL);
PORT_LONG count = 0;
if (argc == 2) {
count = strtol(argv[1],NULL,10);
} else {
count = 5000000;
}
start_benchmark();
for (j = 0; j < count; j++) {
int retval = dictAdd(dict,sdsfromlonglong(j),(void*)j);
assert(retval == DICT_OK);
}
end_benchmark("Inserting");
assert((PORT_LONG)dictSize(dict) == count);
/* Wait for rehashing. */
while (dictIsRehashing(dict)) {
dictRehashMilliseconds(dict,100);
}
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(j);
dictEntry *de = dictFind(dict,key);
assert(de != NULL);
sdsfree(key);
}
end_benchmark("Linear access of existing elements");
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(j);
dictEntry *de = dictFind(dict,key);
assert(de != NULL);
sdsfree(key);
}
end_benchmark("Linear access of existing elements (2nd round)");
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(rand() % count);
dictEntry *de = dictFind(dict,key);
assert(de != NULL);
sdsfree(key);
}
end_benchmark("Random access of existing elements");
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(rand() % count);
key[0] = 'X';
dictEntry *de = dictFind(dict,key);
assert(de == NULL);
sdsfree(key);
}
end_benchmark("Accessing missing");
start_benchmark();
for (j = 0; j < count; j++) {
sds key = sdsfromlonglong(j);
int retval = dictDelete(dict,key);
assert(retval == DICT_OK);
key[0] += 17; /* Change first number to letter. */
retval = dictAdd(dict,key,(void*)j);
assert(retval == DICT_OK);
}
end_benchmark("Removing and adding");
}
#endif
src/dict.h
+21 -17
View File
@@ -59,7 +59,7 @@ typedef struct dictEntry {
} dictEntry;
typedef struct dictType {
unsigned int (*hashFunction)(const void *key);
uint64_t (*hashFunction)(const void *key);
void *(*keyDup)(void *privdata, const void *key);
void *(*valDup)(void *privdata, const void *obj);
int (*keyCompare)(void *privdata, const void *key1, const void *key2);
@@ -81,7 +81,7 @@ typedef struct dict {
void *privdata;
dictht ht[2];
PORT_LONG rehashidx; /* rehashing not in progress if rehashidx == -1 */
int iterators; /* number of iterators currently running */
PORT_ULONG iterators; /* number of iterators currently running */
} dict;
/* If safe is set to 1 this is a safe iterator, that means, you can call
@@ -98,6 +98,7 @@ typedef struct dictIterator {
} dictIterator;
typedef void (dictScanFunction)(void *privdata, const dictEntry *de);
typedef void (dictScanBucketFunction)(void *privdata, dictEntry **bucketref);
/* This is the initial size of every hash table */
#define DICT_HT_INITIAL_SIZE 4
@@ -109,19 +110,19 @@ typedef void (dictScanFunction)(void *privdata, const dictEntry *de);
#define dictSetVal(d, entry, _val_) do { \
if ((d)->type->valDup) \
entry->v.val = (d)->type->valDup((d)->privdata, _val_); \
(entry)->v.val = (d)->type->valDup((d)->privdata, _val_); \
else \
entry->v.val = (_val_); \
(entry)->v.val = (_val_); \
} while(0)
#define dictSetSignedIntegerVal(entry, _val_) \
do { entry->v.s64 = _val_; } while(0)
do { (entry)->v.s64 = _val_; } while(0)
#define dictSetUnsignedIntegerVal(entry, _val_) \
do { entry->v.u64 = _val_; } while(0)
do { (entry)->v.u64 = _val_; } while(0)
#define dictSetDoubleVal(entry, _val_) \
do { entry->v.d = _val_; } while(0)
do { (entry)->v.d = _val_; } while(0)
#define dictFreeKey(d, entry) \
if ((d)->type->keyDestructor) \
@@ -129,9 +130,9 @@ typedef void (dictScanFunction)(void *privdata, const dictEntry *de);
#define dictSetKey(d, entry, _key_) do { \
if ((d)->type->keyDup) \
entry->key = (d)->type->keyDup((d)->privdata, _key_); \
(entry)->key = (d)->type->keyDup((d)->privdata, _key_); \
else \
entry->key = (_key_); \
(entry)->key = (_key_); \
} while(0)
#define dictCompareKeys(d, key1, key2) \
@@ -153,11 +154,12 @@ typedef void (dictScanFunction)(void *privdata, const dictEntry *de);
dict *dictCreate(dictType *type, void *privDataPtr);
int dictExpand(dict *d, PORT_ULONG size);
int dictAdd(dict *d, void *key, void *val);
dictEntry *dictAddRaw(dict *d, void *key);
dictEntry *dictAddRaw(dict *d, void *key, dictEntry **existing);
dictEntry *dictAddOrFind(dict *d, void *key);
int dictReplace(dict *d, void *key, void *val);
dictEntry *dictReplaceRaw(dict *d, void *key);
int dictDelete(dict *d, const void *key);
int dictDeleteNoFree(dict *d, const void *key);
dictEntry *dictUnlink(dict *ht, const void *key);
void dictFreeUnlinkedEntry(dict *d, dictEntry *he);
void dictRelease(dict *d);
dictEntry * dictFind(dict *d, const void *key);
void *dictFetchValue(dict *d, const void *key);
@@ -169,16 +171,18 @@ void dictReleaseIterator(dictIterator *iter);
dictEntry *dictGetRandomKey(dict *d);
unsigned int dictGetSomeKeys(dict *d, dictEntry **des, unsigned int count);
void dictGetStats(char *buf, size_t bufsize, dict *d);
unsigned int dictGenHashFunction(const void *key, int len);
unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len);
uint64_t dictGenHashFunction(const void *key, int len);
uint64_t dictGenCaseHashFunction(const unsigned char *buf, int len);
void dictEmpty(dict *d, void(callback)(void*));
void dictEnableResize(void);
void dictDisableResize(void);
int dictRehash(dict *d, int n);
int dictRehashMilliseconds(dict *d, int ms);
void dictSetHashFunctionSeed(unsigned int initval);
unsigned int dictGetHashFunctionSeed(void);
PORT_ULONG dictScan(dict *d, PORT_ULONG v, dictScanFunction *fn, void *privdata);
void dictSetHashFunctionSeed(uint8_t *seed);
uint8_t *dictGetHashFunctionSeed(void);
PORT_ULONG dictScan(dict *d, PORT_ULONG v, dictScanFunction *fn, dictScanBucketFunction *bucketfn, void *privdata);
unsigned int dictGetHash(dict *d, const void *key);
dictEntry **dictFindEntryRefByPtrAndHash(dict *d, const void *oldptr, unsigned int hash);
/* Hash table types */
extern dictType dictTypeHeapStringCopyKey;
src/evict.c
+575
View File
@@ -0,0 +1,575 @@
/* Maxmemory directive handling (LRU eviction and other policies).
*
* ----------------------------------------------------------------------------
*
* Copyright (c) 2009-2016, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "server.h"
#include "bio.h"
#include "atomicvar.h"
/* ----------------------------------------------------------------------------
* Data structures
* --------------------------------------------------------------------------*/
/* To improve the quality of the LRU approximation we take a set of keys
* that are good candidate for eviction across freeMemoryIfNeeded() calls.
*
* Entries inside the eviciton pool are taken ordered by idle time, putting
* greater idle times to the right (ascending order).
*
* When an LFU policy is used instead, a reverse frequency indication is used
* instead of the idle time, so that we still evict by larger value (larger
* inverse frequency means to evict keys with the least frequent accesses).
*
* Empty entries have the key pointer set to NULL. */
#define EVPOOL_SIZE 16
#define EVPOOL_CACHED_SDS_SIZE 255
struct evictionPoolEntry {
PORT_ULONGLONG idle; /* Object idle time (inverse frequency for LFU) */
sds key; /* Key name. */
sds cached; /* Cached SDS object for key name. */
int dbid; /* Key DB number. */
};
static struct evictionPoolEntry *EvictionPoolLRU;
PORT_ULONG LFUDecrAndReturn(robj *o);
/* ----------------------------------------------------------------------------
* Implementation of eviction, aging and LRU
* --------------------------------------------------------------------------*/
/* Return the LRU clock, based on the clock resolution. This is a time
* in a reduced-bits format that can be used to set and check the
* object->lru field of redisObject structures. */
unsigned int getLRUClock(void) {
return (mstime()/LRU_CLOCK_RESOLUTION) & LRU_CLOCK_MAX;
}
/* This function is used to obtain the current LRU clock.
* If the current resolution is lower than the frequency we refresh the
* LRU clock (as it should be in production servers) we return the
* precomputed value, otherwise we need to resort to a system call. */
unsigned int LRU_CLOCK(void) {
unsigned int lruclock;
if (1000/server.hz <= LRU_CLOCK_RESOLUTION) {
atomicGet(server.lruclock,lruclock);
} else {
lruclock = getLRUClock();
}
return lruclock;
}
/* Given an object returns the min number of milliseconds the object was never
* requested, using an approximated LRU algorithm. */
PORT_ULONGLONG estimateObjectIdleTime(robj *o) {
PORT_ULONGLONG lruclock = LRU_CLOCK();
if (lruclock >= o->lru) {
return (lruclock - o->lru) * LRU_CLOCK_RESOLUTION;
} else {
return (lruclock + (LRU_CLOCK_MAX - o->lru)) *
LRU_CLOCK_RESOLUTION;
}
}
/* freeMemoryIfNeeded() gets called when 'maxmemory' is set on the config
* file to limit the max memory used by the server, before processing a
* command.
*
* The goal of the function is to free enough memory to keep Redis under the
* configured memory limit.
*
* The function starts calculating how many bytes should be freed to keep
* Redis under the limit, and enters a loop selecting the best keys to
* evict accordingly to the configured policy.
*
* If all the bytes needed to return back under the limit were freed the
* function returns C_OK, otherwise C_ERR is returned, and the caller
* should block the execution of commands that will result in more memory
* used by the server.
*
* ------------------------------------------------------------------------
*
* LRU approximation algorithm
*
* Redis uses an approximation of the LRU algorithm that runs in constant
* memory. Every time there is a key to expire, we sample N keys (with
* N very small, usually in around 5) to populate a pool of best keys to
* evict of M keys (the pool size is defined by EVPOOL_SIZE).
*
* The N keys sampled are added in the pool of good keys to expire (the one
* with an old access time) if they are better than one of the current keys
* in the pool.
*
* After the pool is populated, the best key we have in the pool is expired.
* However note that we don't remove keys from the pool when they are deleted
* so the pool may contain keys that no longer exist.
*
* When we try to evict a key, and all the entries in the pool don't exist
* we populate it again. This time we'll be sure that the pool has at least
* one key that can be evicted, if there is at least one key that can be
* evicted in the whole database. */
/* Create a new eviction pool. */
void evictionPoolAlloc(void) {
struct evictionPoolEntry *ep;
int j;
ep = zmalloc(sizeof(*ep)*EVPOOL_SIZE);
for (j = 0; j < EVPOOL_SIZE; j++) {
ep[j].idle = 0;
ep[j].key = NULL;
ep[j].cached = sdsnewlen(NULL,EVPOOL_CACHED_SDS_SIZE);
ep[j].dbid = 0;
}
EvictionPoolLRU = ep;
}
/* This is an helper function for freeMemoryIfNeeded(), it is used in order
* to populate the evictionPool with a few entries every time we want to
* expire a key. Keys with idle time smaller than one of the current
* keys are added. Keys are always added if there are free entries.
*
* We insert keys on place in ascending order, so keys with the smaller
* idle time are on the left, and keys with the higher idle time on the
* right. */
void evictionPoolPopulate(int dbid, dict *sampledict, dict *keydict, struct evictionPoolEntry *pool) {
int j, k, count;
#ifndef _WIN32
dictEntry *samples[server.maxmemory_samples];
#else
dictEntry **samples;
samples = zmalloc(sizeof(dictEntry*) * server.maxmemory_samples);
#endif
count = dictGetSomeKeys(sampledict,samples,server.maxmemory_samples);
for (j = 0; j < count; j++) {
PORT_ULONGLONG idle;
sds key;
robj *o;
dictEntry *de;
de = samples[j];
key = dictGetKey(de);
/* If the dictionary we are sampling from is not the main
* dictionary (but the expires one) we need to lookup the key
* again in the key dictionary to obtain the value object. */
if (server.maxmemory_policy != MAXMEMORY_VOLATILE_TTL) {
if (sampledict != keydict) de = dictFind(keydict, key);
o = dictGetVal(de);
}
/* Calculate the idle time according to the policy. This is called
* idle just because the code initially handled LRU, but is in fact
* just a score where an higher score means better candidate. */
if (server.maxmemory_policy & MAXMEMORY_FLAG_LRU) {
idle = estimateObjectIdleTime(o);
} else if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) {
/* When we use an LRU policy, we sort the keys by idle time
* so that we expire keys starting from greater idle time.
* However when the policy is an LFU one, we have a frequency
* estimation, and we want to evict keys with lower frequency
* first. So inside the pool we put objects using the inverted
* frequency subtracting the actual frequency to the maximum
* frequency of 255. */
idle = 255-LFUDecrAndReturn(o);
} else if (server.maxmemory_policy == MAXMEMORY_VOLATILE_TTL) {
/* In this case the sooner the expire the better. */
idle = ULLONG_MAX - (PORT_LONG)dictGetVal(de);
} else {
serverPanic("Unknown eviction policy in evictionPoolPopulate()");
}
/* Insert the element inside the pool.
* First, find the first empty bucket or the first populated
* bucket that has an idle time smaller than our idle time. */
k = 0;
while (k < EVPOOL_SIZE &&
pool[k].key &&
pool[k].idle < idle) k++;
if (k == 0 && pool[EVPOOL_SIZE-1].key != NULL) {
/* Can't insert if the element is < the worst element we have
* and there are no empty buckets. */
continue;
} else if (k < EVPOOL_SIZE && pool[k].key == NULL) {
/* Inserting into empty position. No setup needed before insert. */
} else {
/* Inserting in the middle. Now k points to the first element
* greater than the element to insert. */
if (pool[EVPOOL_SIZE-1].key == NULL) {
/* Free space on the right? Insert at k shifting
* all the elements from k to end to the right. */
/* Save SDS before overwriting. */
sds cached = pool[EVPOOL_SIZE-1].cached;
memmove(pool+k+1,pool+k,
sizeof(pool[0])*(EVPOOL_SIZE-k-1));
pool[k].cached = cached;
} else {
/* No free space on right? Insert at k-1 */
k--;
/* Shift all elements on the left of k (included) to the
* left, so we discard the element with smaller idle time. */
sds cached = pool[0].cached; /* Save SDS before overwriting. */
if (pool[0].key != pool[0].cached) sdsfree(pool[0].key);
memmove(pool,pool+1,sizeof(pool[0])*k);
pool[k].cached = cached;
}
}
/* Try to reuse the cached SDS string allocated in the pool entry,
* because allocating and deallocating this object is costly
* (according to the profiler, not my fantasy. Remember:
* premature optimizbla bla bla bla. */
int klen = sdslen(key);
if (klen > EVPOOL_CACHED_SDS_SIZE) {
pool[k].key = sdsdup(key);
} else {
memcpy(pool[k].cached,key,klen+1);
sdssetlen(pool[k].cached,klen);
pool[k].key = pool[k].cached;
}
pool[k].idle = idle;
pool[k].dbid = dbid;
}
#ifdef _WIN32
zfree(samples);
#endif
}
/* ----------------------------------------------------------------------------
* LFU (Least Frequently Used) implementation.
* We have 24 total bits of space in each object in order to implement
* an LFU (Least Frequently Used) eviction policy, since we re-use the
* LRU field for this purpose.
*
* We split the 24 bits into two fields:
*
* 16 bits 8 bits
* +----------------+--------+
* + Last decr time | LOG_C |
* +----------------+--------+
*
* LOG_C is a logarithmic counter that provides an indication of the access
* frequency. However this field must also be decremented otherwise what used
* to be a frequently accessed key in the past, will remain ranked like that
* forever, while we want the algorithm to adapt to access pattern changes.
*
* So the remaining 16 bits are used in order to store the "decrement time",
* a reduced-precision Unix time (we take 16 bits of the time converted
* in minutes since we don't care about wrapping around) where the LOG_C
* counter is halved if it has an high value, or just decremented if it
* has a low value.
*
* New keys don't start at zero, in order to have the ability to collect
* some accesses before being trashed away, so they start at COUNTER_INIT_VAL.
* The logarithmic increment performed on LOG_C takes care of COUNTER_INIT_VAL
* when incrementing the key, so that keys starting at COUNTER_INIT_VAL
* (or having a smaller value) have a very high chance of being incremented
* on access.
*
* During decrement, the value of the logarithmic counter is halved if
* its current value is greater than two times the COUNTER_INIT_VAL, otherwise
* it is just decremented by one.
* --------------------------------------------------------------------------*/
/* Return the current time in minutes, just taking the least significant
* 16 bits. The returned time is suitable to be stored as LDT (last decrement
* time) for the LFU implementation. */
PORT_ULONG LFUGetTimeInMinutes(void) {
return (server.unixtime/60) & 65535;
}
/* Given an object last decrement time, compute the minimum number of minutes
* that elapsed since the last decrement. Handle overflow (ldt greater than
* the current 16 bits minutes time) considering the time as wrapping
* exactly once. */
PORT_ULONG LFUTimeElapsed(PORT_ULONG ldt) {
PORT_ULONG now = LFUGetTimeInMinutes();
if (now >= ldt) return now-ldt;
return 65535-ldt+now;
}
/* Logarithmically increment a counter. The greater is the current counter value
* the less likely is that it gets really implemented. Saturate it at 255. */
uint8_t LFULogIncr(uint8_t counter) {
if (counter == 255) return 255;
double r = (double)rand()/RAND_MAX;
double baseval = counter - LFU_INIT_VAL;
if (baseval < 0) baseval = 0;
double p = 1.0/(baseval*server.lfu_log_factor+1);
if (r < p) counter++;
return counter;
}
/* If the object decrement time is reached, decrement the LFU counter and
* update the decrement time field. Return the object frequency counter.
*
* This function is used in order to scan the dataset for the best object
* to fit: as we check for the candidate, we incrementally decrement the
* counter of the scanned objects if needed. */
#define LFU_DECR_INTERVAL 1
PORT_ULONG LFUDecrAndReturn(robj *o) {
PORT_ULONG ldt = o->lru >> 8;
PORT_ULONG counter = o->lru & 255;
if (LFUTimeElapsed(ldt) >= server.lfu_decay_time && counter) {
if (counter > LFU_INIT_VAL*2) {
counter /= 2;
if (counter < LFU_INIT_VAL*2) counter = LFU_INIT_VAL*2;
} else {
counter--;
}
o->lru = (LFUGetTimeInMinutes()<<8) | counter;
}
return counter;
}
/* ----------------------------------------------------------------------------
* The external API for eviction: freeMemroyIfNeeded() is called by the
* server when there is data to add in order to make space if needed.
* --------------------------------------------------------------------------*/
/* We don't want to count AOF buffers and slaves output buffers as
* used memory: the eviction should use mostly data size. This function
* returns the sum of AOF and slaves buffer. */
size_t freeMemoryGetNotCountedMemory(void) {
size_t overhead = 0;
int slaves = listLength(server.slaves);
if (slaves) {
listIter li;
listNode *ln;
listRewind(server.slaves,&li);
while((ln = listNext(&li))) {
client *slave = listNodeValue(ln);
overhead += getClientOutputBufferMemoryUsage(slave);
}
}
if (server.aof_state != AOF_OFF) {
overhead += sdslen(server.aof_buf)+aofRewriteBufferSize();
}
return overhead;
}
int freeMemoryIfNeeded(void) {
size_t mem_reported, mem_used, mem_tofree, mem_freed;
mstime_t latency, eviction_latency;
PORT_LONGLONG delta;
int slaves = listLength(server.slaves);
/* When clients are paused the dataset should be static not just from the
* POV of clients not being able to write, but also from the POV of
* expires and evictions of keys not being performed. */
if (clientsArePaused()) return C_OK;
/* Check if we are over the memory usage limit. If we are not, no need
* to subtract the slaves output buffers. We can just return ASAP. */
mem_reported = zmalloc_used_memory();
if (mem_reported <= server.maxmemory) return C_OK;
/* Remove the size of slaves output buffers and AOF buffer from the
* count of used memory. */
mem_used = mem_reported;
size_t overhead = freeMemoryGetNotCountedMemory();
mem_used = (mem_used > overhead) ? mem_used-overhead : 0;
/* Check if we are still over the memory limit. */
if (mem_used <= server.maxmemory) return C_OK;
/* Compute how much memory we need to free. */
mem_tofree = mem_used - server.maxmemory;
mem_freed = 0;
if (server.maxmemory_policy == MAXMEMORY_NO_EVICTION)
goto cant_free; /* We need to free memory, but policy forbids. */
latencyStartMonitor(latency);
while (mem_freed < mem_tofree) {
int j, k, i, keys_freed = 0;
static int next_db = 0;
sds bestkey = NULL;
int bestdbid;
redisDb *db;
dict *dict;
dictEntry *de;
if (server.maxmemory_policy & (MAXMEMORY_FLAG_LRU|MAXMEMORY_FLAG_LFU) ||
server.maxmemory_policy == MAXMEMORY_VOLATILE_TTL)
{
struct evictionPoolEntry *pool = EvictionPoolLRU;
while(bestkey == NULL) {
PORT_ULONG total_keys = 0, keys;
/* We don't want to make local-db choices when expiring keys,
* so to start populate the eviction pool sampling keys from
* every DB. */
for (i = 0; i < server.dbnum; i++) {
db = server.db+i;
dict = (server.maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS) ?
db->dict : db->expires;
if ((keys = dictSize(dict)) != 0) {
evictionPoolPopulate(i, dict, db->dict, pool);
total_keys += keys;
}
}
if (!total_keys) break; /* No keys to evict. */
/* Go backward from best to worst element to evict. */
for (k = EVPOOL_SIZE-1; k >= 0; k--) {
if (pool[k].key == NULL) continue;
bestdbid = pool[k].dbid;
if (server.maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS) {
de = dictFind(server.db[pool[k].dbid].dict,
pool[k].key);
} else {
de = dictFind(server.db[pool[k].dbid].expires,
pool[k].key);
}
/* Remove the entry from the pool. */
if (pool[k].key != pool[k].cached)
sdsfree(pool[k].key);
pool[k].key = NULL;
pool[k].idle = 0;
/* If the key exists, is our pick. Otherwise it is
* a ghost and we need to try the next element. */
if (de) {
bestkey = dictGetKey(de);
break;
} else {
/* Ghost... Iterate again. */
}
}
}
}
/* volatile-random and allkeys-random policy */
else if (server.maxmemory_policy == MAXMEMORY_ALLKEYS_RANDOM ||
server.maxmemory_policy == MAXMEMORY_VOLATILE_RANDOM)
{
/* When evicting a random key, we try to evict a key for
* each DB, so we use the static 'next_db' variable to
* incrementally visit all DBs. */
for (i = 0; i < server.dbnum; i++) {
j = (++next_db) % server.dbnum;
db = server.db+j;
dict = (server.maxmemory_policy == MAXMEMORY_ALLKEYS_RANDOM) ?
db->dict : db->expires;
if (dictSize(dict) != 0) {
de = dictGetRandomKey(dict);
bestkey = dictGetKey(de);
bestdbid = j;
break;
}
}
}
/* Finally remove the selected key. */
if (bestkey) {
db = server.db+bestdbid;
robj *keyobj = createStringObject(bestkey,sdslen(bestkey));
propagateExpire(db,keyobj,server.lazyfree_lazy_eviction);
/* We compute the amount of memory freed by db*Delete() alone.
* It is possible that actually the memory needed to propagate
* the DEL in AOF and replication link is greater than the one
* we are freeing removing the key, but we can't account for
* that otherwise we would never exit the loop.
*
* AOF and Output buffer memory will be freed eventually so
* we only care about memory used by the key space. */
delta = (PORT_LONGLONG) zmalloc_used_memory();
latencyStartMonitor(eviction_latency);
if (server.lazyfree_lazy_eviction)
dbAsyncDelete(db,keyobj);
else
dbSyncDelete(db,keyobj);
latencyEndMonitor(eviction_latency);
latencyAddSampleIfNeeded("eviction-del",eviction_latency);
latencyRemoveNestedEvent(latency,eviction_latency);
delta -= (PORT_LONGLONG) zmalloc_used_memory();
mem_freed += delta;
server.stat_evictedkeys++;
notifyKeyspaceEvent(NOTIFY_EVICTED, "evicted",
keyobj, db->id);
decrRefCount(keyobj);
keys_freed++;
/* When the memory to free starts to be big enough, we may
* start spending so much time here that is impossible to
* deliver data to the slaves fast enough, so we force the
* transmission here inside the loop. */
if (slaves) flushSlavesOutputBuffers();
/* Normally our stop condition is the ability to release
* a fixed, pre-computed amount of memory. However when we
* are deleting objects in another thread, it's better to
* check, from time to time, if we already reached our target
* memory, since the "mem_freed" amount is computed only
* across the dbAsyncDelete() call, while the thread can
* release the memory all the time. */
if (server.lazyfree_lazy_eviction && !(keys_freed % 16)) {
overhead = freeMemoryGetNotCountedMemory();
mem_used = zmalloc_used_memory();
mem_used = (mem_used > overhead) ? mem_used-overhead : 0;
if (mem_used <= server.maxmemory) {
mem_freed = mem_tofree;
}
}
}
if (!keys_freed) {
latencyEndMonitor(latency);
latencyAddSampleIfNeeded("eviction-cycle",latency);
goto cant_free; /* nothing to free... */
}
}
latencyEndMonitor(latency);
latencyAddSampleIfNeeded("eviction-cycle",latency);
return C_OK;
cant_free:
/* We are here if we are not able to reclaim memory. There is only one
* last thing we can try: check if the lazyfree thread has jobs in queue
* and wait... */
while(bioPendingJobsOfType(BIO_LAZY_FREE)) {
if (((mem_reported - zmalloc_used_memory()) + mem_freed) >= mem_tofree)
break;
usleep(1000);
}
return C_ERR;
}
src/expire.c
+504
View File
@@ -0,0 +1,504 @@
/* Implementation of EXPIRE (keys with fixed time to live).
*
* ----------------------------------------------------------------------------
*
* Copyright (c) 2009-2016, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "server.h"
/*-----------------------------------------------------------------------------
* Incremental collection of expired keys.
*
* When keys are accessed they are expired on-access. However we need a
* mechanism in order to ensure keys are eventually removed when expired even
* if no access is performed on them.
*----------------------------------------------------------------------------*/
/* Helper function for the activeExpireCycle() function.
* This function will try to expire the key that is stored in the hash table
* entry 'de' of the 'expires' hash table of a Redis database.
*
* If the key is found to be expired, it is removed from the database and
* 1 is returned. Otherwise no operation is performed and 0 is returned.
*
* When a key is expired, server.stat_expiredkeys is incremented.
*
* The parameter 'now' is the current time in milliseconds as is passed
* to the function to avoid too many gettimeofday() syscalls. */
int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, PORT_LONGLONG now) {
PORT_LONGLONG t = dictGetSignedIntegerVal(de);
if (now > t) {
sds key = dictGetKey(de);
robj *keyobj = createStringObject(key,sdslen(key));
propagateExpire(db,keyobj,server.lazyfree_lazy_expire);
if (server.lazyfree_lazy_expire)
dbAsyncDelete(db,keyobj);
else
dbSyncDelete(db,keyobj);
notifyKeyspaceEvent(NOTIFY_EXPIRED,
"expired",keyobj,db->id);
decrRefCount(keyobj);
server.stat_expiredkeys++;
return 1;
} else {
return 0;
}
}
/* Try to expire a few timed out keys. The algorithm used is adaptive and
* will use few CPU cycles if there are few expiring keys, otherwise
* it will get more aggressive to avoid that too much memory is used by
* keys that can be removed from the keyspace.
*
* No more than CRON_DBS_PER_CALL databases are tested at every
* iteration.
*
* This kind of call is used when Redis detects that timelimit_exit is
* true, so there is more work to do, and we do it more incrementally from
* the beforeSleep() function of the event loop.
*
* Expire cycle type:
*
* If type is ACTIVE_EXPIRE_CYCLE_FAST the function will try to run a
* "fast" expire cycle that takes no longer than EXPIRE_FAST_CYCLE_DURATION
* microseconds, and is not repeated again before the same amount of time.
*
* If type is ACTIVE_EXPIRE_CYCLE_SLOW, that normal expire cycle is
* executed, where the time limit is a percentage of the REDIS_HZ period
* as specified by the ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC define. */
void activeExpireCycle(int type) {
/* This function has some global state in order to continue the work
* incrementally across calls. */
static unsigned int current_db = 0; /* Last DB tested. */
static int timelimit_exit = 0; /* Time limit hit in previous call? */
static PORT_LONGLONG last_fast_cycle = 0; /* When last fast cycle ran. */
int j, iteration = 0;
int dbs_per_call = CRON_DBS_PER_CALL;
PORT_LONGLONG start = ustime(), timelimit;
/* When clients are paused the dataset should be static not just from the
* POV of clients not being able to write, but also from the POV of
* expires and evictions of keys not being performed. */
if (clientsArePaused()) return;
if (type == ACTIVE_EXPIRE_CYCLE_FAST) {
/* Don't start a fast cycle if the previous cycle did not exited
* for time limt. Also don't repeat a fast cycle for the same period
* as the fast cycle total duration itself. */
if (!timelimit_exit) return;
if (start < last_fast_cycle + ACTIVE_EXPIRE_CYCLE_FAST_DURATION*2) return;
last_fast_cycle = start;
}
/* We usually should test CRON_DBS_PER_CALL per iteration, with
* two exceptions:
*
* 1) Don't test more DBs than we have.
* 2) If last time we hit the time limit, we want to scan all DBs
* in this iteration, as there is work to do in some DB and we don't want
* expired keys to use memory for too much time. */
if (dbs_per_call > server.dbnum || timelimit_exit)
dbs_per_call = server.dbnum;
/* We can use at max ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC percentage of CPU time
* per iteration. Since this function gets called with a frequency of
* server.hz times per second, the following is the max amount of
* microseconds we can spend in this function. */
timelimit = 1000000*ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC/server.hz/100;
timelimit_exit = 0;
if (timelimit <= 0) timelimit = 1;
if (type == ACTIVE_EXPIRE_CYCLE_FAST)
timelimit = ACTIVE_EXPIRE_CYCLE_FAST_DURATION; /* in microseconds. */
for (j = 0; j < dbs_per_call; j++) {
int expired;
redisDb *db = server.db+(current_db % server.dbnum);
/* Increment the DB now so we are sure if we run out of time
* in the current DB we'll restart from the next. This allows to
* distribute the time evenly across DBs. */
current_db++;
/* Continue to expire if at the end of the cycle more than 25%
* of the keys were expired. */
do {
PORT_ULONG num, slots;
PORT_LONGLONG now, ttl_sum;
int ttl_samples;
/* If there is nothing to expire try next DB ASAP. */
if ((num = dictSize(db->expires)) == 0) {
db->avg_ttl = 0;
break;
}
slots = dictSlots(db->expires);
now = mstime();
/* When there are less than 1% filled slots getting random
* keys is expensive, so stop here waiting for better times...
* The dictionary will be resized asap. */
if (num && slots > DICT_HT_INITIAL_SIZE &&
(num*100/slots < 1)) break;
/* The main collection cycle. Sample random keys among keys
* with an expire set, checking for expired ones. */
expired = 0;
ttl_sum = 0;
ttl_samples = 0;
if (num > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP)
num = ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP;
while (num--) {
dictEntry *de;
PORT_LONGLONG ttl;
if ((de = dictGetRandomKey(db->expires)) == NULL) break;
ttl = dictGetSignedIntegerVal(de)-now;
if (activeExpireCycleTryExpire(db,de,now)) expired++;
if (ttl > 0) {
/* We want the average TTL of keys yet not expired. */
ttl_sum += ttl;
ttl_samples++;
}
}
/* Update the average TTL stats for this database. */
if (ttl_samples) {
PORT_LONGLONG avg_ttl = ttl_sum/ttl_samples;
/* Do a simple running average with a few samples.
* We just use the current estimate with a weight of 2%
* and the previous estimate with a weight of 98%. */
if (db->avg_ttl == 0) db->avg_ttl = avg_ttl;
db->avg_ttl = (db->avg_ttl/50)*49 + (avg_ttl/50);
}
/* We can't block forever here even if there are many keys to
* expire. So after a given amount of milliseconds return to the
* caller waiting for the other active expire cycle. */
iteration++;
if ((iteration & 0xf) == 0) { /* check once every 16 iterations. */
PORT_LONGLONG elapsed = ustime()-start;
latencyAddSampleIfNeeded("expire-cycle",elapsed/1000);
if (elapsed > timelimit) timelimit_exit = 1;
}
if (timelimit_exit) return;
/* We don't repeat the cycle if there are less than 25% of keys
* found expired in the current DB. */
} while (expired > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP/4);
}
}
/*-----------------------------------------------------------------------------
* Expires of keys created in writable slaves
*
* Normally slaves do not process expires: they wait the masters to synthesize
* DEL operations in order to retain consistency. However writable slaves are
* an exception: if a key is created in the slave and an expire is assigned
* to it, we need a way to expire such a key, since the master does not know
* anything about such a key.
*
* In order to do so, we track keys created in the slave side with an expire
* set, and call the expireSlaveKeys() function from time to time in order to
* reclaim the keys if they already expired.
*
* Note that the use case we are trying to cover here, is a popular one where
* slaves are put in writable mode in order to compute slow operations in
* the slave side that are mostly useful to actually read data in a more
* processed way. Think at sets intersections in a tmp key, with an expire so
* that it is also used as a cache to avoid intersecting every time.
*
* This implementation is currently not perfect but a lot better than leaking
* the keys as implemented in 3.2.
*----------------------------------------------------------------------------*/
/* The dictionary where we remember key names and database ID of keys we may
* want to expire from the slave. Since this function is not often used we
* don't even care to initialize the database at startup. We'll do it once
* the feature is used the first time, that is, when rememberSlaveKeyWithExpire()
* is called.
*
* The dictionary has an SDS string representing the key as the hash table
* key, while the value is a 64 bit unsigned integer with the bits corresponding
* to the DB where the keys may exist set to 1. Currently the keys created
* with a DB id > 63 are not expired, but a trivial fix is to set the bitmap
* to the max 64 bit unsigned value when we know there is a key with a DB
* ID greater than 63, and check all the configured DBs in such a case. */
dict *slaveKeysWithExpire = NULL;
/* Check the set of keys created by the master with an expire set in order to
* check if they should be evicted. */
void expireSlaveKeys(void) {
if (slaveKeysWithExpire == NULL ||
dictSize(slaveKeysWithExpire) == 0) return;
int cycles = 0, noexpire = 0;
mstime_t start = mstime();
while(1) {
dictEntry *de = dictGetRandomKey(slaveKeysWithExpire);
sds keyname = dictGetKey(de);
uint64_t dbids = dictGetUnsignedIntegerVal(de);
uint64_t new_dbids = 0;
/* Check the key against every database corresponding to the
* bits set in the value bitmap. */
int dbid = 0;
while(dbids && dbid < server.dbnum) {
if ((dbids & 1) != 0) {
redisDb *db = server.db+dbid;
dictEntry *expire = dictFind(db->expires,keyname);
int expired = 0;
if (expire &&
activeExpireCycleTryExpire(server.db+dbid,expire,start))
{
expired = 1;
}
/* If the key was not expired in this DB, we need to set the
* corresponding bit in the new bitmap we set as value.
* At the end of the loop if the bitmap is zero, it means we
* no longer need to keep track of this key. */
if (expire && !expired) {
noexpire++;
new_dbids |= (uint64_t)1 << dbid;
}
}
dbid++;
dbids >>= 1;
}
/* Set the new bitmap as value of the key, in the dictionary
* of keys with an expire set directly in the writable slave. Otherwise
* if the bitmap is zero, we no longer need to keep track of it. */
if (new_dbids)
dictSetUnsignedIntegerVal(de,new_dbids);
else
dictDelete(slaveKeysWithExpire,keyname);
/* Stop conditions: found 3 keys we cna't expire in a row or
* time limit was reached. */
cycles++;
if (noexpire > 3) break;
if ((cycles % 64) == 0 && mstime()-start > 1) break;
if (dictSize(slaveKeysWithExpire) == 0) break;
}
}
/* Track keys that received an EXPIRE or similar command in the context
* of a writable slave. */
void rememberSlaveKeyWithExpire(redisDb *db, robj *key) {
if (slaveKeysWithExpire == NULL) {
static dictType dt = {
dictSdsHash, /* hash function */
NULL, /* key dup */
NULL, /* val dup */
dictSdsKeyCompare, /* key compare */
dictSdsDestructor, /* key destructor */
NULL /* val destructor */
};
slaveKeysWithExpire = dictCreate(&dt,NULL);
}
if (db->id > 63) return;
dictEntry *de = dictAddOrFind(slaveKeysWithExpire,key->ptr);
/* If the entry was just created, set it to a copy of the SDS string
* representing the key: we don't want to need to take those keys
* in sync with the main DB. The keys will be removed by expireSlaveKeys()
* as it scans to find keys to remove. */
if (de->key == key->ptr) {
de->key = sdsdup(key->ptr);
dictSetUnsignedIntegerVal(de,0);
}
uint64_t dbids = dictGetUnsignedIntegerVal(de);
dbids |= (uint64_t)1 << db->id;
dictSetUnsignedIntegerVal(de,dbids);
}
/* Return the number of keys we are tracking. */
size_t getSlaveKeyWithExpireCount(void) {
if (slaveKeysWithExpire == NULL) return 0;
return dictSize(slaveKeysWithExpire);
}
/* Remove the keys in the hash table. We need to do that when data is
* flushed from the server. We may receive new keys from the master with
* the same name/db and it is no longer a good idea to expire them.
*
* Note: technically we should handle the case of a single DB being flushed
* but it is not worth it since anyway race conditions using the same set
* of key names in a wriatable slave and in its master will lead to
* inconsistencies. This is just a best-effort thing we do. */
void flushSlaveKeysWithExpireList(void) {
if (slaveKeysWithExpire) {
dictRelease(slaveKeysWithExpire);
slaveKeysWithExpire = NULL;
}
}
/*-----------------------------------------------------------------------------
* Expires Commands
*----------------------------------------------------------------------------*/
/* This is the generic command implementation for EXPIRE, PEXPIRE, EXPIREAT
* and PEXPIREAT. Because the commad second argument may be relative or absolute
* the "basetime" argument is used to signal what the base time is (either 0
* for *AT variants of the command, or the current time for relative expires).
*
* unit is either UNIT_SECONDS or UNIT_MILLISECONDS, and is only used for
* the argv[2] parameter. The basetime is always specified in milliseconds. */
void expireGenericCommand(client *c, PORT_LONGLONG basetime, int unit) {
robj *key = c->argv[1], *param = c->argv[2];
PORT_LONGLONG when; /* unix time in milliseconds when the key will expire. */
if (getLongLongFromObjectOrReply(c, param, &when, NULL) != C_OK)
return;
if (unit == UNIT_SECONDS) when *= 1000;
when += basetime;
/* No key, return zero. */
if (lookupKeyWrite(c->db,key) == NULL) {
addReply(c,shared.czero);
return;
}
/* EXPIRE with negative TTL, or EXPIREAT with a timestamp into the past
* should never be executed as a DEL when load the AOF or in the context
* of a slave instance.
*
* Instead we take the other branch of the IF statement setting an expire
* (possibly in the past) and wait for an explicit DEL from the master. */
if (when <= mstime() && !server.loading && !server.masterhost) {
robj *aux;
int deleted = server.lazyfree_lazy_expire ? dbAsyncDelete(c->db,key) :
dbSyncDelete(c->db,key);
serverAssertWithInfo(c,key,deleted);
server.dirty++;
/* Replicate/AOF this as an explicit DEL or UNLINK. */
aux = server.lazyfree_lazy_expire ? shared.unlink : shared.del;
rewriteClientCommandVector(c,2,aux,key);
signalModifiedKey(c->db,key);
notifyKeyspaceEvent(NOTIFY_GENERIC,"del",key,c->db->id);
addReply(c, shared.cone);
return;
} else {
setExpire(c,c->db,key,when);
addReply(c,shared.cone);
signalModifiedKey(c->db,key);
notifyKeyspaceEvent(NOTIFY_GENERIC,"expire",key,c->db->id);
server.dirty++;
return;
}
}
/* EXPIRE key seconds */
void expireCommand(client *c) {
expireGenericCommand(c,mstime(),UNIT_SECONDS);
}
/* EXPIREAT key time */
void expireatCommand(client *c) {
expireGenericCommand(c,0,UNIT_SECONDS);
}
/* PEXPIRE key milliseconds */
void pexpireCommand(client *c) {
expireGenericCommand(c,mstime(),UNIT_MILLISECONDS);
}
/* PEXPIREAT key ms_time */
void pexpireatCommand(client *c) {
expireGenericCommand(c,0,UNIT_MILLISECONDS);
}
/* Implements TTL and PTTL */
void ttlGenericCommand(client *c, int output_ms) {
PORT_LONGLONG expire, ttl = -1;
/* If the key does not exist at all, return -2 */
if (lookupKeyReadWithFlags(c->db,c->argv[1],LOOKUP_NOTOUCH) == NULL) {
addReplyLongLong(c,-2);
return;
}
/* The key exists. Return -1 if it has no expire, or the actual
* TTL value otherwise. */
expire = getExpire(c->db,c->argv[1]);
if (expire != -1) {
ttl = expire-mstime();
if (ttl < 0) ttl = 0;
}
if (ttl == -1) {
addReplyLongLong(c,-1);
} else {
addReplyLongLong(c,output_ms ? ttl : ((ttl+500)/1000));
}
}
/* TTL key */
void ttlCommand(client *c) {
ttlGenericCommand(c, 0);
}
/* PTTL key */
void pttlCommand(client *c) {
ttlGenericCommand(c, 1);
}
/* PERSIST key */
void persistCommand(client *c) {
if (lookupKeyWrite(c->db,c->argv[1])) {
if (removeExpire(c->db,c->argv[1])) {
addReply(c,shared.cone);
server.dirty++;
} else {
addReply(c,shared.czero);
}
} else {
addReply(c,shared.czero);
}
}
/* TOUCH key1 [key2 key3 ... keyN] */
void touchCommand(client *c) {
int touched = 0;
for (int j = 1; j < c->argc; j++)
if (lookupKeyRead(c->db,c->argv[j]) != NULL) touched++;
addReplyLongLong(c,touched);
}
src/geo.c
+72 -38
View File
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2014, Matt Stancliff <matt@genges.com>.
* Copyright (c) 2015, Salvatore Sanfilippo <antirez@gmail.com>.
* Copyright (c) 2015-2016, Salvatore Sanfilippo <antirez@gmail.com>.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -30,6 +30,7 @@
#include "geo.h"
#include "geohash_helper.h"
#include "debugmacro.h"
/* Things exported from t_zset.c only for geo.c, since it is the only other
* part of Redis that requires close zset introspection. */
@@ -112,7 +113,7 @@ int extractLongLatOrReply(client *c, robj **argv, double *xy) {
int longLatFromMember(robj *zobj, robj *member, double *xy) {
double score = 0;
if (zsetScore(zobj, member, &score) == C_ERR) return C_ERR;
if (zsetScore(zobj, member->ptr, &score) == C_ERR) return C_ERR;
if (!decodeGeohash(score, xy)) return C_ERR;
return C_OK;
}
@@ -160,7 +161,7 @@ double extractDistanceOrReply(client *c, robj **argv,
addReplyError(c,"radius cannot be negative");
return -1;
}
double to_meters = extractUnitOrReply(c,argv[1]);
if (to_meters < 0) {
return -1;
@@ -268,16 +269,14 @@ int geoGetPointsInRange(robj *zobj, double min, double max, double lon, double l
}
while (ln) {
robj *o = ln->obj;
sds ele = ln->ele;
/* Abort when the node is no longer in range. */
if (!zslValueLteMax(ln->score, &range))
break;
member = (o->encoding == OBJ_ENCODING_INT) ?
sdsfromlonglong((PORT_LONG)o->ptr) :
sdsdup(o->ptr);
if (geoAppendIfWithinRadius(ga,lon,lat,radius,ln->score,member)
== C_ERR) sdsfree(member);
ele = sdsdup(ele);
if (geoAppendIfWithinRadius(ga,lon,lat,radius,ln->score,ele)
== C_ERR) sdsfree(ele);
ln = ln->level[0].forward;
}
}
@@ -327,6 +326,7 @@ int membersOfGeoHashBox(robj *zobj, GeoHashBits hash, geoArray *ga, double lon,
int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, double lon, double lat, double radius, geoArray *ga) {
GeoHashBits neighbors[9];
unsigned int i, count = 0, last_processed = 0;
int debugmsg = 0;
neighbors[0] = n.hash;
neighbors[1] = n.neighbors.north;
@@ -341,8 +341,26 @@ int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, double lon, double lat, d
/* For each neighbor (*and* our own hashbox), get all the matching
* members and add them to the potential result list. */
for (i = 0; i < sizeof(neighbors) / sizeof(*neighbors); i++) {
if (HASHISZERO(neighbors[i]))
if (HASHISZERO(neighbors[i])) {
if (debugmsg) D("neighbors[%d] is zero",i);
continue;
}
/* Debugging info. */
if (debugmsg) {
GeoHashRange long_range, lat_range;
geohashGetCoordRange(&long_range,&lat_range);
GeoHashArea myarea = {{0}};
geohashDecode(long_range, lat_range, neighbors[i], &myarea);
/* Dump center square. */
D("neighbors[%d]:\n",i);
D("area.longitude.min: %f\n", myarea.longitude.min);
D("area.longitude.max: %f\n", myarea.longitude.max);
D("area.latitude.min: %f\n", myarea.latitude.min);
D("area.latitude.max: %f\n", myarea.latitude.max);
D("\n");
}
/* When a huge Radius (in the 5000 km range or more) is used,
* adjacent neighbors can be the same, leading to duplicated
@@ -351,7 +369,11 @@ int membersOfAllNeighbors(robj *zobj, GeoHashRadius n, double lon, double lat, d
if (last_processed &&
neighbors[i].bits == neighbors[last_processed].bits &&
neighbors[i].step == neighbors[last_processed].step)
{
if (debugmsg)
D("Skipping processing of %d, same as previous\n",i);
continue;
}
count += membersOfGeoHashBox(zobj, neighbors[i], ga, lon, lat, radius);
last_processed = i;
}
@@ -379,7 +401,7 @@ static int sort_gp_desc(const void *a, const void *b) {
* Commands
* ==================================================================== */
/* GEOADD key PORT_LONG lat name [long2 lat2 name2 ... longN latN nameN] */
/* GEOADD key long lat name [long2 lat2 name2 ... longN latN nameN] */
void geoaddCommand(client *c) {
/* Check arguments number for sanity. */
if ((c->argc - 2) % 3 != 0) {
@@ -398,7 +420,7 @@ void geoaddCommand(client *c) {
/* Create the argument vector to call ZADD in order to add all
* the score,value pairs to the requested zset, where score is actually
* an encoded version of lat,PORT_LONG. */
* an encoded version of lat,long. */
int i;
for (i = 0; i < elements; i++) {
double xy[2];
@@ -430,13 +452,14 @@ void geoaddCommand(client *c) {
#define SORT_ASC 1
#define SORT_DESC 2
#define RADIUS_COORDS 1
#define RADIUS_MEMBER 2
#define RADIUS_COORDS (1<<0) /* Search around coordinates. */
#define RADIUS_MEMBER (1<<1) /* Search around member. */
#define RADIUS_NOSTORE (1<<2) /* Do not acceot STORE/STOREDIST option. */
/* GEORADIUS key x y radius unit [WITHDIST] [WITHHASH] [WITHCOORD] [ASC|DESC]
* [COUNT count] [STORE key] [STOREDIST key]
* GEORADIUSBYMEMBER key member radius unit ... options ... */
void georadiusGeneric(client *c, int type) {
void georadiusGeneric(client *c, int flags) {
robj *key = c->argv[1];
robj *storekey = NULL;
int storedist = 0; /* 0 for STORE, 1 for STOREDIST. */
@@ -448,14 +471,14 @@ void georadiusGeneric(client *c, int type) {
return;
}
/* Find PORT_LONG/lat to use for radius search based on inquiry type */
/* Find long/lat to use for radius search based on inquiry type */
int base_args;
double xy[2] = { 0 };
if (type == RADIUS_COORDS) {
if (flags & RADIUS_COORDS) {
base_args = 6;
if (extractLongLatOrReply(c, c->argv + 2, xy) == C_ERR)
return;
} else if (type == RADIUS_MEMBER) {
} else if (flags & RADIUS_MEMBER) {
base_args = 5;
robj *member = c->argv[2];
if (longLatFromMember(zobj, member, xy) == C_ERR) {
@@ -463,7 +486,7 @@ void georadiusGeneric(client *c, int type) {
return;
}
} else {
addReplyError(c, "unknown georadius search type");
addReplyError(c, "Unknown georadius search type");
return;
}
@@ -500,11 +523,17 @@ void georadiusGeneric(client *c, int type) {
return;
}
i++;
} else if (!strcasecmp(arg, "store") && (i+1) < remaining) {
} else if (!strcasecmp(arg, "store") &&
(i+1) < remaining &&
!(flags & RADIUS_NOSTORE))
{
storekey = c->argv[base_args+i+1];
storedist = 0;
i++;
} else if (!strcasecmp(arg, "storedist") && (i+1) < remaining) {
} else if (!strcasecmp(arg, "storedist") &&
(i+1) < remaining &&
!(flags & RADIUS_NOSTORE))
{
storekey = c->argv[base_args+i+1];
storedist = 1;
i++;
@@ -619,13 +648,10 @@ void georadiusGeneric(client *c, int type) {
gp->dist /= conversion; /* Fix according to unit. */
double score = storedist ? gp->dist : gp->score;
size_t elelen = sdslen(gp->member);
robj *ele = createObject(OBJ_STRING,gp->member);
if (maxelelen < elelen) maxelelen = elelen;
incrRefCount(ele); /* Set refcount to 2 since we reference the
object both in the skiplist and dict. */
znode = zslInsert(zs->zsl,score,ele);
serverAssert(dictAdd(zs->dict,ele,&znode->score) == DICT_OK);
znode = zslInsert(zs->zsl,score,gp->member);
serverAssert(dictAdd(zs->dict,gp->member,&znode->score) == DICT_OK);
gp->member = NULL;
}
@@ -652,10 +678,20 @@ void georadiusCommand(client *c) {
}
/* GEORADIUSBYMEMBER wrapper function. */
void georadiusByMemberCommand(client *c) {
void georadiusbymemberCommand(client *c) {
georadiusGeneric(c, RADIUS_MEMBER);
}
/* GEORADIUS_RO wrapper function. */
void georadiusroCommand(client *c) {
georadiusGeneric(c, RADIUS_COORDS|RADIUS_NOSTORE);
}
/* GEORADIUSBYMEMBER_RO wrapper function. */
void georadiusbymemberroCommand(client *c) {
georadiusGeneric(c, RADIUS_MEMBER|RADIUS_NOSTORE);
}
/* GEOHASH key ele1 ele2 ... eleN
*
* Returns an array with an 11 characters geohash representation of the
@@ -665,16 +701,15 @@ void geohashCommand(client *c) {
int j;
/* Look up the requested zset */
robj *zobj = NULL;
if ((zobj = lookupKeyReadOrReply(c, c->argv[1], shared.emptymultibulk))
== NULL || checkType(c, zobj, OBJ_ZSET)) return;
robj *zobj = lookupKeyRead(c->db, c->argv[1]);
if (zobj && checkType(c, zobj, OBJ_ZSET)) return;
/* Geohash elements one after the other, using a null bulk reply for
* missing elements. */
addReplyMultiBulkLen(c,c->argc-2);
for (j = 2; j < c->argc; j++) {
double score;
if (zsetScore(zobj, c->argv[j], &score) == C_ERR) {
if (!zobj || zsetScore(zobj, c->argv[j]->ptr, &score) == C_ERR) {
addReply(c,shared.nullbulk);
} else {
/* The internal format we use for geocoding is a bit different
@@ -719,16 +754,15 @@ void geoposCommand(client *c) {
int j;
/* Look up the requested zset */
robj *zobj = NULL;
if ((zobj = lookupKeyReadOrReply(c, c->argv[1], shared.emptymultibulk))
== NULL || checkType(c, zobj, OBJ_ZSET)) return;
robj *zobj = lookupKeyRead(c->db, c->argv[1]);
if (zobj && checkType(c, zobj, OBJ_ZSET)) return;
/* Report elements one after the other, using a null bulk reply for
* missing elements. */
addReplyMultiBulkLen(c,c->argc-2);
for (j = 2; j < c->argc; j++) {
double score;
if (zsetScore(zobj, c->argv[j], &score) == C_ERR) {
if (!zobj || zsetScore(zobj, c->argv[j]->ptr, &score) == C_ERR) {
addReply(c,shared.nullmultibulk);
} else {
/* Decode... */
@@ -763,13 +797,13 @@ void geodistCommand(client *c) {
/* Look up the requested zset */
robj *zobj = NULL;
if ((zobj = lookupKeyReadOrReply(c, c->argv[1], shared.emptybulk))
if ((zobj = lookupKeyReadOrReply(c, c->argv[1], shared.nullbulk))
== NULL || checkType(c, zobj, OBJ_ZSET)) return;
/* Get the scores. We need both otherwise NULL is returned. */
double score1, score2, xyxy[4];
if (zsetScore(zobj, c->argv[2], &score1) == C_ERR ||
zsetScore(zobj, c->argv[3], &score2) == C_ERR)
if (zsetScore(zobj, c->argv[2]->ptr, &score1) == C_ERR ||
zsetScore(zobj, c->argv[3]->ptr, &score2) == C_ERR)
{
addReply(c,shared.nullbulk);
return;
src/geohash.c
+295
View File
@@ -0,0 +1,295 @@
/*
* Copyright (c) 2013-2014, yinqiwen <yinqiwen@gmail.com>
* Copyright (c) 2014, Matt Stancliff <matt@genges.com>.
* Copyright (c) 2015-2016, Salvatore Sanfilippo <antirez@gmail.com>.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "geohash.h"
/**
* Hashing works like this:
* Divide the world into 4 buckets. Label each one as such:
* -----------------
* | | |
* | | |
* | 0,1 | 1,1 |
* -----------------
* | | |
* | | |
* | 0,0 | 1,0 |
* -----------------
*/
/* Interleave lower bits of x and y, so the bits of x
* are in the even positions and bits from y in the odd;
* x and y must initially be less than 2**32 (65536).
* From: https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
*/
static inline uint64_t interleave64(uint32_t xlo, uint32_t ylo) {
static const uint64_t B[] = {0x5555555555555555ULL, 0x3333333333333333ULL,
0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
0x0000FFFF0000FFFFULL};
static const unsigned int S[] = {1, 2, 4, 8, 16};
uint64_t x = xlo;
uint64_t y = ylo;
x = (x | (x << S[4])) & B[4];
y = (y | (y << S[4])) & B[4];
x = (x | (x << S[3])) & B[3];
y = (y | (y << S[3])) & B[3];
x = (x | (x << S[2])) & B[2];
y = (y | (y << S[2])) & B[2];
x = (x | (x << S[1])) & B[1];
y = (y | (y << S[1])) & B[1];
x = (x | (x << S[0])) & B[0];
y = (y | (y << S[0])) & B[0];
return x | (y << 1);
}
/* reverse the interleave process
* derived from http://stackoverflow.com/questions/4909263
*/
static inline uint64_t deinterleave64(uint64_t interleaved) {
static const uint64_t B[] = {0x5555555555555555ULL, 0x3333333333333333ULL,
0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
static const unsigned int S[] = {0, 1, 2, 4, 8, 16};
uint64_t x = interleaved;
uint64_t y = interleaved >> 1;
x = (x | (x >> S[0])) & B[0];
y = (y | (y >> S[0])) & B[0];
x = (x | (x >> S[1])) & B[1];
y = (y | (y >> S[1])) & B[1];
x = (x | (x >> S[2])) & B[2];
y = (y | (y >> S[2])) & B[2];
x = (x | (x >> S[3])) & B[3];
y = (y | (y >> S[3])) & B[3];
x = (x | (x >> S[4])) & B[4];
y = (y | (y >> S[4])) & B[4];
x = (x | (x >> S[5])) & B[5];
y = (y | (y >> S[5])) & B[5];
return x | (y << 32);
}
void geohashGetCoordRange(GeoHashRange *long_range, GeoHashRange *lat_range) {
/* These are constraints from EPSG:900913 / EPSG:3785 / OSGEO:41001 */
/* We can't geocode at the north/south pole. */
long_range->max = GEO_LONG_MAX;
long_range->min = GEO_LONG_MIN;
lat_range->max = GEO_LAT_MAX;
lat_range->min = GEO_LAT_MIN;
}
int geohashEncode(const GeoHashRange *long_range, const GeoHashRange *lat_range,
double longitude, double latitude, uint8_t step,
GeoHashBits *hash) {
/* Check basic arguments sanity. */
if (hash == NULL || step > 32 || step == 0 ||
RANGEPISZERO(lat_range) || RANGEPISZERO(long_range)) return 0;
/* Return an error when trying to index outside the supported
* constraints. */
if (longitude > 180 || longitude < -180 ||
latitude > 85.05112878 || latitude < -85.05112878) return 0;
hash->bits = 0;
hash->step = step;
if (latitude < lat_range->min || latitude > lat_range->max ||
longitude < long_range->min || longitude > long_range->max) {
return 0;
}
double lat_offset =
(latitude - lat_range->min) / (lat_range->max - lat_range->min);
double long_offset =
(longitude - long_range->min) / (long_range->max - long_range->min);
/* convert to fixed point based on the step size */
lat_offset *= (1 << step);
long_offset *= (1 << step);
hash->bits = interleave64(lat_offset, long_offset);
return 1;
}
int geohashEncodeType(double longitude, double latitude, uint8_t step, GeoHashBits *hash) {
GeoHashRange r[2] = {{0}};
geohashGetCoordRange(&r[0], &r[1]);
return geohashEncode(&r[0], &r[1], longitude, latitude, step, hash);
}
int geohashEncodeWGS84(double longitude, double latitude, uint8_t step,
GeoHashBits *hash) {
return geohashEncodeType(longitude, latitude, step, hash);
}
int geohashDecode(const GeoHashRange long_range, const GeoHashRange lat_range,
const GeoHashBits hash, GeoHashArea *area) {
if (HASHISZERO(hash) || NULL == area || RANGEISZERO(lat_range) ||
RANGEISZERO(long_range)) {
return 0;
}
area->hash = hash;
uint8_t step = hash.step;
uint64_t hash_sep = deinterleave64(hash.bits); /* hash = [LAT][LONG] */
double lat_scale = lat_range.max - lat_range.min;
double long_scale = long_range.max - long_range.min;
uint32_t ilato = hash_sep; /* get lat part of deinterleaved hash */
uint32_t ilono = hash_sep >> 32; /* shift over to get long part of hash */
/* divide by 2**step.
* Then, for 0-1 coordinate, multiply times scale and add
to the min to get the absolute coordinate. */
area->latitude.min =
lat_range.min + (ilato * 1.0 / (1ull << step)) * lat_scale;
area->latitude.max =
lat_range.min + ((ilato + 1) * 1.0 / (1ull << step)) * lat_scale;
area->longitude.min =
long_range.min + (ilono * 1.0 / (1ull << step)) * long_scale;
area->longitude.max =
long_range.min + ((ilono + 1) * 1.0 / (1ull << step)) * long_scale;
return 1;
}
int geohashDecodeType(const GeoHashBits hash, GeoHashArea *area) {
GeoHashRange r[2] = {{0}};
geohashGetCoordRange(&r[0], &r[1]);
return geohashDecode(r[0], r[1], hash, area);
}
int geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area) {
return geohashDecodeType(hash, area);
}
int geohashDecodeAreaToLongLat(const GeoHashArea *area, double *xy) {
if (!xy) return 0;
xy[0] = (area->longitude.min + area->longitude.max) / 2;
xy[1] = (area->latitude.min + area->latitude.max) / 2;
return 1;
}
int geohashDecodeToLongLatType(const GeoHashBits hash, double *xy) {
GeoHashArea area = {{0}};
if (!xy || !geohashDecodeType(hash, &area))
return 0;
return geohashDecodeAreaToLongLat(&area, xy);
}
int geohashDecodeToLongLatWGS84(const GeoHashBits hash, double *xy) {
return geohashDecodeToLongLatType(hash, xy);
}
static void geohash_move_x(GeoHashBits *hash, int8_t d) {
if (d == 0)
return;
uint64_t x = hash->bits & 0xaaaaaaaaaaaaaaaaULL;
uint64_t y = hash->bits & 0x5555555555555555ULL;
uint64_t zz = 0x5555555555555555ULL >> (64 - hash->step * 2);
if (d > 0) {
x = x + (zz + 1);
} else {
x = x | zz;
x = x - (zz + 1);
}
x &= (0xaaaaaaaaaaaaaaaaULL >> (64 - hash->step * 2));
hash->bits = (x | y);
}
static void geohash_move_y(GeoHashBits *hash, int8_t d) {
if (d == 0)
return;
uint64_t x = hash->bits & 0xaaaaaaaaaaaaaaaaULL;
uint64_t y = hash->bits & 0x5555555555555555ULL;
uint64_t zz = 0xaaaaaaaaaaaaaaaaULL >> (64 - hash->step * 2);
if (d > 0) {
y = y + (zz + 1);
} else {
y = y | zz;
y = y - (zz + 1);
}
y &= (0x5555555555555555ULL >> (64 - hash->step * 2));
hash->bits = (x | y);
}
void geohashNeighbors(const GeoHashBits *hash, GeoHashNeighbors *neighbors) {
neighbors->east = *hash;
neighbors->west = *hash;
neighbors->north = *hash;
neighbors->south = *hash;
neighbors->south_east = *hash;
neighbors->south_west = *hash;
neighbors->north_east = *hash;
neighbors->north_west = *hash;
geohash_move_x(&neighbors->east, 1);
geohash_move_y(&neighbors->east, 0);
geohash_move_x(&neighbors->west, -1);
geohash_move_y(&neighbors->west, 0);
geohash_move_x(&neighbors->south, 0);
geohash_move_y(&neighbors->south, -1);
geohash_move_x(&neighbors->north, 0);
geohash_move_y(&neighbors->north, 1);
geohash_move_x(&neighbors->north_west, -1);
geohash_move_y(&neighbors->north_west, 1);
geohash_move_x(&neighbors->north_east, 1);
geohash_move_y(&neighbors->north_east, 1);
geohash_move_x(&neighbors->south_east, 1);
geohash_move_y(&neighbors->south_east, -1);
geohash_move_x(&neighbors->south_west, -1);
geohash_move_y(&neighbors->south_west, -1);
}
src/geohash.h
+118
View File
@@ -0,0 +1,118 @@
/*
* Copyright (c) 2013-2014, yinqiwen <yinqiwen@gmail.com>
* Copyright (c) 2014, Matt Stancliff <matt@genges.com>.
* Copyright (c) 2015, Salvatore Sanfilippo <antirez@gmail.com>.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef GEOHASH_H_
#define GEOHASH_H_
#include <stddef.h>
#include <stdint.h>
#include <stdint.h>
#if defined(__cplusplus)
extern "C" {
#endif
#define HASHISZERO(r) (!(r).bits && !(r).step)
#define RANGEISZERO(r) (!(r).max && !(r).min)
#define RANGEPISZERO(r) (r == NULL || RANGEISZERO(*r))
#define GEO_STEP_MAX 26 /* 26*2 = 52 bits. */
/* Limits from EPSG:900913 / EPSG:3785 / OSGEO:41001 */
#define GEO_LAT_MIN -85.05112878
#define GEO_LAT_MAX 85.05112878
#define GEO_LONG_MIN -180
#define GEO_LONG_MAX 180
typedef enum {
GEOHASH_NORTH = 0,
GEOHASH_EAST,
GEOHASH_WEST,
GEOHASH_SOUTH,
GEOHASH_SOUTH_WEST,
GEOHASH_SOUTH_EAST,
GEOHASH_NORT_WEST,
GEOHASH_NORT_EAST
} GeoDirection;
typedef struct {
uint64_t bits;
uint8_t step;
} GeoHashBits;
typedef struct {
double min;
double max;
} GeoHashRange;
typedef struct {
GeoHashBits hash;
GeoHashRange longitude;
GeoHashRange latitude;
} GeoHashArea;
typedef struct {
GeoHashBits north;
GeoHashBits east;
GeoHashBits west;
GeoHashBits south;
GeoHashBits north_east;
GeoHashBits south_east;
GeoHashBits north_west;
GeoHashBits south_west;
} GeoHashNeighbors;
/*
* 0:success
* -1:failed
*/
void geohashGetCoordRange(GeoHashRange *long_range, GeoHashRange *lat_range);
int geohashEncode(const GeoHashRange *long_range, const GeoHashRange *lat_range,
double longitude, double latitude, uint8_t step,
GeoHashBits *hash);
int geohashEncodeType(double longitude, double latitude,
uint8_t step, GeoHashBits *hash);
int geohashEncodeWGS84(double longitude, double latitude, uint8_t step,
GeoHashBits *hash);
int geohashDecode(const GeoHashRange long_range, const GeoHashRange lat_range,
const GeoHashBits hash, GeoHashArea *area);
int geohashDecodeType(const GeoHashBits hash, GeoHashArea *area);
int geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area);
int geohashDecodeAreaToLongLat(const GeoHashArea *area, double *xy);
int geohashDecodeToLongLatType(const GeoHashBits hash, double *xy);
int geohashDecodeToLongLatWGS84(const GeoHashBits hash, double *xy);
int geohashDecodeToLongLatMercator(const GeoHashBits hash, double *xy);
void geohashNeighbors(const GeoHashBits *hash, GeoHashNeighbors *neighbors);
#if defined(__cplusplus)
}
#endif
#endif /* GEOHASH_H_ */
src/geohash_helper.c
+237
View File
@@ -0,0 +1,237 @@
/*
* Copyright (c) 2013-2014, yinqiwen <yinqiwen@gmail.com>
* Copyright (c) 2014, Matt Stancliff <matt@genges.com>.
* Copyright (c) 2015-2016, Salvatore Sanfilippo <antirez@gmail.com>.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
/* This is a C++ to C conversion from the ardb project.
* This file started out as:
* https://github.com/yinqiwen/ardb/blob/d42503/src/geo/geohash_helper.cpp
*/
#define _USE_MATH_DEFINES
#include "fmacros.h"
#include "geohash_helper.h"
#include "debugmacro.h"
#include <math.h>
#define D_R (M_PI / 180.0)
#define R_MAJOR 6378137.0
#define R_MINOR 6356752.3142
#define RATIO (R_MINOR / R_MAJOR)
#define ECCENT (sqrt(1.0 - (RATIO *RATIO)))
#define COM (0.5 * ECCENT)
/// @brief The usual PI/180 constant
const double DEG_TO_RAD = 0.017453292519943295769236907684886;
/// @brief Earth's quatratic mean radius for WGS-84
const double EARTH_RADIUS_IN_METERS = 6372797.560856;
const double MERCATOR_MAX = 20037726.37;
const double MERCATOR_MIN = -20037726.37;
static inline double deg_rad(double ang) { return ang * D_R; }
static inline double rad_deg(double ang) { return ang / D_R; }
/* This function is used in order to estimate the step (bits precision)
* of the 9 search area boxes during radius queries. */
uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) {
if (range_meters == 0) return 26;
int step = 1;
while (range_meters < MERCATOR_MAX) {
range_meters *= 2;
step++;
}
step -= 2; /* Make sure range is included in most of the base cases. */
/* Wider range torwards the poles... Note: it is possible to do better
* than this approximation by computing the distance between meridians
* at this latitude, but this does the trick for now. */
if (lat > 66 || lat < -66) {
step--;
if (lat > 80 || lat < -80) step--;
}
/* Frame to valid range. */
if (step < 1) step = 1;
if (step > 26) step = 26;
return step;
}
/* Return the bounding box of the search area centered at latitude,longitude
* having a radius of radius_meter. bounds[0] - bounds[2] is the minimum
* and maxium longitude, while bounds[1] - bounds[3] is the minimum and
* maximum latitude.
*
* This function does not behave correctly with very large radius values, for
* instance for the coordinates 81.634948934258375 30.561509253718668 and a
* radius of 7083 kilometers, it reports as bounding boxes:
*
* min_lon 7.680495, min_lat -33.119473, max_lon 155.589402, max_lat 94.242491
*
* However, for instance, a min_lon of 7.680495 is not correct, because the
* point -1.27579540014266968 61.33421815228281559 is at less than 7000
* kilometers away.
*
* Since this function is currently only used as an optimization, the
* optimization is not used for very big radiuses, however the function
* should be fixed. */
int geohashBoundingBox(double longitude, double latitude, double radius_meters,
double *bounds) {
if (!bounds) return 0;
bounds[0] = longitude - rad_deg(radius_meters/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude)));
bounds[2] = longitude + rad_deg(radius_meters/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude)));
bounds[1] = latitude - rad_deg(radius_meters/EARTH_RADIUS_IN_METERS);
bounds[3] = latitude + rad_deg(radius_meters/EARTH_RADIUS_IN_METERS);
return 1;
}
/* Return a set of areas (center + 8) that are able to cover a range query
* for the specified position and radius. */
GeoHashRadius geohashGetAreasByRadius(double longitude, double latitude, double radius_meters) {
GeoHashRange long_range, lat_range;
GeoHashRadius radius;
GeoHashBits hash;
GeoHashNeighbors neighbors;
GeoHashArea area;
double min_lon, max_lon, min_lat, max_lat;
double bounds[4];
int steps;
geohashBoundingBox(longitude, latitude, radius_meters, bounds);
min_lon = bounds[0];
min_lat = bounds[1];
max_lon = bounds[2];
max_lat = bounds[3];
steps = geohashEstimateStepsByRadius(radius_meters,latitude);
geohashGetCoordRange(&long_range,&lat_range);
geohashEncode(&long_range,&lat_range,longitude,latitude,steps,&hash);
geohashNeighbors(&hash,&neighbors);
geohashDecode(long_range,lat_range,hash,&area);
/* Check if the step is enough at the limits of the covered area.
* Sometimes when the search area is near an edge of the
* area, the estimated step is not small enough, since one of the
* north / south / west / east square is too near to the search area
* to cover everything. */
int decrease_step = 0;
{
GeoHashArea north, south, east, west;
geohashDecode(long_range, lat_range, neighbors.north, &north);
geohashDecode(long_range, lat_range, neighbors.south, &south);
geohashDecode(long_range, lat_range, neighbors.east, &east);
geohashDecode(long_range, lat_range, neighbors.west, &west);
if (geohashGetDistance(longitude,latitude,longitude,north.latitude.max)
< radius_meters) decrease_step = 1;
if (geohashGetDistance(longitude,latitude,longitude,south.latitude.min)
< radius_meters) decrease_step = 1;
if (geohashGetDistance(longitude,latitude,east.longitude.max,latitude)
< radius_meters) decrease_step = 1;
if (geohashGetDistance(longitude,latitude,west.longitude.min,latitude)
< radius_meters) decrease_step = 1;
}
if (steps > 1 && decrease_step) {
steps--;
geohashEncode(&long_range,&lat_range,longitude,latitude,steps,&hash);
geohashNeighbors(&hash,&neighbors);
geohashDecode(long_range,lat_range,hash,&area);
}
/* Exclude the search areas that are useless. */
if (steps >= 2) {
if (area.latitude.min < min_lat) {
GZERO(neighbors.south);
GZERO(neighbors.south_west);
GZERO(neighbors.south_east);
}
if (area.latitude.max > max_lat) {
GZERO(neighbors.north);
GZERO(neighbors.north_east);
GZERO(neighbors.north_west);
}
if (area.longitude.min < min_lon) {
GZERO(neighbors.west);
GZERO(neighbors.south_west);
GZERO(neighbors.north_west);
}
if (area.longitude.max > max_lon) {
GZERO(neighbors.east);
GZERO(neighbors.south_east);
GZERO(neighbors.north_east);
}
}
radius.hash = hash;
radius.neighbors = neighbors;
radius.area = area;
return radius;
}
GeoHashRadius geohashGetAreasByRadiusWGS84(double longitude, double latitude,
double radius_meters) {
return geohashGetAreasByRadius(longitude, latitude, radius_meters);
}
GeoHashFix52Bits geohashAlign52Bits(const GeoHashBits hash) {
uint64_t bits = hash.bits;
bits <<= (52 - hash.step * 2);
return bits;
}
/* Calculate distance using haversin great circle distance formula. */
double geohashGetDistance(double lon1d, double lat1d, double lon2d, double lat2d) {
double lat1r, lon1r, lat2r, lon2r, u, v;
lat1r = deg_rad(lat1d);
lon1r = deg_rad(lon1d);
lat2r = deg_rad(lat2d);
lon2r = deg_rad(lon2d);
u = sin((lat2r - lat1r) / 2);
v = sin((lon2r - lon1r) / 2);
return 2.0 * EARTH_RADIUS_IN_METERS *
asin(sqrt(u * u + cos(lat1r) * cos(lat2r) * v * v));
}
int geohashGetDistanceIfInRadius(double x1, double y1,
double x2, double y2, double radius,
double *distance) {
*distance = geohashGetDistance(x1, y1, x2, y2);
if (*distance > radius) return 0;
return 1;
}
int geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2,
double y2, double radius,
double *distance) {
return geohashGetDistanceIfInRadius(x1, y1, x2, y2, radius, distance);
}
src/geohash_helper.h
+70
View File
@@ -0,0 +1,70 @@
/*
* Copyright (c) 2013-2014, yinqiwen <yinqiwen@gmail.com>
* Copyright (c) 2014, Matt Stancliff <matt@genges.com>.
* Copyright (c) 2015, Salvatore Sanfilippo <antirez@gmail.com>.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef GEOHASH_HELPER_HPP_
#define GEOHASH_HELPER_HPP_
#include "geohash.h"
#define GZERO(s) s.bits = s.step = 0;
#define GISZERO(s) (!s.bits && !s.step)
#define GISNOTZERO(s) (s.bits || s.step)
typedef uint64_t GeoHashFix52Bits;
typedef uint64_t GeoHashVarBits;
typedef struct {
GeoHashBits hash;
GeoHashArea area;
GeoHashNeighbors neighbors;
} GeoHashRadius;
int GeoHashBitsComparator(const GeoHashBits *a, const GeoHashBits *b);
uint8_t geohashEstimateStepsByRadius(double range_meters, double lat);
int geohashBoundingBox(double longitude, double latitude, double radius_meters,
double *bounds);
GeoHashRadius geohashGetAreasByRadius(double longitude,
double latitude, double radius_meters);
GeoHashRadius geohashGetAreasByRadiusWGS84(double longitude, double latitude,
double radius_meters);
GeoHashRadius geohashGetAreasByRadiusMercator(double longitude, double latitude,
double radius_meters);
GeoHashFix52Bits geohashAlign52Bits(const GeoHashBits hash);
double geohashGetDistance(double lon1d, double lat1d,
double lon2d, double lat2d);
int geohashGetDistanceIfInRadius(double x1, double y1,
double x2, double y2, double radius,
double *distance);
int geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2,
double y2, double radius,
double *distance);
#endif /* GEOHASH_HELPER_HPP_ */
src/hyperloglog.c
+19 -25
View File
@@ -401,7 +401,11 @@ uint64_t MurmurHash64A (const void * key, int len, unsigned int seed) {
uint64_t k;
#if (BYTE_ORDER == LITTLE_ENDIAN)
#ifdef USE_ALIGNED_ACCESS
memcpy(&k,data,sizeof(uint64_t));
#else
k = *((uint64_t*)data);
#endif
#else
k = (uint64_t) data[0];
k |= (uint64_t) data[1] << 8;
@@ -994,32 +998,21 @@ uint64_t hllCount(struct hllhdr *hdr, int *invalid) {
serverPanic("Unknown HyperLogLog encoding in hllCount()");
}
/* Muliply the inverse of E for alpha_m * m^2 to have the raw estimate. */
E = (1/E)*alpha*m*m;
/* Apply loglog-beta to the raw estimate. See:
* "LogLog-Beta and More: A New Algorithm for Cardinality Estimation
* Based on LogLog Counting" Jason Qin, Denys Kim, Yumei Tung
* arXiv:1612.02284 */
double zl = log(ez + 1);
double beta = -0.370393911*ez +
0.070471823*zl +
0.17393686*pow(zl,2) +
0.16339839*pow(zl,3) +
-0.09237745*pow(zl,4) +
0.03738027*pow(zl,5) +
-0.005384159*pow(zl,6) +
0.00042419*pow(zl,7);
/* Use the LINEARCOUNTING algorithm for small cardinalities.
* For larger values but up to 72000 HyperLogLog raw approximation is
* used since linear counting error starts to increase. However HyperLogLog
* shows a strong bias in the range 2.5*16384 - 72000, so we try to
* compensate for it. */
if (E < m*2.5 && ez != 0) {
E = m*log(m/ez); /* LINEARCOUNTING() */
} else if (m == 16384 && E < 72000) {
/* We did polynomial regression of the bias for this range, this
* way we can compute the bias for a given cardinality and correct
* according to it. Only apply the correction for P=14 that's what
* we use and the value the correction was verified with. */
double bias = 5.9119*1.0e-18*(E*E*E*E)
-1.4253*1.0e-12*(E*E*E)+
1.2940*1.0e-7*(E*E)
-5.2921*1.0e-3*E+
83.3216;
E -= E*(bias/100);
}
/* We don't apply the correction for E > 1/30 of 2^32 since we use
* a 64 bit function and 6 bit counters. To apply the correction for
* 1/30 of 2^64 is not needed since it would require a huge set
* to approach such a value. */
E = llroundl(alpha*m*(m-ez)*(1/(E+beta)));
return (uint64_t) E;
}
@@ -1128,6 +1121,7 @@ int isHLLObjectOrReply(client *c, robj *o) {
if (checkType(c,o,OBJ_STRING))
return C_ERR; /* Error already sent. */
if (!sdsEncodedObject(o)) goto invalid;
if (stringObjectLen(o) < sizeof(*hdr)) goto invalid;
hdr = o->ptr;
src/intset.c
+2 -2
View File
@@ -265,7 +265,7 @@ int64_t intsetRandom(intset *is) {
return _intsetGet(is,rand()%intrev32ifbe(is->length));
}
/* Sets the value to the value at the given position. When this position is
/* Get the value at the given position. When this position is
* out of range the function returns 0, when in range it returns 1. */
uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value) {
if (pos < intrev32ifbe(is->length)) {
@@ -276,7 +276,7 @@ uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value) {
}
/* Return intset length */
uint32_t intsetLen(intset *is) {
uint32_t intsetLen(const intset *is) {
return intrev32ifbe(is->length);
}
src/intset.h
+1 -1
View File
@@ -44,7 +44,7 @@ intset *intsetRemove(intset *is, int64_t value, int *success);
uint8_t intsetFind(intset *is, int64_t value);
int64_t intsetRandom(intset *is);
uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value);
uint32_t intsetLen(intset *is);
uint32_t intsetLen(const intset *is);
size_t intsetBlobLen(intset *is);
#ifdef REDIS_TEST
src/latency.c
+3 -3
View File
@@ -41,7 +41,7 @@ int dictStringKeyCompare(void *privdata, const void *key1, const void *key2) {
return strcmp(key1,key2) == 0;
}
unsigned int dictStringHash(const void *key) {
uint64_t dictStringHash(const void *key) {
return dictGenHashFunction(key, (int)strlen(key)); WIN_PORT_FIX /* cast (int) */
}
@@ -79,7 +79,7 @@ int THPIsEnabled(void) {
* value of the function is non-zero, the process is being targeted by
* THP support, and is likely to have memory usage / latency issues. */
int THPGetAnonHugePagesSize(void) {
return (int)zmalloc_get_smap_bytes_by_field("AnonHugePages:"); WIN_PORT_FIX /* cast (int) */
return (int)zmalloc_get_smap_bytes_by_field("AnonHugePages:",-1); WIN_PORT_FIX /* cast (int) */
}
/* ---------------------------- Latency API --------------------------------- */
@@ -262,7 +262,7 @@ sds createLatencyReport(void) {
analyzeLatencyForEvent(event,&ls);
report = sdscatprintf(report,
"%d. %s: %d latency spikes (average %lums, mean deviation %lums, period %.2f sec). Worst all time event %lums.",
"%d. %s: %d latency spikes (average %Iums, mean deviation %Iums, period %.2f sec). Worst all time event %Iums.", WIN_PORT_FIX /* %lu -> %Iu */
eventnum, event,
ls.samples,
(PORT_ULONG) ls.avg,
src/lazyfree.c
+139
View File
@@ -0,0 +1,139 @@
#include "server.h"
#include "bio.h"
#include "atomicvar.h"
#include "cluster.h"
static size_t lazyfree_objects = 0;
#ifndef _WIN32
pthread_mutex_t lazyfree_objects_mutex = PTHREAD_MUTEX_INITIALIZER;
#else
pthread_mutex_t lazyfree_objects_mutex;
#endif
/* Return the number of currently pending objects to free. */
size_t lazyfreeGetPendingObjectsCount(void) {
size_t aux;
atomicGet(lazyfree_objects,aux);
return aux;
}
/* Return the amount of work needed in order to free an object.
* The return value is not always the actual number of allocations the
* object is compoesd of, but a number proportional to it.
*
* For strings the function always returns 1.
*
* For aggregated objects represented by hash tables or other data structures
* the function just returns the number of elements the object is composed of.
*
* Objects composed of single allocations are always reported as having a
* single item even if they are actaully logical composed of multiple
* elements.
*
* For lists the funciton returns the number of elements in the quicklist
* representing the list. */
size_t lazyfreeGetFreeEffort(robj *obj) {
if (obj->type == OBJ_LIST) {
quicklist *ql = obj->ptr;
return ql->len;
} else if (obj->type == OBJ_SET && obj->encoding == OBJ_ENCODING_HT) {
dict *ht = obj->ptr;
return dictSize(ht);
} else if (obj->type == OBJ_ZSET && obj->encoding == OBJ_ENCODING_SKIPLIST){
zset *zs = obj->ptr;
return zs->zsl->length;
} else if (obj->type == OBJ_HASH && obj->encoding == OBJ_ENCODING_HT) {
dict *ht = obj->ptr;
return dictSize(ht);
} else {
return 1; /* Everything else is a single allocation. */
}
}
/* Delete a key, value, and associated expiration entry if any, from the DB.
* If there are enough allocations to free the value object may be put into
* a lazy free list instead of being freed synchronously. The lazy free list
* will be reclaimed in a different bio.c thread. */
#define LAZYFREE_THRESHOLD 64
int dbAsyncDelete(redisDb *db, robj *key) {
/* Deleting an entry from the expires dict will not free the sds of
* the key, because it is shared with the main dictionary. */
if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr);
/* If the value is composed of a few allocations, to free in a lazy way
* is actually just slower... So under a certain limit we just free
* the object synchronously. */
dictEntry *de = dictUnlink(db->dict,key->ptr);
if (de) {
robj *val = dictGetVal(de);
size_t free_effort = lazyfreeGetFreeEffort(val);
/* If releasing the object is too much work, let's put it into the
* lazy free list. */
if (free_effort > LAZYFREE_THRESHOLD) {
atomicIncr(lazyfree_objects,1);
bioCreateBackgroundJob(BIO_LAZY_FREE,val,NULL,NULL);
dictSetVal(db->dict,de,NULL);
}
}
/* Release the key-val pair, or just the key if we set the val
* field to NULL in order to lazy free it later. */
if (de) {
dictFreeUnlinkedEntry(db->dict,de);
if (server.cluster_enabled) slotToKeyDel(key);
return 1;
} else {
return 0;
}
}
/* Empty a Redis DB asynchronously. What the function does actually is to
* create a new empty set of hash tables and scheduling the old ones for
* lazy freeing. */
void emptyDbAsync(redisDb *db) {
dict *oldht1 = db->dict, *oldht2 = db->expires;
db->dict = dictCreate(&dbDictType,NULL);
db->expires = dictCreate(&keyptrDictType,NULL);
atomicIncr(lazyfree_objects,dictSize(oldht1));
bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,oldht1,oldht2);
}
/* Empty the slots-keys map of Redis CLuster by creating a new empty one
* and scheduiling the old for lazy freeing. */
void slotToKeyFlushAsync(void) {
rax *old = server.cluster->slots_to_keys;
server.cluster->slots_to_keys = raxNew();
memset(server.cluster->slots_keys_count,0,
sizeof(server.cluster->slots_keys_count));
atomicIncr(lazyfree_objects,old->numele);
bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,NULL,old);
}
/* Release objects from the lazyfree thread. It's just decrRefCount()
* updating the count of objects to release. */
void lazyfreeFreeObjectFromBioThread(robj *o) {
decrRefCount(o);
atomicDecr(lazyfree_objects,1);
}
/* Release a database from the lazyfree thread. The 'db' pointer is the
* database which was substitutied with a fresh one in the main thread
* when the database was logically deleted. 'sl' is a skiplist used by
* Redis Cluster in order to take the hash slots -> keys mapping. This
* may be NULL if Redis Cluster is disabled. */
void lazyfreeFreeDatabaseFromBioThread(dict *ht1, dict *ht2) {
size_t numkeys = dictSize(ht1);
dictRelease(ht1);
dictRelease(ht2);
atomicDecr(lazyfree_objects,numkeys);
}
/* Release the skiplist mapping Redis Cluster keys to slots in the
* lazyfree thread. */
void lazyfreeFreeSlotsMapFromBioThread(rax *rt) {
size_t len = rt->numele;
raxFree(rt);
atomicDecr(lazyfree_objects,len);
}
src/memtest.c
+3 -2
View File
@@ -29,6 +29,7 @@
#ifdef _WIN32
#include "Win32_Interop/Win32_Portability.h"
#include "Win32_Interop/win32_types.h"
#include "Win32_Interop/Win32_Error.h"
#endif
#include <stdint.h>
#include <stdlib.h>
@@ -356,8 +357,8 @@ void memtest_alloc_and_test(size_t megabytes, int passes) {
PORT_ULONG *m = malloc(bytes);
if (m == NULL) {
fprintf(stderr,"Unable to allocate %zu megabytes: %s",
megabytes, strerror(errno));
fprintf(stderr,"Unable to allocate %Iu megabytes: %s", WIN_PORT_FIX /* %zu -> %Iu */
megabytes, IF_WIN32(wsa_strerror(errno), strerror(errno)));
exit(1);
}
memtest_test(m,bytes,passes,1);
src/module.c
+3946
View File
File diff suppressed because it is too large Load Diff
src/multi.c
+18 -3
View File
@@ -117,6 +117,7 @@ void execCommand(client *c) {
int orig_argc;
struct redisCommand *orig_cmd;
int must_propagate = 0; /* Need to propagate MULTI/EXEC to AOF / slaves? */
int was_master = server.masterhost == NULL;
if (!(c->flags & CLIENT_MULTI)) {
addReplyError(c,"EXEC without MULTI");
@@ -147,11 +148,12 @@ void execCommand(client *c) {
c->argv = c->mstate.commands[j].argv;
c->cmd = c->mstate.commands[j].cmd;
/* Propagate a MULTI request once we encounter the first write op.
/* Propagate a MULTI request once we encounter the first command which
* is not readonly nor an administrative one.
* This way we'll deliver the MULTI/..../EXEC block as a whole and
* both the AOF and the replication link will have the same consistency
* and atomicity guarantees. */
if (!must_propagate && !(c->cmd->flags & CMD_READONLY)) {
if (!must_propagate && !(c->cmd->flags & (CMD_READONLY|CMD_ADMIN))) {
execCommandPropagateMulti(c);
must_propagate = 1;
}
@@ -167,9 +169,22 @@ void execCommand(client *c) {
c->argc = orig_argc;
c->cmd = orig_cmd;
discardTransaction(c);
/* Make sure the EXEC command will be propagated as well if MULTI
* was already propagated. */
if (must_propagate) server.dirty++;
if (must_propagate) {
int is_master = server.masterhost == NULL;
server.dirty++;
/* If inside the MULTI/EXEC block this instance was suddenly
* switched from master to slave (using the SLAVEOF command), the
* initial MULTI was propagated into the replication backlog, but the
* rest was not. We need to make sure to at least terminate the
* backlog with the final EXEC. */
if (server.repl_backlog && was_master && !is_master) {
char *execcmd = "*1\r\n$4\r\nEXEC\r\n";
feedReplicationBacklog(execcmd,strlen(execcmd));
}
}
handle_monitor:
/* Send EXEC to clients waiting data from MONITOR. We do it here
src/networking.c
+218 -126
View File
@@ -38,10 +38,12 @@
#include <sys/uio.h>
#endif
#include <math.h>
#include <ctype.h>
#include "atomicvar.h"
WIN32_ONLY(extern int WSIOCP_QueueAccept(int listenfd);)
static void setProtocolError(client *c, int pos);
static void setProtocolError(const char *errstr, client *c, int pos);
/* Return the size consumed from the allocator, for the specified SDS string,
* including internal fragmentation. This function is used in order to compute
@@ -62,9 +64,13 @@ size_t getStringObjectSdsUsedMemory(robj *o) {
}
}
/* Client.reply list dup and free methods. */
void *dupClientReplyValue(void *o) {
incrRefCount((robj*)o);
return o;
return sdsdup(o);
}
void freeClientReplyValue(void *o) {
sdsfree(o);
}
int listMatchObjects(void *a, void *b) {
@@ -93,11 +99,14 @@ client *createClient(int fd) {
}
selectDb(c,0);
c->id = server.next_client_id++;
uint64_t client_id;
atomicGetIncr(server.next_client_id,client_id,1);
c->id = client_id;
c->fd = fd;
c->name = NULL;
c->bufpos = 0;
c->querybuf = sdsempty();
c->pending_querybuf = sdsempty();
c->querybuf_peak = 0;
c->reqtype = 0;
c->argc = 0;
@@ -112,24 +121,26 @@ client *createClient(int fd) {
c->replstate = REPL_STATE_NONE;
c->repl_put_online_on_ack = 0;
c->reploff = 0;
c->read_reploff = 0;
c->repl_ack_off = 0;
c->repl_ack_time = 0;
c->slave_listening_port = 0;
c->slave_ip[0] = '\0';
c->slave_capa = SLAVE_CAPA_NONE;
c->reply = listCreate();
c->reply_bytes = 0;
c->obuf_soft_limit_reached_time = 0;
listSetFreeMethod(c->reply,decrRefCountVoid);
listSetFreeMethod(c->reply,freeClientReplyValue);
listSetDupMethod(c->reply,dupClientReplyValue);
c->btype = BLOCKED_NONE;
c->bpop.timeout = 0;
c->bpop.keys = dictCreate(&setDictType,NULL);
c->bpop.keys = dictCreate(&objectKeyPointerValueDictType,NULL);
c->bpop.target = NULL;
c->bpop.numreplicas = 0;
c->bpop.reploffset = 0;
c->woff = 0;
c->watched_keys = listCreate();
c->pubsub_channels = dictCreate(&setDictType,NULL);
c->pubsub_channels = dictCreate(&objectKeyPointerValueDictType,NULL);
c->pubsub_patterns = listCreate();
c->peerid = NULL;
listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid);
@@ -154,7 +165,7 @@ client *createClient(int fd) {
* event handler in the following cases:
*
* 1) The event handler should already be installed since the output buffer
* already contained something.
* already contains something.
* 2) The client is a slave but not yet online, so we want to just accumulate
* writes in the buffer but not actually sending them yet.
*
@@ -164,7 +175,7 @@ client *createClient(int fd) {
int prepareClientToWrite(client *c) {
/* If it's the Lua client we always return ok without installing any
* handler since there is no socket at all. */
if (c->flags & CLIENT_LUA) return C_OK;
if (c->flags & (CLIENT_LUA|CLIENT_MODULE)) return C_OK;
/* CLIENT REPLY OFF / SKIP handling: don't send replies. */
if (c->flags & (CLIENT_REPLY_OFF|CLIENT_REPLY_SKIP)) return C_ERR;
@@ -199,22 +210,6 @@ int prepareClientToWrite(client *c) {
return C_OK;
}
/* Create a duplicate of the last object in the reply list when
* it is not exclusively owned by the reply list. */
robj *dupLastObjectIfNeeded(list *reply) {
robj *new, *cur;
listNode *ln;
serverAssert(listLength(reply) > 0);
ln = listLast(reply);
cur = listNodeValue(ln);
if (cur->refcount > 1) {
new = dupStringObject(cur);
decrRefCount(cur);
listNodeValue(ln) = new;
}
return listNodeValue(ln);
}
/* -----------------------------------------------------------------------------
* Low level functions to add more data to output buffers.
* -------------------------------------------------------------------------- */
@@ -237,30 +232,26 @@ int _addReplyToBuffer(client *c, const char *s, size_t len) {
}
void _addReplyObjectToList(client *c, robj *o) {
robj *tail;
if (c->flags & CLIENT_CLOSE_AFTER_REPLY) return;
if (listLength(c->reply) == 0) {
incrRefCount(o);
listAddNodeTail(c->reply,o);
c->reply_bytes += getStringObjectSdsUsedMemory(o);
sds s = sdsdup(o->ptr);
listAddNodeTail(c->reply,s);
c->reply_bytes += sdslen(s);
} else {
tail = listNodeValue(listLast(c->reply));
listNode *ln = listLast(c->reply);
sds tail = listNodeValue(ln);
/* Append to this object when possible. */
if (tail->ptr != NULL &&
tail->encoding == OBJ_ENCODING_RAW &&
sdslen(tail->ptr)+sdslen(o->ptr) <= PROTO_REPLY_CHUNK_BYTES)
{
c->reply_bytes -= sdsZmallocSize(tail->ptr);
tail = dupLastObjectIfNeeded(c->reply);
tail->ptr = sdscatlen(tail->ptr,o->ptr,sdslen(o->ptr));
c->reply_bytes += sdsZmallocSize(tail->ptr);
/* Append to this object when possible. If tail == NULL it was
* set via addDeferredMultiBulkLength(). */
if (tail && sdslen(tail)+sdslen(o->ptr) <= PROTO_REPLY_CHUNK_BYTES) {
tail = sdscatsds(tail,o->ptr);
listNodeValue(ln) = tail;
c->reply_bytes += sdslen(o->ptr);
} else {
incrRefCount(o);
listAddNodeTail(c->reply,o);
c->reply_bytes += getStringObjectSdsUsedMemory(o);
sds s = sdsdup(o->ptr);
listAddNodeTail(c->reply,s);
c->reply_bytes += sdslen(s);
}
}
asyncCloseClientOnOutputBufferLimitReached(c);
@@ -269,62 +260,54 @@ void _addReplyObjectToList(client *c, robj *o) {
/* This method takes responsibility over the sds. When it is no longer
* needed it will be free'd, otherwise it ends up in a robj. */
void _addReplySdsToList(client *c, sds s) {
robj *tail;
if (c->flags & CLIENT_CLOSE_AFTER_REPLY) {
sdsfree(s);
return;
}
if (listLength(c->reply) == 0) {
listAddNodeTail(c->reply,createObject(OBJ_STRING,s));
c->reply_bytes += sdsZmallocSize(s);
listAddNodeTail(c->reply,s);
c->reply_bytes += sdslen(s);
} else {
tail = listNodeValue(listLast(c->reply));
listNode *ln = listLast(c->reply);
sds tail = listNodeValue(ln);
/* Append to this object when possible. */
if (tail->ptr != NULL && tail->encoding == OBJ_ENCODING_RAW &&
sdslen(tail->ptr)+sdslen(s) <= PROTO_REPLY_CHUNK_BYTES)
{
c->reply_bytes -= sdsZmallocSize(tail->ptr);
tail = dupLastObjectIfNeeded(c->reply);
tail->ptr = sdscatlen(tail->ptr,s,sdslen(s));
c->reply_bytes += sdsZmallocSize(tail->ptr);
/* Append to this object when possible. If tail == NULL it was
* set via addDeferredMultiBulkLength(). */
if (tail && sdslen(tail)+sdslen(s) <= PROTO_REPLY_CHUNK_BYTES) {
tail = sdscatsds(tail,s);
listNodeValue(ln) = tail;
c->reply_bytes += sdslen(s);
sdsfree(s);
} else {
listAddNodeTail(c->reply,createObject(OBJ_STRING,s));
c->reply_bytes += sdsZmallocSize(s);
listAddNodeTail(c->reply,s);
c->reply_bytes += sdslen(s);
}
}
asyncCloseClientOnOutputBufferLimitReached(c);
}
void _addReplyStringToList(client *c, const char *s, size_t len) {
robj *tail;
if (c->flags & CLIENT_CLOSE_AFTER_REPLY) return;
if (listLength(c->reply) == 0) {
robj *o = createStringObject(s,len);
listAddNodeTail(c->reply,o);
c->reply_bytes += getStringObjectSdsUsedMemory(o);
sds node = sdsnewlen(s,len);
listAddNodeTail(c->reply,node);
c->reply_bytes += len;
} else {
tail = listNodeValue(listLast(c->reply));
listNode *ln = listLast(c->reply);
sds tail = listNodeValue(ln);
/* Append to this object when possible. */
if (tail->ptr != NULL && tail->encoding == OBJ_ENCODING_RAW &&
sdslen(tail->ptr)+len <= PROTO_REPLY_CHUNK_BYTES)
{
c->reply_bytes -= sdsZmallocSize(tail->ptr);
tail = dupLastObjectIfNeeded(c->reply);
tail->ptr = sdscatlen(tail->ptr,s,len);
c->reply_bytes += sdsZmallocSize(tail->ptr);
/* Append to this object when possible. If tail == NULL it was
* set via addDeferredMultiBulkLength(). */
if (tail && sdslen(tail)+len <= PROTO_REPLY_CHUNK_BYTES) {
tail = sdscatlen(tail,s,len);
listNodeValue(ln) = tail;
c->reply_bytes += len;
} else {
robj *o = createStringObject(s,len);
listAddNodeTail(c->reply,o);
c->reply_bytes += getStringObjectSdsUsedMemory(o);
sds node = sdsnewlen(s,len);
listAddNodeTail(c->reply,node);
c->reply_bytes += len;
}
}
asyncCloseClientOnOutputBufferLimitReached(c);
@@ -385,6 +368,14 @@ void addReplySds(client *c, sds s) {
}
}
/* This low level function just adds whatever protocol you send it to the
* client buffer, trying the static buffer initially, and using the string
* of objects if not possible.
*
* It is efficient because does not create an SDS object nor an Redis object
* if not needed. The object will only be created by calling
* _addReplyStringToList() if we fail to extend the existing tail object
* in the list of objects. */
void addReplyString(client *c, const char *s, size_t len) {
if (prepareClientToWrite(c) != C_OK) return;
if (_addReplyToBuffer(c,s,len) != C_OK)
@@ -443,32 +434,32 @@ void *addDeferredMultiBulkLength(client *c) {
* ready to be sent, since we are sure that before returning to the
* event loop setDeferredMultiBulkLength() will be called. */
if (prepareClientToWrite(c) != C_OK) return NULL;
listAddNodeTail(c->reply,createObject(OBJ_STRING,NULL));
listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */
return listLast(c->reply);
}
/* Populate the length object and try gluing it to the next chunk. */
void setDeferredMultiBulkLength(client *c, void *node, PORT_LONG length) {
listNode *ln = (listNode*)node;
robj *len, *next;
sds len, next;
/* Abort when *node is NULL (see addDeferredMultiBulkLength). */
/* Abort when *node is NULL: when the client should not accept writes
* we return NULL in addDeferredMultiBulkLength() */
if (node == NULL) return;
len = listNodeValue(ln);
len->ptr = sdscatprintf(sdsempty(),"*%ld\r\n",length);
len->encoding = OBJ_ENCODING_RAW; /* in case it was an EMBSTR. */
c->reply_bytes += sdsZmallocSize(len->ptr);
len = sdscatprintf(sdsnewlen("*",1),"%Id\r\n",length); WIN_PORT_FIX /* %ld -> %Id */
listNodeValue(ln) = len;
c->reply_bytes += sdslen(len);
if (ln->next != NULL) {
next = listNodeValue(ln->next);
/* Only glue when the next node is non-NULL (an sds in this case) */
if (next->ptr != NULL) {
c->reply_bytes -= sdsZmallocSize(len->ptr);
c->reply_bytes -= getStringObjectSdsUsedMemory(next);
len->ptr = sdscatlen(len->ptr,next->ptr,sdslen(next->ptr));
c->reply_bytes += sdsZmallocSize(len->ptr);
if (next != NULL) {
len = sdscatsds(len,next);
listDelNode(c->reply,ln->next);
listNodeValue(ln) = len;
/* No need to update c->reply_bytes: we are just moving the same
* amount of bytes from one node to another. */
}
}
asyncCloseClientOnOutputBufferLimitReached(c);
@@ -580,8 +571,7 @@ void addReplyBulkCBuffer(client *c, const void *p, size_t len) {
/* Add sds to reply (takes ownership of sds and frees it) */
void addReplyBulkSds(client *c, sds s) {
addReplySds(c,sdscatfmt(sdsempty(),"$%u\r\n",
(PORT_ULONG)sdslen(s)));
addReplyLongLongWithPrefix(c,sdslen(s),'$');
addReplySds(c,s);
addReply(c,shared.crlf);
}
@@ -627,7 +617,7 @@ static void acceptCommonHandler(int fd, int flags, char *ip) {
if ((c = createClient(fd)) == NULL) {
serverLog(LL_WARNING,
"Error registering fd event for the new client: %s (fd=%d)",
strerror(errno),fd);
IF_WIN32(wsa_strerror(errno), strerror(errno)),fd);
close(fd); /* May be already closed, just ignore errors */
return;
}
@@ -827,6 +817,7 @@ void freeClient(client *c) {
/* Free the query buffer */
sdsfree(c->querybuf);
sdsfree(c->pending_querybuf);
c->querybuf = NULL;
/* Deallocate structures used to block on blocking ops. */
@@ -965,7 +956,7 @@ int writeToClient(int fd, client *c, int handler_installed) {
ssize_t nwritten = 0, totwritten = 0;
size_t objlen;
size_t objmem;
robj *o;
sds o;
listIter li;
listNode *ln;
@@ -988,8 +979,8 @@ int writeToClient(int fd, client *c, int handler_installed) {
} else {
o = listNodeValue(ln);
objlen = (int)sdslen(o->ptr);
objmem = sdsZmallocSize(o->ptr);
objlen = (int)sdslen(o);
objmem = sdsZmallocSize(o);
if (objlen == 0) {
listDelNode(c->reply,ln);
@@ -998,13 +989,13 @@ int writeToClient(int fd, client *c, int handler_installed) {
}
/* object ref placed in request, release in sendReplyListDone */
incrRefCount(o);
int result = WSIOCP_SocketSend(fd, ((char*) o->ptr), objlen,
//incrRefCount(o);
int result = WSIOCP_SocketSend(fd, o, objlen,
server.el, c, o, sendReplyListDone);
if (result == SOCKET_ERROR && errno != WSA_IO_PENDING) {
serverLog(LL_VERBOSE,
"Error writing to client: %s", wsa_strerror(errno));
decrRefCount(o);
//decrRefCount(o);
freeClient(c);
return C_ERR;
}
@@ -1042,8 +1033,7 @@ int writeToClient(int fd, client *c, int handler_installed) {
int writeToClient(int fd, client *c, int handler_installed) {
ssize_t nwritten = 0, totwritten = 0;
size_t objlen;
size_t objmem;
robj *o;
sds o;
while(clientHasPendingReplies(c)) {
if (c->bufpos > 0) {
@@ -1060,16 +1050,14 @@ int writeToClient(int fd, client *c, int handler_installed) {
}
} else {
o = listNodeValue(listFirst(c->reply));
objlen = sdslen(o->ptr);
objmem = getStringObjectSdsUsedMemory(o);
objlen = sdslen(o);
if (objlen == 0) {
listDelNode(c->reply,listFirst(c->reply));
c->reply_bytes -= objmem;
continue;
}
nwritten = write(fd, ((char*)o->ptr)+c->sentlen,objlen-c->sentlen);
nwritten = write(fd, o + c->sentlen, objlen - c->sentlen);
if (nwritten <= 0) break;
c->sentlen += nwritten;
totwritten += nwritten;
@@ -1078,7 +1066,11 @@ int writeToClient(int fd, client *c, int handler_installed) {
if (c->sentlen == objlen) {
listDelNode(c->reply,listFirst(c->reply));
c->sentlen = 0;
c->reply_bytes -= objmem;
c->reply_bytes -= objlen;
/* If there are no longer objects in the list, we expect
* the count of reply bytes to be exactly zero. */
if (listLength(c->reply) == 0)
serverAssert(c->reply_bytes == 0);
}
}
/* Note that we avoid to send more than NET_MAX_WRITES_PER_EVENT
@@ -1089,11 +1081,11 @@ int writeToClient(int fd, client *c, int handler_installed) {
*
* However if we are over the maxmemory limit we ignore that and
* just deliver as much data as it is possible to deliver. */
server.stat_net_output_bytes += totwritten;
if (totwritten > NET_MAX_WRITES_PER_EVENT &&
(server.maxmemory == 0 ||
zmalloc_used_memory() < server.maxmemory)) break;
}
server.stat_net_output_bytes += totwritten;
if (nwritten == -1) {
if (errno == EAGAIN) {
nwritten = 0;
@@ -1186,6 +1178,13 @@ void resetClient(client *c) {
}
}
/* Like processMultibulkBuffer(), but for the inline protocol instead of RESP,
* this function consumes the client query buffer and creates a command ready
* to be executed inside the client structure. Returns C_OK if the command
* is ready to be executed, or C_ERR if there is still protocol to read to
* have a well formed command. The function also returns C_ERR when there is
* a protocol error: in such a case the client structure is setup to reply
* with the error and close the connection. */
int processInlineBuffer(client *c) {
char *newline;
int argc, j;
@@ -1199,7 +1198,7 @@ int processInlineBuffer(client *c) {
if (newline == NULL) {
if (sdslen(c->querybuf) > PROTO_INLINE_MAX_SIZE) {
addReplyError(c,"Protocol error: too big inline request");
setProtocolError(c,0);
setProtocolError("too big inline request",c,0);
}
return C_ERR;
}
@@ -1215,7 +1214,7 @@ int processInlineBuffer(client *c) {
sdsfree(aux);
if (argv == NULL) {
addReplyError(c,"Protocol error: unbalanced quotes in request");
setProtocolError(c,0);
setProtocolError("unbalanced quotes in inline request",c,0);
return C_ERR;
}
@@ -1249,17 +1248,46 @@ int processInlineBuffer(client *c) {
/* Helper function. Trims query buffer to make the function that processes
* multi bulk requests idempotent. */
static void setProtocolError(client *c, int pos) {
#define PROTO_DUMP_LEN 128
static void setProtocolError(const char *errstr, client *c, int pos) {
if (server.verbosity <= LL_VERBOSE) {
sds client = catClientInfoString(sdsempty(),c);
/* Sample some protocol to given an idea about what was inside. */
char buf[256];
if (sdslen(c->querybuf) < PROTO_DUMP_LEN) {
snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%s'", c->querybuf);
} else {
snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%.*s' (... more %Iu bytes ...) '%.*s'", PROTO_DUMP_LEN/2, c->querybuf, sdslen(c->querybuf)-PROTO_DUMP_LEN, PROTO_DUMP_LEN/2, c->querybuf+sdslen(c->querybuf)-PROTO_DUMP_LEN/2); WIN_PORT_FIX /* %zu -> %Iu */
}
/* Remove non printable chars. */
char *p = buf;
while (*p != '\0') {
if (!isprint(*p)) *p = '.';
p++;
}
/* Log all the client and protocol info. */
serverLog(LL_VERBOSE,
"Protocol error from client: %s", client);
"Protocol error (%s) from client: %s. %s", errstr, client, buf);
sdsfree(client);
}
c->flags |= CLIENT_CLOSE_AFTER_REPLY;
sdsrange(c->querybuf,pos,-1);
}
/* Process the query buffer for client 'c', setting up the client argument
* vector for command execution. Returns C_OK if after running the function
* the client has a well-formed ready to be processed command, otherwise
* C_ERR if there is still to read more buffer to get the full command.
* The function also returns C_ERR when there is a protocol error: in such a
* case the client structure is setup to reply with the error and close
* the connection.
*
* This function is called if processInputBuffer() detects that the next
* command is in RESP format, so the first byte in the command is found
* to be '*'. Otherwise for inline commands processInlineBuffer() is called. */
int processMultibulkBuffer(client *c) {
char *newline = NULL;
int pos = 0, ok;
@@ -1274,7 +1302,7 @@ int processMultibulkBuffer(client *c) {
if (newline == NULL) {
if (sdslen(c->querybuf) > PROTO_INLINE_MAX_SIZE) {
addReplyError(c,"Protocol error: too big mbulk count string");
setProtocolError(c,0);
setProtocolError("too big mbulk count string",c,0);
}
return C_ERR;
}
@@ -1289,7 +1317,7 @@ int processMultibulkBuffer(client *c) {
ok = string2ll(c->querybuf+1,newline-(c->querybuf+1),&ll);
if (!ok || ll > 1024*1024) {
addReplyError(c,"Protocol error: invalid multibulk length");
setProtocolError(c,pos);
setProtocolError("invalid mbulk count",c,pos);
return C_ERR;
}
@@ -1315,7 +1343,7 @@ int processMultibulkBuffer(client *c) {
if (sdslen(c->querybuf) > PROTO_INLINE_MAX_SIZE) {
addReplyError(c,
"Protocol error: too big bulk count string");
setProtocolError(c,0);
setProtocolError("too big bulk count string",c,0);
return C_ERR;
}
break;
@@ -1329,14 +1357,14 @@ int processMultibulkBuffer(client *c) {
addReplyErrorFormat(c,
"Protocol error: expected '$', got '%c'",
c->querybuf[pos]);
setProtocolError(c,pos);
setProtocolError("expected $ but got something else",c,pos);
return C_ERR;
}
ok = string2ll(c->querybuf+pos+1,newline-(c->querybuf+pos+1),&ll);
if (!ok || ll < 0 || ll > 512*1024*1024) {
addReplyError(c,"Protocol error: invalid bulk length");
setProtocolError(c,pos);
setProtocolError("invalid bulk length",c,pos);
return C_ERR;
}
@@ -1394,10 +1422,14 @@ int processMultibulkBuffer(client *c) {
/* We're done when c->multibulk == 0 */
if (c->multibulklen == 0) return C_OK;
/* Still not read to process the command */
/* Still not ready to process the command */
return C_ERR;
}
/* This function is called every time, in the client structure 'c', there is
* more query buffer to process, because we read more data from the socket
* or because a client was blocked and later reactivated, so there could be
* pending query buffer, already representing a full command, to process. */
void processInputBuffer(client *c) {
server.current_client = c;
/* Keep processing while there is something in the input buffer */
@@ -1410,8 +1442,10 @@ void processInputBuffer(client *c) {
/* CLIENT_CLOSE_AFTER_REPLY closes the connection once the reply is
* written to the client. Make sure to not let the reply grow after
* this flag has been set (i.e. don't process more commands). */
if (c->flags & CLIENT_CLOSE_AFTER_REPLY) break;
* this flag has been set (i.e. don't process more commands).
*
* The same applies for clients we want to terminate ASAP. */
if (c->flags & (CLIENT_CLOSE_AFTER_REPLY|CLIENT_CLOSE_ASAP)) break;
/* Determine request type when unknown. */
if (!c->reqtype) {
@@ -1435,10 +1469,22 @@ void processInputBuffer(client *c) {
resetClient(c);
} else {
/* Only reset the client when the command was executed. */
if (processCommand(c) == C_OK)
resetClient(c);
/* freeMemoryIfNeeded may flush slave output buffers. This may result
* into a slave, that may be the active client, to be freed. */
if (processCommand(c) == C_OK) {
if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) {
/* Update the applied replication offset of our master. */
c->reploff = c->read_reploff - sdslen(c->querybuf);
}
/* Don't reset the client structure for clients blocked in a
* module blocking command, so that the reply callback will
* still be able to access the client argv and argc field.
* The client will be reset in unblockClientFromModule(). */
if (!(c->flags & CLIENT_BLOCKED) || c->btype != BLOCKED_MODULE)
resetClient(c);
}
/* freeMemoryIfNeeded may flush slave output buffers. This may
* result into a slave, that may be the active client, to be
* freed. */
if (server.current_client == NULL) break;
}
}
@@ -1483,11 +1529,17 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) {
serverLog(LL_VERBOSE, "Client closed connection");
freeClient(c);
return;
} else if (c->flags & CLIENT_MASTER) {
/* Append the query buffer to the pending (not applied) buffer
* of the master. We'll use this buffer later in order to have a
* copy of the string applied by the last command executed. */
c->pending_querybuf = sdscatlen(c->pending_querybuf,
c->querybuf+qblen,nread);
}
WIN32_ONLY(WSIOCP_QueueNextRead(fd);)
sdsIncrLen(c->querybuf,nread);
c->lastinteraction = server.unixtime;
if (c->flags & CLIENT_MASTER) c->reploff += nread;
if (c->flags & CLIENT_MASTER) c->read_reploff += nread;
server.stat_net_input_bytes += nread;
if (sdslen(c->querybuf) > server.client_max_querybuf_len) {
sds ci = catClientInfoString(sdsempty(),c), bytes = sdsempty();
@@ -1499,7 +1551,25 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) {
freeClient(c);
return;
}
processInputBuffer(c);
/* Time to process the buffer. If the client is a master we need to
* compute the difference between the applied offset before and after
* processing the buffer, to understand how much of the replication stream
* was actually applied to the master state: this quantity, and its
* corresponding part of the replication stream, will be propagated to
* the sub-slaves and to the replication backlog. */
if (!(c->flags & CLIENT_MASTER)) {
processInputBuffer(c);
} else {
size_t prev_offset = c->reploff;
processInputBuffer(c);
size_t applied = c->reploff - prev_offset;
if (applied) {
replicationFeedSlavesFromMasterStream(server.slaves,
c->pending_querybuf, applied);
sdsrange(c->pending_querybuf,applied,-1);
}
}
}
void getClientsMaxBuffers(PORT_ULONG *longest_output_list,
@@ -1778,6 +1848,26 @@ void clientCommand(client *c) {
}
}
/* This callback is bound to POST and "Host:" command names. Those are not
* really commands, but are used in security attacks in order to talk to
* Redis instances via HTTP, with a technique called "cross protocol scripting"
* which exploits the fact that services like Redis will discard invalid
* HTTP headers and will process what follows.
*
* As a protection against this attack, Redis will terminate the connection
* when a POST or "Host:" header is seen, and will log the event from
* time to time (to avoid creating a DOS as a result of too many logs). */
void securityWarningCommand(client *c) {
static time_t logged_time;
time_t now = time(NULL);
if (labs(now-logged_time) > 60) {
serverLog(LL_WARNING,"Possible SECURITY ATTACK detected. It looks like somebody is sending POST or Host: commands to Redis. This is likely due to an attacker attempting to use Cross Protocol Scripting to compromise your Redis instance. Connection aborted.");
logged_time = now;
}
freeClientAsync(c);
}
/* Rewrite the command vector of the client. All the new objects ref count
* is incremented. The old command vector is freed, and the old objects
* ref count is decremented. */
@@ -1863,7 +1953,9 @@ void rewriteClientCommandArgument(client *c, int i, robj *newval) {
* the caller wishes. The main usage of this function currently is
* enforcing the client output length limits. */
PORT_ULONG getClientOutputBufferMemoryUsage(client *c) {
PORT_ULONG list_item_size = sizeof(listNode)+sizeof(robj);
PORT_ULONG list_item_size = sizeof(listNode)+5;
/* The +5 above means we assume an sds16 hdr, may not be true
* but is not going to be a problem. */
return c->reply_bytes + (list_item_size*listLength(c->reply));
}
@@ -2053,7 +2145,7 @@ int clientsArePaused(void) {
* and so forth.
*
* It calls the event loop in order to process a few events. Specifically we
* try to call the event loop 4 times as PORT_LONG as we receive acknowledge that
* try to call the event loop 4 times as long as we receive acknowledge that
* some event was processed, in order to go forward with the accept, read,
* write, close sequence needed to serve a client.
*
src/object.c
+514 -67
View File
@@ -41,6 +41,8 @@
#define strtold(a,b) ((PORT_LONGDOUBLE)strtod((a),(b)))
#endif
/* ===================== Creation and parsing of objects ==================== */
robj *createObject(int type, void *ptr) {
robj *o = zmalloc(sizeof(*o));
o->type = type;
@@ -48,15 +50,37 @@ robj *createObject(int type, void *ptr) {
o->ptr = ptr;
o->refcount = 1;
/* Set the LRU to the current lruclock (minutes resolution). */
o->lru = LRU_CLOCK();
/* Set the LRU to the current lruclock (minutes resolution), or
* alternatively the LFU counter. */
if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) {
o->lru = (LFUGetTimeInMinutes()<<8) | LFU_INIT_VAL;
} else {
o->lru = LRU_CLOCK();
}
return o;
}
/* Set a special refcount in the object to make it "shared":
* incrRefCount and decrRefCount() will test for this special refcount
* and will not touch the object. This way it is free to access shared
* objects such as small integers from different threads without any
* mutex.
*
* A common patter to create shared objects:
*
* robj *myobject = makeObjectShared(createObject(...));
*
*/
robj *makeObjectShared(robj *o) {
serverAssert(o->refcount == 1);
o->refcount = OBJ_SHARED_REFCOUNT;
return o;
}
/* Create a string object with encoding OBJ_ENCODING_RAW, that is a plain
* string object where o->ptr points to a proper sds string. */
robj *createRawStringObject(const char *ptr, size_t len) {
return createObject(OBJ_STRING,sdsnewlen(ptr,len));
return createObject(OBJ_STRING, sdsnewlen(ptr,len));
}
/* Create a string object with encoding OBJ_ENCODING_EMBSTR, that is
@@ -70,9 +94,13 @@ robj *createEmbeddedStringObject(const char *ptr, size_t len) {
o->encoding = OBJ_ENCODING_EMBSTR;
o->ptr = sh+1;
o->refcount = 1;
o->lru = LRU_CLOCK();
if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) {
o->lru = (LFUGetTimeInMinutes()<<8) | LFU_INIT_VAL;
} else {
o->lru = LRU_CLOCK();
}
sh->len = (unsigned int)len; WIN_PORT_FIX /* cast (unsigned int) */
sh->len = len;
sh->alloc = len;
sh->flags = SDS_TYPE_8;
if (ptr) {
@@ -85,7 +113,7 @@ robj *createEmbeddedStringObject(const char *ptr, size_t len) {
}
/* Create a string object with EMBSTR encoding if it is smaller than
* REIDS_ENCODING_EMBSTR_SIZE_LIMIT, otherwise the RAW encoding is
* OBJ_ENCODING_EMBSTR_SIZE_LIMIT, otherwise the RAW encoding is
* used.
*
* The current limit of 39 is chosen so that the biggest string object
@@ -123,37 +151,7 @@ robj *createStringObjectFromLongLong(PORT_LONGLONG value) {
* The 'humanfriendly' option is used for INCRBYFLOAT and HINCRBYFLOAT. */
robj *createStringObjectFromLongDouble(PORT_LONGDOUBLE value, int humanfriendly) {
char buf[256];
int len;
if (isinf(value)) {
/* Libc in odd systems (Hi Solaris!) will format infinite in a
* different way, so better to handle it in an explicit way. */
if (value > 0) {
memcpy(buf,"inf",3);
len = 3;
} else {
memcpy(buf,"-inf",4);
len = 4;
}
} else if (humanfriendly) {
/* We use 17 digits precision since with 128 bit floats that precision
* after rounding is able to represent most small decimal numbers in a
* way that is "non surprising" for the user (that is, most small
* decimal numbers will be represented in a way that when converted
* back into a string are exactly the same as what the user typed.) */
len = snprintf(buf,sizeof(buf),"%.15Lf",value); WIN_PORT_FIX /* %.17 -> %.15 on Windows the magic number is 15 */
/* Now remove trailing zeroes after the '.' */
if (strchr(buf,'.') != NULL) {
char *p = buf+len-1;
while(*p == '0') {
p--;
len--;
}
if (*p == '.') len--;
}
} else {
len = snprintf(buf,sizeof(buf),"%.17Lg", value); /* TODO: verify if it needs to be changed to %.15 as well*/
}
int len = ld2string(buf,sizeof(buf),value,humanfriendly);
return createStringObject(buf,len);
}
@@ -165,7 +163,7 @@ robj *createStringObjectFromLongDouble(PORT_LONGDOUBLE value, int humanfriendly)
* will always result in a fresh object that is unshared (refcount == 1).
*
* The resulting object always has refcount set to 1. */
robj *dupStringObject(robj *o) {
robj *dupStringObject(const robj *o) {
robj *d;
serverAssert(o->type == OBJ_STRING);
@@ -239,6 +237,13 @@ robj *createZsetZiplistObject(void) {
return o;
}
robj *createModuleObject(moduleType *mt, void *value) {
moduleValue *mv = zmalloc(sizeof(*mv));
mv->type = mt;
mv->value = value;
return createObject(OBJ_MODULE,mv);
}
void freeStringObject(robj *o) {
if (o->encoding == OBJ_ENCODING_RAW) {
sdsfree(o->ptr);
@@ -246,11 +251,9 @@ void freeStringObject(robj *o) {
}
void freeListObject(robj *o) {
switch (o->encoding) {
case OBJ_ENCODING_QUICKLIST:
if (o->encoding == OBJ_ENCODING_QUICKLIST) {
quicklistRelease(o->ptr);
break;
default:
} else {
serverPanic("Unknown list encoding type");
}
}
@@ -299,12 +302,17 @@ void freeHashObject(robj *o) {
}
}
void freeModuleObject(robj *o) {
moduleValue *mv = o->ptr;
mv->type->free(mv->value);
zfree(mv);
}
void incrRefCount(robj *o) {
o->refcount++;
if (o->refcount != OBJ_SHARED_REFCOUNT) o->refcount++;
}
void decrRefCount(robj *o) {
if (o->refcount <= 0) serverPanic("decrRefCount against refcount <= 0");
if (o->refcount == 1) {
switch(o->type) {
case OBJ_STRING: freeStringObject(o); break;
@@ -312,11 +320,13 @@ void decrRefCount(robj *o) {
case OBJ_SET: freeSetObject(o); break;
case OBJ_ZSET: freeZsetObject(o); break;
case OBJ_HASH: freeHashObject(o); break;
case OBJ_MODULE: freeModuleObject(o); break;
default: serverPanic("Unknown object type"); break;
}
zfree(o);
} else {
o->refcount--;
if (o->refcount <= 0) serverPanic("decrRefCount against refcount <= 0");
if (o->refcount != OBJ_SHARED_REFCOUNT) o->refcount--;
}
}
@@ -352,13 +362,17 @@ int checkType(client *c, robj *o, int type) {
return 0;
}
int isSdsRepresentableAsLongLong(sds s, PORT_LONGLONG *llval) {
return string2ll(s,sdslen(s),llval) ? C_OK : C_ERR;
}
int isObjectRepresentableAsLongLong(robj *o, PORT_LONGLONG *llval) {
serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);
if (o->encoding == OBJ_ENCODING_INT) {
if (llval) *llval = (PORT_LONG) o->ptr;
return C_OK;
} else {
return string2ll(o->ptr,sdslen(o->ptr),llval) ? C_OK : C_ERR;
return isSdsRepresentableAsLongLong(o->ptr,llval);
}
}
@@ -385,17 +399,16 @@ robj *tryObjectEncoding(robj *o) {
if (o->refcount > 1) return o;
/* Check if we can represent this string as a long integer.
* Note that we are sure that a string larger than 21 chars is not
* Note that we are sure that a string larger than 20 chars is not
* representable as a 32 nor 64 bit integer. */
len = sdslen(s);
if (len <= 21 && string2l(s,len,&value)) {
/* This object is encodable as a PORT_LONG. Try to use a shared object.
if (len <= 20 && string2l(s,len,&value)) {
/* This object is encodable as a long. Try to use a shared object.
* Note that we avoid using shared integers when maxmemory is used
* because every object needs to have a private LRU field for the LRU
* algorithm to work well. */
if ((server.maxmemory == 0 ||
(server.maxmemory_policy != MAXMEMORY_VOLATILE_LRU &&
server.maxmemory_policy != MAXMEMORY_ALLKEYS_LRU)) &&
!(server.maxmemory_policy & MAXMEMORY_FLAG_NO_SHARED_INTEGERS)) &&
value >= 0 &&
value < OBJ_SHARED_INTEGERS)
{
@@ -539,7 +552,7 @@ size_t stringObjectLen(robj *o) {
}
}
int getDoubleFromObject(robj *o, double *target) {
int getDoubleFromObject(const robj *o, double *target) {
double value;
char *eptr;
@@ -550,7 +563,7 @@ int getDoubleFromObject(robj *o, double *target) {
if (sdsEncodedObject(o)) {
errno = 0;
value = strtod(o->ptr, &eptr);
if (isspace(((char*)o->ptr)[0]) ||
if (isspace(((const char*)o->ptr)[0]) ||
eptr[0] != '\0' ||
(errno == ERANGE &&
(value == HUGE_VAL || value == -HUGE_VAL || value == 0)) ||
@@ -628,11 +641,7 @@ int getLongLongFromObject(robj *o, PORT_LONGLONG *target) {
} else {
serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);
if (sdsEncodedObject(o)) {
errno = 0;
value = strtol(o->ptr, &eptr, 10);
if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' ||
errno == ERANGE)
return C_ERR;
if (string2ll(o->ptr,sdslen(o->ptr),&value) == 0) return C_ERR;
} else if (o->encoding == OBJ_ENCODING_INT) {
value = (PORT_LONG)o->ptr;
} else {
@@ -687,18 +696,307 @@ char *strEncoding(int encoding) {
}
}
/* Given an object returns the min number of milliseconds the object was never
* requested, using an approximated LRU algorithm. */
PORT_ULONGLONG estimateObjectIdleTime(robj *o) {
PORT_ULONGLONG lruclock = LRU_CLOCK();
if (lruclock >= o->lru) {
return (lruclock - o->lru) * LRU_CLOCK_RESOLUTION;
/* =========================== Memory introspection ========================== */
/* Returns the size in bytes consumed by the key's value in RAM.
* Note that the returned value is just an approximation, especially in the
* case of aggregated data types where only "sample_size" elements
* are checked and averaged to estimate the total size. */
#define OBJ_COMPUTE_SIZE_DEF_SAMPLES 5 /* Default sample size. */
size_t objectComputeSize(robj *o, size_t sample_size) {
sds ele, ele2;
dict *d;
dictIterator *di;
struct dictEntry *de;
size_t asize = 0, elesize = 0, samples = 0;
if (o->type == OBJ_STRING) {
if(o->encoding == OBJ_ENCODING_INT) {
asize = sizeof(*o);
} else if(o->encoding == OBJ_ENCODING_RAW) {
asize = sdsAllocSize(o->ptr)+sizeof(*o);
} else if(o->encoding == OBJ_ENCODING_EMBSTR) {
asize = sdslen(o->ptr)+2+sizeof(*o);
} else {
serverPanic("Unknown string encoding");
}
} else if (o->type == OBJ_LIST) {
if (o->encoding == OBJ_ENCODING_QUICKLIST) {
quicklist *ql = o->ptr;
quicklistNode *node = ql->head;
asize = sizeof(*o)+sizeof(quicklist);
do {
elesize += sizeof(quicklistNode)+ziplistBlobLen(node->zl);
samples++;
} while ((node = node->next) && samples < sample_size);
asize += (double)elesize/samples*listTypeLength(o);
} else if (o->encoding == OBJ_ENCODING_ZIPLIST) {
asize = sizeof(*o)+ziplistBlobLen(o->ptr);
} else {
serverPanic("Unknown list encoding");
}
} else if (o->type == OBJ_SET) {
if (o->encoding == OBJ_ENCODING_HT) {
d = o->ptr;
di = dictGetIterator(d);
asize = sizeof(*o)+sizeof(dict)+(sizeof(struct dictEntry*)*dictSlots(d));
while((de = dictNext(di)) != NULL && samples < sample_size) {
ele = dictGetKey(de);
elesize += sizeof(struct dictEntry) + sdsAllocSize(ele);
samples++;
}
dictReleaseIterator(di);
if (samples) asize += (double)elesize/samples*dictSize(d);
} else if (o->encoding == OBJ_ENCODING_INTSET) {
intset *is = o->ptr;
asize = sizeof(*o)+sizeof(*is)+is->encoding*is->length;
} else {
serverPanic("Unknown set encoding");
}
} else if (o->type == OBJ_ZSET) {
if (o->encoding == OBJ_ENCODING_ZIPLIST) {
asize = sizeof(*o)+(ziplistBlobLen(o->ptr));
} else if (o->encoding == OBJ_ENCODING_SKIPLIST) {
d = ((zset*)o->ptr)->dict;
zskiplist *zsl = ((zset*)o->ptr)->zsl;
zskiplistNode *znode = zsl->header->level[0].forward;
asize = sizeof(*o)+sizeof(zset)+(sizeof(struct dictEntry*)*dictSlots(d));
while(znode != NULL && samples < sample_size) {
elesize += sdsAllocSize(znode->ele);
elesize += sizeof(struct dictEntry) + zmalloc_size(znode);
samples++;
znode = znode->level[0].forward;
}
if (samples) asize += (double)elesize/samples*dictSize(d);
} else {
serverPanic("Unknown sorted set encoding");
}
} else if (o->type == OBJ_HASH) {
if (o->encoding == OBJ_ENCODING_ZIPLIST) {
asize = sizeof(*o)+(ziplistBlobLen(o->ptr));
} else if (o->encoding == OBJ_ENCODING_HT) {
d = o->ptr;
di = dictGetIterator(d);
asize = sizeof(*o)+sizeof(dict)+(sizeof(struct dictEntry*)*dictSlots(d));
while((de = dictNext(di)) != NULL && samples < sample_size) {
ele = dictGetKey(de);
ele2 = dictGetVal(de);
elesize += sdsAllocSize(ele) + sdsAllocSize(ele2);
elesize += sizeof(struct dictEntry);
samples++;
}
dictReleaseIterator(di);
if (samples) asize += (double)elesize/samples*dictSize(d);
} else {
serverPanic("Unknown hash encoding");
}
} else if (o->type == OBJ_MODULE) {
moduleValue *mv = o->ptr;
moduleType *mt = mv->type;
if (mt->mem_usage != NULL) {
asize = mt->mem_usage(mv->value);
} else {
asize = 0;
}
} else {
return (lruclock + (LRU_CLOCK_MAX - o->lru)) *
LRU_CLOCK_RESOLUTION;
serverPanic("Unknown object type");
}
return asize;
}
/* Release data obtained with getMemoryOverheadData(). */
void freeMemoryOverheadData(struct redisMemOverhead *mh) {
zfree(mh->db);
zfree(mh);
}
/* Return a struct redisMemOverhead filled with memory overhead
* information used for the MEMORY OVERHEAD and INFO command. The returned
* structure pointer should be freed calling freeMemoryOverheadData(). */
struct redisMemOverhead *getMemoryOverheadData(void) {
int j;
size_t mem_total = 0;
size_t mem = 0;
size_t zmalloc_used = zmalloc_used_memory();
struct redisMemOverhead *mh = zcalloc(sizeof(*mh));
mh->total_allocated = zmalloc_used;
mh->startup_allocated = server.initial_memory_usage;
mh->peak_allocated = server.stat_peak_memory;
mh->fragmentation =
zmalloc_get_fragmentation_ratio(server.resident_set_size);
mem_total += server.initial_memory_usage;
mem = 0;
if (server.repl_backlog)
mem += zmalloc_size(server.repl_backlog);
mh->repl_backlog = mem;
mem_total += mem;
mem = 0;
if (listLength(server.slaves)) {
listIter li;
listNode *ln;
listRewind(server.slaves,&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
mem += getClientOutputBufferMemoryUsage(c);
mem += sdsAllocSize(c->querybuf);
mem += sizeof(client);
}
}
mh->clients_slaves = mem;
mem_total+=mem;
mem = 0;
if (listLength(server.clients)) {
listIter li;
listNode *ln;
listRewind(server.clients,&li);
while((ln = listNext(&li))) {
client *c = listNodeValue(ln);
if (c->flags & CLIENT_SLAVE)
continue;
mem += getClientOutputBufferMemoryUsage(c);
mem += sdsAllocSize(c->querybuf);
mem += sizeof(client);
}
}
mh->clients_normal = mem;
mem_total+=mem;
mem = 0;
if (server.aof_state != AOF_OFF) {
mem += sdslen(server.aof_buf);
mem += aofRewriteBufferSize();
}
mh->aof_buffer = mem;
mem_total+=mem;
for (j = 0; j < server.dbnum; j++) {
redisDb *db = server.db+j;
PORT_LONGLONG keyscount = dictSize(db->dict);
if (keyscount==0) continue;
mh->total_keys += keyscount;
mh->db = zrealloc(mh->db,sizeof(mh->db[0])*(mh->num_dbs+1));
mh->db[mh->num_dbs].dbid = j;
mem = dictSize(db->dict) * sizeof(dictEntry) +
dictSlots(db->dict) * sizeof(dictEntry*) +
dictSize(db->dict) * sizeof(robj);
mh->db[mh->num_dbs].overhead_ht_main = mem;
mem_total+=mem;
mem = dictSize(db->expires) * sizeof(dictEntry) +
dictSlots(db->expires) * sizeof(dictEntry*);
mh->db[mh->num_dbs].overhead_ht_expires = mem;
mem_total+=mem;
mh->num_dbs++;
}
mh->overhead_total = mem_total;
mh->dataset = zmalloc_used - mem_total;
mh->peak_perc = (float)zmalloc_used*100/mh->peak_allocated;
/* Metrics computed after subtracting the startup memory from
* the total memory. */
size_t net_usage = 1;
if (zmalloc_used > mh->startup_allocated)
net_usage = zmalloc_used - mh->startup_allocated;
mh->dataset_perc = (float)mh->dataset*100/net_usage;
mh->bytes_per_key = mh->total_keys ? (net_usage / mh->total_keys) : 0;
return mh;
}
/* Helper for "MEMORY allocator-stats", used as a callback for the jemalloc
* stats output. */
void inputCatSds(void *result, const char *str) {
/* result is actually a (sds *), so re-cast it here */
sds *info = (sds *)result;
*info = sdscat(*info, str);
}
/* This implements MEMORY DOCTOR. An human readable analysis of the Redis
* memory condition. */
sds getMemoryDoctorReport(void) {
int empty = 0; /* Instance is empty or almost empty. */
int big_peak = 0; /* Memory peak is much larger than used mem. */
int high_frag = 0; /* High fragmentation. */
int big_slave_buf = 0; /* Slave buffers are too big. */
int big_client_buf = 0; /* Client buffers are too big. */
int num_reports = 0;
struct redisMemOverhead *mh = getMemoryOverheadData();
if (mh->total_allocated < (1024*1024*5)) {
empty = 1;
num_reports++;
} else {
/* Peak is > 150% of current used memory? */
if (((float)mh->peak_allocated / mh->total_allocated) > 1.5) {
big_peak = 1;
num_reports++;
}
/* Fragmentation is higher than 1.4? */
if (mh->fragmentation > 1.4) {
high_frag = 1;
num_reports++;
}
/* Clients using more than 200k each average? */
PORT_LONG numslaves = listLength(server.slaves);
PORT_LONG numclients = listLength(server.clients)-numslaves;
if (mh->clients_normal / numclients > (1024*200)) {
big_client_buf = 1;
num_reports++;
}
/* Slaves using more than 10 MB each? */
if (numslaves > 0 && mh->clients_slaves / numslaves > (1024*1024*10)) {
big_slave_buf = 1;
num_reports++;
}
}
sds s;
if (num_reports == 0) {
s = sdsnew(
"Hi Sam, I can't find any memory issue in your instance. "
"I can only account for what occurs on this base.\n");
} else if (empty == 1) {
s = sdsnew(
"Hi Sam, this instance is empty or is using very little memory, "
"my issues detector can't be used in these conditions. "
"Please, leave for your mission on Earth and fill it with some data. "
"The new Sam and I will be back to our programming as soon as I "
"finished rebooting.\n");
} else {
s = sdsnew("Sam, I detected a few issues in this Redis instance memory implants:\n\n");
if (big_peak) {
s = sdscat(s," * Peak memory: In the past this instance used more than 150% the memory that is currently using. The allocator is normally not able to release memory after a peak, so you can expect to see a big fragmentation ratio, however this is actually harmless and is only due to the memory peak, and if the Redis instance Resident Set Size (RSS) is currently bigger than expected, the memory will be used as soon as you fill the Redis instance with more data. If the memory peak was only occasional and you want to try to reclaim memory, please try the MEMORY PURGE command, otherwise the only other option is to shutdown and restart the instance.\n\n");
}
if (high_frag) {
s = sdscatprintf(s," * High fragmentation: This instance has a memory fragmentation greater than 1.4 (this means that the Resident Set Size of the Redis process is much larger than the sum of the logical allocations Redis performed). This problem is usually due either to a large peak memory (check if there is a peak memory entry above in the report) or may result from a workload that causes the allocator to fragment memory a lot. If the problem is a large peak memory, then there is no issue. Otherwise, make sure you are using the Jemalloc allocator and not the default libc malloc. Note: The currently used allocator is \"%s\".\n\n", ZMALLOC_LIB);
}
if (big_slave_buf) {
s = sdscat(s," * Big slave buffers: The slave output buffers in this instance are greater than 10MB for each slave (on average). This likely means that there is some slave instance that is struggling receiving data, either because it is too slow or because of networking issues. As a result, data piles on the master output buffers. Please try to identify what slave is not receiving data correctly and why. You can use the INFO output in order to check the slaves delays and the CLIENT LIST command to check the output buffers of each slave.\n\n");
}
if (big_client_buf) {
s = sdscat(s," * Big client buffers: The clients output buffers in this instance are greater than 200K per client (on average). This may result from different causes, like Pub/Sub clients subscribed to channels bot not receiving data fast enough, so that data piles on the Redis instance output buffer, or clients sending commands with large replies or very large sequences of commands in the same pipeline. Please use the CLIENT LIST command in order to investigate the issue if it causes problems in your instance, or to understand better why certain clients are using a big amount of memory.\n\n");
}
s = sdscat(s,"I'm here to keep you safe, Sam. I want to help you.\n");
}
freeMemoryOverheadData(mh);
return s;
}
/* ======================= The OBJECT and MEMORY commands =================== */
/* This is a helper function for the OBJECT command. We need to lookup keys
* without any modification of LRU or other parameters. */
robj *objectCommandLookup(client *c, robj *key) {
@@ -731,9 +1029,158 @@ void objectCommand(client *c) {
} else if (!strcasecmp(c->argv[1]->ptr,"idletime") && c->argc == 3) {
if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nullbulk))
== NULL) return;
if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) {
addReplyError(c,"An LFU maxmemory policy is selected, idle time not tracked. Please note that when switching between policies at runtime LRU and LFU data will take some time to adjust.");
return;
}
addReplyLongLong(c,estimateObjectIdleTime(o)/1000);
} else if (!strcasecmp(c->argv[1]->ptr,"freq") && c->argc == 3) {
if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nullbulk))
== NULL) return;
if (server.maxmemory_policy & MAXMEMORY_FLAG_LRU) {
addReplyError(c,"An LRU maxmemory policy is selected, access frequency not tracked. Please note that when switching between policies at runtime LRU and LFU data will take some time to adjust.");
return;
}
addReplyLongLong(c,o->lru&255);
} else {
addReplyError(c,"Syntax error. Try OBJECT (refcount|encoding|idletime)");
addReplyError(c,"Syntax error. Try OBJECT (refcount|encoding|idletime|freq)");
}
}
/* The memory command will eventually be a complete interface for the
* memory introspection capabilities of Redis.
*
* Usage: MEMORY usage <key> */
void memoryCommand(client *c) {
robj *o;
if (!strcasecmp(c->argv[1]->ptr,"usage") && c->argc >= 3) {
PORT_LONGLONG samples = OBJ_COMPUTE_SIZE_DEF_SAMPLES;
for (int j = 3; j < c->argc; j++) {
if (!strcasecmp(c->argv[j]->ptr,"samples") &&
j+1 < c->argc)
{
if (getLongLongFromObjectOrReply(c,c->argv[j+1],&samples,NULL)
== C_ERR) return;
if (samples < 0) {
addReply(c,shared.syntaxerr);
return;
}
if (samples == 0) samples = LLONG_MAX;;
j++; /* skip option argument. */
} else {
addReply(c,shared.syntaxerr);
return;
}
}
if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nullbulk))
== NULL) return;
size_t usage = objectComputeSize(o,samples);
usage += sdsAllocSize(c->argv[1]->ptr);
usage += sizeof(dictEntry);
addReplyLongLong(c,usage);
} else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) {
struct redisMemOverhead *mh = getMemoryOverheadData();
addReplyMultiBulkLen(c,(14+mh->num_dbs)*2);
addReplyBulkCString(c,"peak.allocated");
addReplyLongLong(c,mh->peak_allocated);
addReplyBulkCString(c,"total.allocated");
addReplyLongLong(c,mh->total_allocated);
addReplyBulkCString(c,"startup.allocated");
addReplyLongLong(c,mh->startup_allocated);
addReplyBulkCString(c,"replication.backlog");
addReplyLongLong(c,mh->repl_backlog);
addReplyBulkCString(c,"clients.slaves");
addReplyLongLong(c,mh->clients_slaves);
addReplyBulkCString(c,"clients.normal");
addReplyLongLong(c,mh->clients_normal);
addReplyBulkCString(c,"aof.buffer");
addReplyLongLong(c,mh->aof_buffer);
for (size_t j = 0; j < mh->num_dbs; j++) {
char dbname[32];
snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid);
addReplyBulkCString(c,dbname);
addReplyMultiBulkLen(c,4);
addReplyBulkCString(c,"overhead.hashtable.main");
addReplyLongLong(c,mh->db[j].overhead_ht_main);
addReplyBulkCString(c,"overhead.hashtable.expires");
addReplyLongLong(c,mh->db[j].overhead_ht_expires);
}
addReplyBulkCString(c,"overhead.total");
addReplyLongLong(c,mh->overhead_total);
addReplyBulkCString(c,"keys.count");
addReplyLongLong(c,mh->total_keys);
addReplyBulkCString(c,"keys.bytes-per-key");
addReplyLongLong(c,mh->bytes_per_key);
addReplyBulkCString(c,"dataset.bytes");
addReplyLongLong(c,mh->dataset);
addReplyBulkCString(c,"dataset.percentage");
addReplyDouble(c,mh->dataset_perc);
addReplyBulkCString(c,"peak.percentage");
addReplyDouble(c,mh->peak_perc);
addReplyBulkCString(c,"fragmentation");
addReplyDouble(c,mh->fragmentation);
freeMemoryOverheadData(mh);
} else if (!strcasecmp(c->argv[1]->ptr,"malloc-stats") && c->argc == 2) {
#if defined(USE_JEMALLOC)
sds info = sdsempty();
je_malloc_stats_print(inputCatSds, &info, NULL);
addReplyBulkSds(c, info);
#else
addReplyBulkCString(c,"Stats not supported for the current allocator");
#endif
} else if (!strcasecmp(c->argv[1]->ptr,"doctor") && c->argc == 2) {
sds report = getMemoryDoctorReport();
addReplyBulkSds(c,report);
} else if (!strcasecmp(c->argv[1]->ptr,"purge") && c->argc == 2) {
#if defined(USE_JEMALLOC)
char tmp[32];
unsigned narenas = 0;
size_t sz = sizeof(unsigned);
if (!je_mallctl("arenas.narenas", &narenas, &sz, NULL, 0)) {
sprintf(tmp, "arena.%d.purge", narenas);
if (!je_mallctl(tmp, NULL, 0, NULL, 0)) {
addReply(c, shared.ok);
return;
}
}
addReplyError(c, "Error purging dirty pages");
#else
addReply(c, shared.ok);
/* Nothing to do for other allocators. */
#endif
} else if (!strcasecmp(c->argv[1]->ptr,"help") && c->argc == 2) {
addReplyMultiBulkLen(c,5);
addReplyBulkCString(c,
"MEMORY DOCTOR - Outputs memory problems report");
addReplyBulkCString(c,
"MEMORY USAGE <key> [SAMPLES <count>] - Estimate memory usage of key");
addReplyBulkCString(c,
"MEMORY STATS - Show memory usage details");
addReplyBulkCString(c,
"MEMORY PURGE - Ask the allocator to release memory");
addReplyBulkCString(c,
"MEMORY MALLOC-STATS - Show allocator internal stats");
} else {
addReplyError(c,"Syntax error. Try MEMORY HELP");
}
}
src/quicklist.c
+8 -7
View File
@@ -149,7 +149,7 @@ REDIS_STATIC quicklistNode *quicklistCreateNode(void) {
}
/* Return cached quicklist count */
unsigned int quicklistCount(quicklist *ql) { return (unsigned int)(ql->count); } WIN_PORT_FIX /* cast (unsigned int) */
unsigned int quicklistCount(const quicklist *ql) { return (unsigned int)(ql->count); } WIN_PORT_FIX /* cast (unsigned int) */
/* Free entire quicklist. */
void quicklistRelease(quicklist *quicklist) {
@@ -671,6 +671,7 @@ int quicklistReplaceAtIndex(quicklist *quicklist, PORT_LONG index, void *data,
/* quicklistIndex provides an uncompressed node */
entry.node->zl = ziplistDelete(entry.node->zl, &entry.zi);
entry.node->zl = ziplistInsert(entry.node->zl, entry.zi, data, sz);
quicklistNodeUpdateSz(entry.node);
quicklistCompress(quicklist, entry.node);
return 1;
} else {
@@ -848,7 +849,7 @@ REDIS_STATIC void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry,
/* Populate accounting flags for easier boolean checks later */
if (!_quicklistNodeAllowInsert(node, fill, sz)) {
D("Current node is full with count %d with requested fill %lu",
D("Current node is full with count %d with requested fill %Iu", WIN_PORT_FIX /* %lu -> %Iu */
node->count, fill);
full = 1;
}
@@ -973,7 +974,7 @@ int quicklistDelRange(quicklist *quicklist, const PORT_LONG start,
if (!quicklistIndex(quicklist, start, &entry))
return 0;
D("Quicklist delete request for start %ld, count %ld, extent: %ld", start,
D("Quicklist delete request for start %Id, count %Id, extent: %Id", start, WIN_PORT_FIX /* %ld -> %Id */
count, extent);
quicklistNode *node = entry.node;
@@ -1011,7 +1012,7 @@ int quicklistDelRange(quicklist *quicklist, const PORT_LONG start,
del = extent;
}
D("[%ld]: asking to del: %ld because offset: %d; (ENTIRE NODE: %d), "
D("[%Id]: asking to del: %Id because offset: %d; (ENTIRE NODE: %d), " WIN_PORT_FIX /* %ld -> %Id */
"node count: %u",
extent, del, entry.offset, delete_entire_node, node->count);
@@ -1191,12 +1192,12 @@ quicklist *quicklistDup(quicklist *orig) {
current = current->next) {
quicklistNode *node = quicklistCreateNode();
if (node->encoding == QUICKLIST_NODE_ENCODING_LZF) {
quicklistLZF *lzf = (quicklistLZF *)node->zl;
if (current->encoding == QUICKLIST_NODE_ENCODING_LZF) {
quicklistLZF *lzf = (quicklistLZF *)current->zl;
size_t lzf_sz = sizeof(*lzf) + lzf->sz;
node->zl = zmalloc(lzf_sz);
memcpy(node->zl, current->zl, lzf_sz);
} else if (node->encoding == QUICKLIST_NODE_ENCODING_RAW) {
} else if (current->encoding == QUICKLIST_NODE_ENCODING_RAW) {
node->zl = zmalloc(current->sz);
memcpy(node->zl, current->zl, current->sz);
}
src/quicklist.h
+1 -1
View File
@@ -158,7 +158,7 @@ int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data,
void *(*saver)(unsigned char *data, unsigned int sz));
int quicklistPop(quicklist *quicklist, int where, unsigned char **data,
unsigned int *sz, PORT_LONGLONG *slong);
unsigned int quicklistCount(quicklist *ql);
unsigned int quicklistCount(const quicklist *ql);
int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len);
size_t quicklistGetLzf(const quicklistNode *node, void **data);
src/rax.c
+1730
View File
File diff suppressed because it is too large Load Diff
src/rax.h
+160
View File
@@ -0,0 +1,160 @@
#ifndef RAX_H
#define RAX_H
#include <stdint.h>
/* Representation of a radix tree as implemented in this file, that contains
* the strings "foo", "foobar" and "footer" after the insertion of each
* word. When the node represents a key inside the radix tree, we write it
* between [], otherwise it is written between ().
*
* This is the vanilla representation:
*
* (f) ""
* \
* (o) "f"
* \
* (o) "fo"
* \
* [t b] "foo"
* / \
* "foot" (e) (a) "foob"
* / \
* "foote" (r) (r) "fooba"
* / \
* "footer" [] [] "foobar"
*
* However, this implementation implements a very common optimization where
* successive nodes having a single child are "compressed" into the node
* itself as a string of characters, each representing a next-level child,
* and only the link to the node representing the last character node is
* provided inside the representation. So the above representation is turend
* into:
*
* ["foo"] ""
* |
* [t b] "foo"
* / \
* "foot" ("er") ("ar") "foob"
* / \
* "footer" [] [] "foobar"
*
* However this optimization makes the implementation a bit more complex.
* For instance if a key "first" is added in the above radix tree, a
* "node splitting" operation is needed, since the "foo" prefix is no longer
* composed of nodes having a single child one after the other. This is the
* above tree and the resulting node splitting after this event happens:
*
*
* (f) ""
* /
* (i o) "f"
* / \
* "firs" ("rst") (o) "fo"
* / \
* "first" [] [t b] "foo"
* / \
* "foot" ("er") ("ar") "foob"
* / \
* "footer" [] [] "foobar"
*
* Similarly after deletion, if a new chain of nodes having a single child
* is created (the chain must also not include nodes that represent keys),
* it must be compressed back into a single node.
*
*/
#define RAX_NODE_MAX_SIZE ((1<<29)-1)
typedef struct raxNode {
uint32_t iskey:1; /* Does this node contain a key? */
uint32_t isnull:1; /* Associated value is NULL (don't store it). */
uint32_t iscompr:1; /* Node is compressed. */
uint32_t size:29; /* Number of children, or compressed string len. */
/* Data layout is as follows:
*
* If node is not compressed we have 'size' bytes, one for each children
* character, and 'size' raxNode pointers, point to each child node.
* Note how the character is not stored in the children but in the
* edge of the parents:
*
* [header strlen=0][abc][a-ptr][b-ptr][c-ptr](value-ptr?)
*
* if node is compressed (strlen != 0) the node has 1 children.
* In that case the 'size' bytes of the string stored immediately at
* the start of the data section, represent a sequence of successive
* nodes linked one after the other, for which only the last one in
* the sequence is actually represented as a node, and pointed to by
* the current compressed node.
*
* [header strlen=3][xyz][z-ptr](value-ptr?)
*
* Both compressed and not compressed nodes can represent a key
* with associated data in the radix tree at any level (not just terminal
* nodes).
*
* If the node has an associated key (iskey=1) and is not NULL
* (isnull=0), then after the raxNode pointers poiting to the
* childen, an additional value pointer is present (as you can see
* in the representation above as "value-ptr" field).
*/
unsigned char data[];
} raxNode;
typedef struct rax {
raxNode *head;
uint64_t numele;
uint64_t numnodes;
} rax;
/* Stack data structure used by raxLowWalk() in order to, optionally, return
* a list of parent nodes to the caller. The nodes do not have a "parent"
* field for space concerns, so we use the auxiliary stack when needed. */
#define RAX_STACK_STATIC_ITEMS 32
typedef struct raxStack {
void **stack; /* Points to static_items or an heap allocated array. */
size_t items, maxitems; /* Number of items contained and total space. */
/* Up to RAXSTACK_STACK_ITEMS items we avoid to allocate on the heap
* and use this static array of pointers instead. */
void *static_items[RAX_STACK_STATIC_ITEMS];
int oom; /* True if pushing into this stack failed for OOM at some point. */
} raxStack;
/* Radix tree iterator state is encapsulated into this data structure. */
#define RAX_ITER_STATIC_LEN 128
#define RAX_ITER_JUST_SEEKED (1<<0) /* Iterator was just seeked. Return current
element for the first iteration and
clear the flag. */
#define RAX_ITER_EOF (1<<1) /* End of iteration reached. */
#define RAX_ITER_SAFE (1<<2) /* Safe iterator, allows operations while
iterating. But it is slower. */
typedef struct raxIterator {
int flags;
rax *rt; /* Radix tree we are iterating. */
unsigned char *key; /* The current string. */
void *data; /* Data associated to this key. */
size_t key_len; /* Current key length. */
size_t key_max; /* Max key len the current key buffer can hold. */
unsigned char key_static_string[RAX_ITER_STATIC_LEN];
raxNode *node; /* Current node. Only for unsafe iteration. */
raxStack stack; /* Stack used for unsafe iteration. */
} raxIterator;
/* A special pointer returned for not found items. */
extern void *raxNotFound;
/* Exported API. */
rax *raxNew(void);
int raxInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old);
int raxRemove(rax *rax, unsigned char *s, size_t len, void **old);
void *raxFind(rax *rax, unsigned char *s, size_t len);
void raxFree(rax *rax);
void raxStart(raxIterator *it, rax *rt);
int raxSeek(raxIterator *it, const char *op, unsigned char *ele, size_t len);
int raxNext(raxIterator *it);
int raxPrev(raxIterator *it);
int raxRandomWalk(raxIterator *it, size_t steps);
int raxCompare(raxIterator *iter, const char *op, unsigned char *key, size_t key_len);
void raxStop(raxIterator *it);
void raxShow(rax *rax);
#endif
src/rax_malloc.h
+44
View File
@@ -0,0 +1,44 @@
/* Rax -- A radix tree implementation.
*
* Copyright (c) 2017, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/* Allocator selection.
*
* This file is used in order to change the Rax allocator at compile time.
* Just define the following defines to what you want to use. Also add
* the include of your alternate allocator if needed (not needed in order
* to use the default libc allocator). */
#ifndef RAX_ALLOC_H
#define RAX_ALLOC_H
#include "zmalloc.h"
#define rax_malloc zmalloc
#define rax_realloc zrealloc
#define rax_free zfree
#endif
src/rdb.c
+508 -184
View File
File diff suppressed because it is too large Load Diff
src/rdb.h
+48 -15
View File
@@ -38,16 +38,17 @@
/* The current RDB version. When the format changes in a way that is no longer
* backward compatible this number gets incremented. */
#define RDB_VERSION 7
#define RDB_VERSION 8
/* Defines related to the dump file format. To store 32 bits lengths for short
* keys requires a lot of space, so we check the most significant 2 bits of
* the first byte to interpreter the length:
*
* 00|000000 => if the two MSB are 00 the len is the 6 bits of this byte
* 01|000000 00000000 => 01, the len is 14 byes, 6 bits + 8 bits of next byte
* 10|000000 [32 bit integer] => if it's 01, a full 32 bit len will follow
* 11|000000 this means: specially encoded object will follow. The six bits
* 00|XXXXXX => if the two MSB are 00 the len is the 6 bits of this byte
* 01|XXXXXX XXXXXXXX => 01, the len is 14 byes, 6 bits + 8 bits of next byte
* 10|000000 [32 bit integer] => A full 32 bit len in net byte order will follow
* 10|000001 [64 bit integer] => A full 64 bit len in net byte order will follow
* 11|OBKIND this means: specially encoded object will follow. The six bits
* number specify the kind of object that follows.
* See the RDB_ENC_* defines.
*
@@ -55,12 +56,13 @@
* values, will fit inside. */
#define RDB_6BITLEN 0
#define RDB_14BITLEN 1
#define RDB_32BITLEN 2
#define RDB_32BITLEN 0x80
#define RDB_64BITLEN 0x81
#define RDB_ENCVAL 3
#define RDB_LENERR UINT_MAX
#define RDB_LENERR UINT64_MAX
/* When a length of a string object stored on disk has the first two bits
* set, the remaining two bits specify a special encoding for the object
* set, the remaining six bits specify a special encoding for the object
* accordingly to the following defines: */
#define RDB_ENC_INT8 0 /* 8 bit signed integer */
#define RDB_ENC_INT16 1 /* 16 bit signed integer */
@@ -74,6 +76,10 @@
#define RDB_TYPE_SET 2
#define RDB_TYPE_ZSET 3
#define RDB_TYPE_HASH 4
#define RDB_TYPE_ZSET_2 5 /* ZSET version 2 with doubles stored in binary. */
#define RDB_TYPE_MODULE 6
#define RDB_TYPE_MODULE_2 7 /* Module value with annotations for parsing without
the generating module being loaded. */
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Object types for encoded objects. */
@@ -86,7 +92,7 @@
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Test if a type is an object type. */
#define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 14))
#define rdbIsObjectType(t) ((t >= 0 && t <= 7) || (t >= 9 && t <= 14))
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define RDB_OPCODE_AUX 250
@@ -96,24 +102,51 @@
#define RDB_OPCODE_SELECTDB 254
#define RDB_OPCODE_EOF 255
/* Module serialized values sub opcodes */
#define RDB_MODULE_OPCODE_EOF 0 /* End of module value. */
#define RDB_MODULE_OPCODE_SINT 1 /* Signed integer. */
#define RDB_MODULE_OPCODE_UINT 2 /* Unsigned integer. */
#define RDB_MODULE_OPCODE_FLOAT 3 /* Float. */
#define RDB_MODULE_OPCODE_DOUBLE 4 /* Double. */
#define RDB_MODULE_OPCODE_STRING 5 /* String. */
/* rdbLoad...() functions flags. */
#define RDB_LOAD_NONE 0
#define RDB_LOAD_ENC (1<<0)
#define RDB_LOAD_PLAIN (1<<1)
#define RDB_LOAD_SDS (1<<2)
#define RDB_SAVE_NONE 0
#define RDB_SAVE_AOF_PREAMBLE (1<<0)
int rdbSaveType(rio *rdb, unsigned char type);
int rdbLoadType(rio *rdb);
int rdbSaveTime(rio *rdb, time_t t);
time_t rdbLoadTime(rio *rdb);
int rdbSaveLen(rio *rdb, uint32_t len);
uint32_t rdbLoadLen(rio *rdb, int *isencoded);
int rdbSaveLen(rio *rdb, uint64_t len);
uint64_t rdbLoadLen(rio *rdb, int *isencoded);
int rdbLoadLenByRef(rio *rdb, int *isencoded, uint64_t *lenptr);
int rdbSaveObjectType(rio *rdb, robj *o);
int rdbLoadObjectType(rio *rdb);
int rdbLoad(char *filename);
int rdbSaveBackground(char *filename);
int rdbSaveToSlavesSockets(void);
int rdbLoad(char *filename, rdbSaveInfo *rsi);
int rdbSaveBackground(char *filename, rdbSaveInfo *rsi);
int rdbSaveToSlavesSockets(rdbSaveInfo *rsi);
void rdbRemoveTempFile(pid_t childpid);
int rdbSave(char *filename);
int rdbSave(char *filename, rdbSaveInfo *rsi);
ssize_t rdbSaveObject(rio *rdb, robj *o);
size_t rdbSavedObjectLen(robj *o);
robj *rdbLoadObject(int type, rio *rdb);
void backgroundSaveDoneHandler(int exitcode, int bysignal);
int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, PORT_LONGLONG expiretime, PORT_LONGLONG now);
robj *rdbLoadStringObject(rio *rdb);
int rdbSaveStringObject(rio *rdb, robj *obj);
ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len);
void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr);
int rdbSaveBinaryDoubleValue(rio *rdb, double val);
int rdbLoadBinaryDoubleValue(rio *rdb, double *val);
int rdbSaveBinaryFloatValue(rio *rdb, float val);
int rdbLoadBinaryFloatValue(rio *rdb, float *val);
int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi);
rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi);
#endif
src/redis-benchmark.c
+27 -4
View File
@@ -76,6 +76,7 @@ static struct config {
int randomkeys_keyspacelen;
int keepalive;
int pipeline;
int showerrors;
PORT_LONGLONG start;
PORT_LONGLONG totlatency;
PORT_LONGLONG *latency;
@@ -250,6 +251,16 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
exit(1);
}
if (config.showerrors) {
static time_t lasterr_time = 0;
time_t now = time(NULL);
redisReply *r = reply;
if (r->type == REDIS_REPLY_ERROR && lasterr_time != now) {
lasterr_time = now;
printf("Error from server: %s\n", r->str);
}
}
freeReplyObject(reply);
/* This is an OK for prefix commands such as auth and select.*/
if (c->prefix_pending > 0) {
@@ -265,7 +276,7 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
c->randptr[j] -= c->prefixlen;
c->prefixlen = 0;
}
continue;
continue;
}
if (config.requests_finished < config.requests)
@@ -593,6 +604,8 @@ int parseOptions(int argc, const char **argv) {
config.loop = 1;
} else if (!strcmp(argv[i],"-I")) {
config.idlemode = 1;
} else if (!strcmp(argv[i],"-e")) {
config.showerrors = 1;
} else if (!strcmp(argv[i],"-t")) {
if (lastarg) goto invalid;
/* We get the list of tests to run as a string in the form
@@ -627,15 +640,15 @@ invalid:
usage:
printf(
"Usage: redis-benchmark [-h <host>] [-p <port>] [-c <clients>] [-n <requests]> [-k <boolean>]\n\n"
"Usage: redis-benchmark [-h <host>] [-p <port>] [-c <clients>] [-n <requests>] [-k <boolean>]\n\n"
" -h <hostname> Server hostname (default 127.0.0.1)\n"
" -p <port> Server port (default 6379)\n"
" -s <socket> Server socket (overrides host and port)\n"
" -a <password> Password for Redis Auth\n"
" -c <clients> Number of parallel connections (default 50)\n"
" -n <requests> Total number of requests (default 100000)\n"
" -d <size> Data size of SET/GET value in bytes (default 2)\n"
" -dbnum <db> SELECT the specified db number (default 0)\n"
" -d <size> Data size of SET/GET value in bytes (default 3)\n"
" --dbnum <db> SELECT the specified db number (default 0)\n"
" -k <boolean> 1=keep alive 0=reconnect (default 1)\n"
" -r <keyspacelen> Use random keys for SET/GET/INCR, random values for SADD\n"
" Using this option the benchmark will expand the string __rand_int__\n"
@@ -644,6 +657,8 @@ usage:
" is executed. Default tests use this to hit random keys in the\n"
" specified range.\n"
" -P <numreq> Pipeline <numreq> requests. Default 1 (no pipeline).\n"
" -e If server replies with errors, show them on stdout.\n"
" (no more than 1 error per second is displayed)\n"
" -q Quiet. Just show query/sec values\n"
" --csv Output in CSV format\n"
" -l Loop. Run the tests forever\n"
@@ -728,6 +743,7 @@ int main(int argc, const char **argv) {
config.keepalive = 1;
config.datasize = 3;
config.pipeline = 1;
config.showerrors = 0;
config.randomkeys = 0;
config.randomkeys_keyspacelen = 0;
config.quiet = 0;
@@ -842,6 +858,13 @@ int main(int argc, const char **argv) {
free(cmd);
}
if (test_is_selected("hset")) {
len = redisFormatCommand(&cmd,
"HSET myset:__rand_int__ element:__rand_int__ %s",data);
benchmark("HSET",cmd,len);
free(cmd);
}
if (test_is_selected("spop")) {
len = redisFormatCommand(&cmd,"SPOP myset");
benchmark("SPOP",cmd,len);
src/redis-check-aof.c
+39 -7
View File
@@ -33,8 +33,10 @@
#include "Win32_Interop/win32_types.h"
#include "Win32_Interop/Win32_Error.h"
#include "Win32_Interop/win32fixes.h"
#include "zmalloc.h"
#endif
#include "server.h"
#include "fmacros.h"
#include <stdlib.h>
#include <stdio.h>
@@ -72,7 +74,7 @@ int readLong(FILE *fp, char prefix, PORT_LONG *target) {
return 0;
}
if (buf[0] != prefix) {
ERROR("Expected prefix '%c', got: '%c'",buf[0],prefix);
ERROR("Expected prefix '%c', got: '%c'",prefix,buf[0]);
return 0;
}
*target = strtol(buf+1,&eptr,10);
@@ -84,7 +86,7 @@ int readBytes(FILE *fp, char *target, PORT_LONG length) {
epos = ftello(fp);
real = (PORT_LONG) fread(target, 1, length, fp);
if (real != length) {
ERROR("Expected to read %ld bytes, got %ld bytes", length, real); /* TODO: verify %ld */
ERROR("Expected to read %Id bytes, got %Id bytes",length,real); WIN_PORT_FIX /* %ld -> %Id */
return 0;
}
return 1;
@@ -99,7 +101,7 @@ int readString(FILE *fp, char** target) {
/* Increase length to also consume \r\n */
len += 2;
*target = (char*)malloc(len);
*target = (char*)zmalloc(len);
if (!readBytes(fp,*target,len)) {
return 0;
}
@@ -139,12 +141,12 @@ off_t process(FILE *fp) {
}
}
}
free(str);
zfree(str);
}
/* Stop if the loop did not finish */
if (i < argc) {
if (str) free(str);
if (str) zfree(str);
break;
}
}
@@ -158,7 +160,8 @@ off_t process(FILE *fp) {
return pos;
}
int main(int argc, char **argv) {
int redis_check_aof_main(int argc, char **argv) {
char *filename;
int fix = 0;
#ifdef _WIN32
@@ -203,6 +206,28 @@ int main(int argc, char **argv) {
exit(1);
}
//TODO: _WIN32
#ifndef _WIN32
/* This AOF file may have an RDB preamble. Check this to start, and if this
* is the case, start processing the RDB part. */
if (size >= 8) { /* There must be at least room for the RDB header. */
char sig[5];
int has_preamble = fread(sig,sizeof(sig),1,fp) == 1 &&
memcmp(sig,"REDIS",sizeof(sig)) == 0;
rewind(fp);
if (has_preamble) {
printf("The AOF appears to start with an RDB preamble.\n"
"Checking the RDB preamble to start:\n");
if (redis_check_rdb_main(argc,argv,fp) == C_ERR) {
printf("RDB preamble of AOF file is not sane, aborting.\n");
exit(1);
} else {
printf("RDB preamble is OK, proceeding with AOF tail...\n");
}
}
}
#endif
off_t pos = process(fp);
off_t diff = size-pos;
printf("AOF analyzed: size=%lld, ok_up_to=%lld, diff=%lld\n",
@@ -224,7 +249,8 @@ int main(int argc, char **argv) {
printf("Successfully truncated AOF\n");
}
} else {
printf("AOF is not valid\n");
printf("AOF is not valid. "
"Use the --fix option to try fixing it.\n");
exit(1);
}
} else {
@@ -234,3 +260,9 @@ int main(int argc, char **argv) {
fclose(fp);
return 0;
}
#ifdef _WIN32_REDIS_CHECK_AOF_EXE
int main(int argc, char **argv) {
return redis_check_aof_main(argc, argv);
}
#endif
src/redis-check-rdb.c
+310 -692
View File
File diff suppressed because it is too large Load Diff
src/redis-cli.c
+83 -30
View File
@@ -188,7 +188,7 @@ static void cliRefreshPrompt(void) {
len = anetFormatAddr(config.prompt, sizeof(config.prompt),
config.hostip, config.hostport);
/* Add [dbnum] if needed */
if (config.dbnum != 0 && config.last_cmd_type != REDIS_REPLY_ERROR)
if (config.dbnum != 0)
len += snprintf(config.prompt+len,sizeof(config.prompt)-len,"[%d]",
config.dbnum);
snprintf(config.prompt+len,sizeof(config.prompt)-len,"> ");
@@ -250,9 +250,9 @@ static sds cliVersion(void) {
version = sdscatprintf(sdsempty(), "%s", REDIS_VERSION);
/* Add git commit and working tree status when available */
if (strtol(redisGitSHA1(),NULL,16)) {
if (strtoll(redisGitSHA1(),NULL,16)) {
version = sdscatprintf(version, " (git:%s", redisGitSHA1());
if (strtol(redisGitDirty(),NULL,10))
if (strtoll(redisGitDirty(),NULL,10))
version = sdscatprintf(version, "-dirty");
version = sdscat(version, ")");
}
@@ -302,6 +302,10 @@ static void cliIntegrateHelp(void) {
* don't already match what we have. */
for (size_t j = 0; j < reply->elements; j++) {
redisReply *entry = reply->element[j];
if (entry->type != REDIS_REPLY_ARRAY || entry->elements < 4 ||
entry->element[0]->type != REDIS_REPLY_STRING ||
entry->element[1]->type != REDIS_REPLY_INTEGER ||
entry->element[3]->type != REDIS_REPLY_INTEGER) return;
char *cmdname = entry->element[0]->str;
int i;
@@ -363,7 +367,7 @@ static void cliOutputGenericHelp(void) {
" \"help <tab>\" to get a list of possible help topics\n"
" \"quit\" to exit\n"
"\n"
"To set redis-cli perferences:\n"
"To set redis-cli preferences:\n"
" \":set hints\" enable online hints\n"
" \":set nohints\" disable online hints\n"
"Set your preferences in ~/.redisclirc\n",
@@ -655,7 +659,6 @@ sds sdscatcolor(sds o, char *s, size_t len, char *color) {
int bold = strstr(color,"bold") != NULL;
int ccode = 37; /* Defaults to white. */
if (strstr(color,"red")) ccode = 31;
else if (strstr(color,"red")) ccode = 31;
else if (strstr(color,"green")) ccode = 32;
else if (strstr(color,"yellow")) ccode = 33;
else if (strstr(color,"blue")) ccode = 34;
@@ -875,8 +878,10 @@ static int cliSendCommand(int argc, char **argv, int repeat) {
output_raw = 0;
if (!strcasecmp(command,"info") ||
(argc >= 2 && !strcasecmp(command,"debug") &&
((!strcasecmp(argv[1],"jemalloc") && !strcasecmp(argv[2],"info")) ||
!strcasecmp(argv[1],"htstats"))) ||
!strcasecmp(argv[1],"htstats")) ||
(argc >= 2 && !strcasecmp(command,"memory") &&
(!strcasecmp(argv[1],"malloc-stats") ||
!strcasecmp(argv[1],"doctor"))) ||
(argc == 2 && !strcasecmp(command,"cluster") &&
(!strcasecmp(argv[1],"nodes") ||
!strcasecmp(argv[1],"info"))) ||
@@ -948,7 +953,7 @@ static int cliSendCommand(int argc, char **argv, int repeat) {
return REDIS_ERR;
} else {
/* Store database number when SELECT was successfully executed. */
if (!strcasecmp(command,"select") && argc == 2) {
if (!strcasecmp(command,"select") && argc == 2 && config.last_cmd_type != REDIS_REPLY_ERROR) {
config.dbnum = atoi(argv[1]);
cliRefreshPrompt();
} else if (!strcasecmp(command,"auth") && argc == 2) {
@@ -1047,7 +1052,7 @@ static int parseOptions(int argc, char **argv) {
config.latency_history = 1;
} else if (!strcmp(argv[i],"--lru-test") && !lastarg) {
config.lru_test_mode = 1;
config.lru_test_sample_size = strtol(argv[++i],NULL,10);
config.lru_test_sample_size = strtoll(argv[++i],NULL,10);
} else if (!strcmp(argv[i],"--slave")) {
config.slave_mode = 1;
} else if (!strcmp(argv[i],"--stat")) {
@@ -1149,6 +1154,12 @@ static void usage(void) {
" --csv Output in CSV format.\n"
" --stat Print rolling stats about server: mem, clients, ...\n"
" --latency Enter a special mode continuously sampling latency.\n"
" If you use this mode in an interactive session it runs\n"
" forever displaying real-time stats. Otherwise if --raw or\n"
" --csv is specified, or if you redirect the output to a non\n"
" TTY, it samples the latency for 1 second (you can use\n"
" -i to change the interval), then produces a single output\n"
" and exits.\n"
" --latency-history Like --latency but tracking latency changes over time.\n"
" Default time interval is 15 sec. Change it using -i.\n"
" --latency-dist Shows latency as a spectrum, requires xterm 256 colors.\n"
@@ -1252,7 +1263,7 @@ static sds *cliSplitArgs(char *line, int *argc) {
}
}
/* Set the CLI perferences. This function is invoked when an interactive
/* Set the CLI preferences. This function is invoked when an interactive
* ":command" is called, or when reading ~/.redisclirc file, in order to
* set user preferences. */
void cliSetPreferences(char **argv, int argc, int interactive) {
@@ -1287,6 +1298,7 @@ void cliLoadPreferences(void) {
if (argc > 0) cliSetPreferences(argv,argc,0);
sdsfreesplitres(argv,argc);
}
fclose(fp);
}
sdsfree(rcfile);
}
@@ -1298,6 +1310,11 @@ static void repl(void) {
int argc;
sds *argv;
/* Initialize the help and, if possible, use the COMMAND command in order
* to retrieve missing entries. */
cliInitHelp();
cliIntegrateHelp();
config.interactive = 1;
linenoiseSetMultiLine(1);
linenoiseSetCompletionCallback(completionCallback);
@@ -1352,9 +1369,10 @@ static void repl(void) {
} else {
PORT_LONGLONG start_time = mstime(), elapsed;
int repeat, skipargs = 0;
char *endptr;
repeat = atoi(argv[0]);
if (argc > 1 && repeat) {
repeat = strtol(argv[0], &endptr, 10);
if (argc > 1 && *endptr == '\0' && repeat) {
skipargs = 1;
} else {
repeat = 1;
@@ -1373,7 +1391,9 @@ static void repl(void) {
}
elapsed = mstime()-start_time;
if (elapsed >= 500) {
if (elapsed >= 500 &&
config.output == OUTPUT_STANDARD)
{
printf("(%.2fs)\n",(double)elapsed/1000);
}
}
@@ -1490,6 +1510,18 @@ static int evalMode(int argc, char **argv) {
* Latency and latency history modes
*--------------------------------------------------------------------------- */
static void latencyModePrint(PORT_LONGLONG min, PORT_LONGLONG max, double avg, PORT_LONGLONG count) {
if (config.output == OUTPUT_STANDARD) {
printf("min: %lld, max: %lld, avg: %.2f (%lld samples)",
min, max, avg, count);
fflush(stdout);
} else if (config.output == OUTPUT_CSV) {
printf("%lld,%lld,%.2f,%lld\n", min, max, avg, count);
} else if (config.output == OUTPUT_RAW) {
printf("%lld %lld %.2f %lld\n", min, max, avg, count);
}
}
#define LATENCY_SAMPLE_RATE 10 /* milliseconds. */
#define LATENCY_HISTORY_DEFAULT_INTERVAL 15000 /* milliseconds. */
static void latencyMode(void) {
@@ -1501,6 +1533,14 @@ static void latencyMode(void) {
double avg;
PORT_LONGLONG history_start = mstime();
/* Set a default for the interval in case of --latency option
* with --raw, --csv or when it is redirected to non tty. */
if (config.interval == 0) {
config.interval = 1000;
} else {
config.interval /= 1000; /* We need to convert to milliseconds. */
}
if (!context) exit(1);
while(1) {
start = mstime();
@@ -1521,9 +1561,19 @@ static void latencyMode(void) {
tot += latency;
avg = (double) tot/count;
}
printf("\x1b[0G\x1b[2Kmin: %lld, max: %lld, avg: %.2f (%lld samples)",
min, max, avg, count);
fflush(stdout);
if (config.output == OUTPUT_STANDARD) {
printf("\x1b[0G\x1b[2K"); /* Clear the line. */
latencyModePrint(min,max,avg,count);
} else {
if (config.latency_history) {
latencyModePrint(min,max,avg,count);
} else if (mstime()-history_start > config.interval) {
latencyModePrint(min,max,avg,count);
exit(0);
}
}
if (config.latency_history && mstime()-history_start > history_interval)
{
printf(" -- %.2f seconds range\n", (float)(mstime()-history_start)/1000);
@@ -2049,8 +2099,13 @@ static void getKeyTypes(redisReply *keys, int *types) {
keys->element[i]->str, context->err, context->errstr);
exit(1);
} else if(reply->type != REDIS_REPLY_STATUS) {
fprintf(stderr, "Invalid reply type (%d) for TYPE on key '%s'!\n",
reply->type, keys->element[i]->str);
if(reply->type == REDIS_REPLY_ERROR) {
fprintf(stderr, "TYPE returned an error: %s\n", reply->str);
} else {
fprintf(stderr,
"Invalid reply type (%d) for TYPE on key '%s'!\n",
reply->type, keys->element[i]->str);
}
exit(1);
}
@@ -2327,7 +2382,7 @@ static void statMode(void) {
if (k == PORT_LONG_MIN) continue;
aux += k;
}
sprintf(buf,"%ld",aux);
sprintf(buf,"%Id",aux); WIN_PORT_FIX /* %ld -> %Id */
printf("%-11s",buf);
/* Used memory */
@@ -2337,23 +2392,23 @@ static void statMode(void) {
/* Clients */
aux = getLongInfoField(reply->str,"connected_clients");
sprintf(buf,"%ld",aux);
sprintf(buf,"%Id",aux); WIN_PORT_FIX /* %ld -> %Id */
printf(" %-8s",buf);
/* Blocked (BLPOPPING) Clients */
aux = getLongInfoField(reply->str,"blocked_clients");
sprintf(buf,"%ld",aux);
sprintf(buf,"%Id",aux); WIN_PORT_FIX /* %ld -> %Id */
printf("%-8s",buf);
/* Requets */
aux = getLongInfoField(reply->str,"total_commands_processed");
sprintf(buf,"%ld (+%ld)",aux,requests == 0 ? 0 : aux-requests);
sprintf(buf,"%Id (+%Id)",aux,requests == 0 ? 0 : aux-requests); WIN_PORT_FIX /* %ld -> %Id */
printf("%-19s",buf);
requests = aux;
/* Connections */
aux = getLongInfoField(reply->str,"total_connections_received");
sprintf(buf,"%ld",aux);
sprintf(buf,"%Id",aux); WIN_PORT_FIX /* %ld -> %Id */
printf(" %-12s",buf);
/* Children */
@@ -2439,7 +2494,7 @@ PORT_LONGLONG powerLawRand(PORT_LONGLONG min, PORT_LONGLONG max, double alpha) {
/* Generates a key name among a set of lru_test_sample_size keys, using
* an 80-20 distribution. */
void LRUTestGenKey(char *buf, size_t buflen) {
snprintf(buf, buflen, "lru:%lld\n",
snprintf(buf, buflen, "lru:%lld",
powerLawRand(1, config.lru_test_sample_size, 6.2));
}
@@ -2461,8 +2516,11 @@ static void LRUTestMode(void) {
while(mstime() - start_cycle < 1000) {
/* Write cycle. */
for (j = 0; j < LRU_CYCLE_PIPELINE_SIZE; j++) {
char val[6];
val[5] = '\0';
for (int i = 0; i < 5; i++) val[i] = 'A'+rand()%('z'-'A');
LRUTestGenKey(key,sizeof(key));
redisAppendCommand(context, "SET %s val",key);
redisAppendCommand(context, "SET %s %s",key,val);
}
for (j = 0; j < LRU_CYCLE_PIPELINE_SIZE; j++)
redisGetReply(context, (void**)&reply);
@@ -2633,11 +2691,6 @@ int main(int argc, char **argv) {
argc -= firstarg;
argv += firstarg;
/* Initialize the help and, if possible, use the COMMAND command in order
* to retrieve missing entries. */
cliInitHelp();
cliIntegrateHelp();
/* Latency mode */
if (config.latency_mode) {
if (cliConnect(0) == REDIS_ERR) exit(1);
src/redis-trib.rb
+7 -3
View File
@@ -56,7 +56,7 @@ end
class ClusterNode
def initialize(addr)
s = addr.split(":")
s = addr.split("@")[0].split(":")
if s.length < 2
puts "Invalid IP or Port (given as #{addr}) - use IP:Port format"
exit 1
@@ -1305,6 +1305,11 @@ class RedisTrib
sleep 1
wait_cluster_join
flush_nodes_config # Useful for the replicas
# Reset the node information, so that when the
# final summary is listed in check_cluster about the newly created cluster
# all the nodes would get properly listed as slaves or masters
reset_nodes
load_cluster_info_from_node(argv[0])
check_cluster
end
@@ -1440,7 +1445,7 @@ class RedisTrib
xputs ">>> Importing data from #{source_addr} to cluster #{argv[1]}"
use_copy = opt['copy']
use_replace = opt['replace']
# Check the existing cluster.
load_cluster_info_from_node(argv[0])
check_cluster
@@ -1664,7 +1669,6 @@ ALLOWED_OPTIONS={
def show_help
puts "Usage: redis-trib <command> <options> <arguments ...>\n\n"
COMMANDS.each{|k,v|
o = ""
puts " #{k.ljust(15)} #{v[2]}"
if ALLOWED_OPTIONS[k]
ALLOWED_OPTIONS[k].each{|optname,has_arg|
src/redisassert.h
+2
View File
@@ -45,7 +45,9 @@
POSIX_ONLY(#include <unistd.h>) /* for _exit() */
#define assert(_e) ((_e)?(void)0 : (_serverAssert(#_e,__FILE__,__LINE__),_exit(1)))
#define panic(...) _serverPanic(__FILE__,__LINE__,__VA_ARGS__),_exit(1)
void _serverAssert(char *estr, char *file, int line);
void _serverPanic(const char *file, int line, const char *msg, ...);
#endif
src/redismodule.h
+358
View File
@@ -0,0 +1,358 @@
#ifndef REDISMODULE_H
#define REDISMODULE_H
#include <sys/types.h>
#include <stdint.h>
#include <stdio.h>
/* ---------------- Defines common between core and modules --------------- */
/* Error status return values. */
#define REDISMODULE_OK 0
#define REDISMODULE_ERR 1
/* API versions. */
#define REDISMODULE_APIVER_1 1
/* API flags and constants */
#define REDISMODULE_READ (1<<0)
#define REDISMODULE_WRITE (1<<1)
#define REDISMODULE_LIST_HEAD 0
#define REDISMODULE_LIST_TAIL 1
/* Key types. */
#define REDISMODULE_KEYTYPE_EMPTY 0
#define REDISMODULE_KEYTYPE_STRING 1
#define REDISMODULE_KEYTYPE_LIST 2
#define REDISMODULE_KEYTYPE_HASH 3
#define REDISMODULE_KEYTYPE_SET 4
#define REDISMODULE_KEYTYPE_ZSET 5
#define REDISMODULE_KEYTYPE_MODULE 6
/* Reply types. */
#define REDISMODULE_REPLY_UNKNOWN -1
#define REDISMODULE_REPLY_STRING 0
#define REDISMODULE_REPLY_ERROR 1
#define REDISMODULE_REPLY_INTEGER 2
#define REDISMODULE_REPLY_ARRAY 3
#define REDISMODULE_REPLY_NULL 4
/* Postponed array length. */
#define REDISMODULE_POSTPONED_ARRAY_LEN -1
/* Expire */
#define REDISMODULE_NO_EXPIRE -1
/* Sorted set API flags. */
#define REDISMODULE_ZADD_XX (1<<0)
#define REDISMODULE_ZADD_NX (1<<1)
#define REDISMODULE_ZADD_ADDED (1<<2)
#define REDISMODULE_ZADD_UPDATED (1<<3)
#define REDISMODULE_ZADD_NOP (1<<4)
/* Hash API flags. */
#define REDISMODULE_HASH_NONE 0
#define REDISMODULE_HASH_NX (1<<0)
#define REDISMODULE_HASH_XX (1<<1)
#define REDISMODULE_HASH_CFIELDS (1<<2)
#define REDISMODULE_HASH_EXISTS (1<<3)
/* A special pointer that we can use between the core and the module to signal
* field deletion, and that is impossible to be a valid pointer. */
#define REDISMODULE_HASH_DELETE ((RedisModuleString*)(PORT_LONG)1)
/* Error messages. */
#define REDISMODULE_ERRORMSG_WRONGTYPE "WRONGTYPE Operation against a key holding the wrong kind of value"
#define REDISMODULE_POSITIVE_INFINITE (1.0/0.0)
#define REDISMODULE_NEGATIVE_INFINITE (-1.0/0.0)
#define REDISMODULE_NOT_USED(V) ((void) V)
/* ------------------------- End of common defines ------------------------ */
#ifndef REDISMODULE_CORE
typedef PORT_LONGLONG mstime_t;
/* Incomplete structures for compiler checks but opaque access. */
typedef struct RedisModuleCtx RedisModuleCtx;
typedef struct RedisModuleKey RedisModuleKey;
typedef struct RedisModuleString RedisModuleString;
typedef struct RedisModuleCallReply RedisModuleCallReply;
typedef struct RedisModuleIO RedisModuleIO;
typedef struct RedisModuleType RedisModuleType;
typedef struct RedisModuleDigest RedisModuleDigest;
typedef struct RedisModuleBlockedClient RedisModuleBlockedClient;
typedef int (*RedisModuleCmdFunc) (RedisModuleCtx *ctx, RedisModuleString **argv, int argc);
typedef void *(*RedisModuleTypeLoadFunc)(RedisModuleIO *rdb, int encver);
typedef void (*RedisModuleTypeSaveFunc)(RedisModuleIO *rdb, void *value);
typedef void (*RedisModuleTypeRewriteFunc)(RedisModuleIO *aof, RedisModuleString *key, void *value);
typedef size_t (*RedisModuleTypeMemUsageFunc)(const void *value);
typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value);
typedef void (*RedisModuleTypeFreeFunc)(void *value);
#define REDISMODULE_TYPE_METHOD_VERSION 1
typedef struct RedisModuleTypeMethods {
uint64_t version;
RedisModuleTypeLoadFunc rdb_load;
RedisModuleTypeSaveFunc rdb_save;
RedisModuleTypeRewriteFunc aof_rewrite;
RedisModuleTypeMemUsageFunc mem_usage;
RedisModuleTypeDigestFunc digest;
RedisModuleTypeFreeFunc free;
} RedisModuleTypeMethods;
#define REDISMODULE_GET_API(name) \
RedisModule_GetApi("RedisModule_" #name, ((void **)&RedisModule_ ## name))
#define REDISMODULE_API_FUNC(x) (*x)
void *REDISMODULE_API_FUNC(RedisModule_Alloc)(size_t bytes);
void *REDISMODULE_API_FUNC(RedisModule_Realloc)(void *ptr, size_t bytes);
void REDISMODULE_API_FUNC(RedisModule_Free)(void *ptr);
void *REDISMODULE_API_FUNC(RedisModule_Calloc)(size_t nmemb, size_t size);
char *REDISMODULE_API_FUNC(RedisModule_Strdup)(const char *str);
int REDISMODULE_API_FUNC(RedisModule_GetApi)(const char *, void *);
int REDISMODULE_API_FUNC(RedisModule_CreateCommand)(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc cmdfunc, const char *strflags, int firstkey, int lastkey, int keystep);
int REDISMODULE_API_FUNC(RedisModule_SetModuleAttribs)(RedisModuleCtx *ctx, const char *name, int ver, int apiver);
int REDISMODULE_API_FUNC(RedisModule_WrongArity)(RedisModuleCtx *ctx);
int REDISMODULE_API_FUNC(RedisModule_ReplyWithLongLong)(RedisModuleCtx *ctx, PORT_LONGLONG ll);
int REDISMODULE_API_FUNC(RedisModule_GetSelectedDb)(RedisModuleCtx *ctx);
int REDISMODULE_API_FUNC(RedisModule_SelectDb)(RedisModuleCtx *ctx, int newid);
void *REDISMODULE_API_FUNC(RedisModule_OpenKey)(RedisModuleCtx *ctx, RedisModuleString *keyname, int mode);
void REDISMODULE_API_FUNC(RedisModule_CloseKey)(RedisModuleKey *kp);
int REDISMODULE_API_FUNC(RedisModule_KeyType)(RedisModuleKey *kp);
size_t REDISMODULE_API_FUNC(RedisModule_ValueLength)(RedisModuleKey *kp);
int REDISMODULE_API_FUNC(RedisModule_ListPush)(RedisModuleKey *kp, int where, RedisModuleString *ele);
RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ListPop)(RedisModuleKey *key, int where);
RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_Call)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...);
const char *REDISMODULE_API_FUNC(RedisModule_CallReplyProto)(RedisModuleCallReply *reply, size_t *len);
void REDISMODULE_API_FUNC(RedisModule_FreeCallReply)(RedisModuleCallReply *reply);
int REDISMODULE_API_FUNC(RedisModule_CallReplyType)(RedisModuleCallReply *reply);
PORT_LONGLONG REDISMODULE_API_FUNC(RedisModule_CallReplyInteger)(RedisModuleCallReply *reply);
size_t REDISMODULE_API_FUNC(RedisModule_CallReplyLength)(RedisModuleCallReply *reply);
RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_CallReplyArrayElement)(RedisModuleCallReply *reply, size_t idx);
RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateString)(RedisModuleCtx *ctx, const char *ptr, size_t len);
RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(RedisModuleCtx *ctx, PORT_LONGLONG ll);
RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str);
RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...);
void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str);
const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleString *str, size_t *len);
int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err);
int REDISMODULE_API_FUNC(RedisModule_ReplyWithSimpleString)(RedisModuleCtx *ctx, const char *msg);
int REDISMODULE_API_FUNC(RedisModule_ReplyWithArray)(RedisModuleCtx *ctx, long len);
void REDISMODULE_API_FUNC(RedisModule_ReplySetArrayLength)(RedisModuleCtx *ctx, long len);
int REDISMODULE_API_FUNC(RedisModule_ReplyWithStringBuffer)(RedisModuleCtx *ctx, const char *buf, size_t len);
int REDISMODULE_API_FUNC(RedisModule_ReplyWithString)(RedisModuleCtx *ctx, RedisModuleString *str);
int REDISMODULE_API_FUNC(RedisModule_ReplyWithNull)(RedisModuleCtx *ctx);
int REDISMODULE_API_FUNC(RedisModule_ReplyWithDouble)(RedisModuleCtx *ctx, double d);
int REDISMODULE_API_FUNC(RedisModule_ReplyWithCallReply)(RedisModuleCtx *ctx, RedisModuleCallReply *reply);
int REDISMODULE_API_FUNC(RedisModule_StringToLongLong)(const RedisModuleString *str, PORT_LONGLONG *ll);
int REDISMODULE_API_FUNC(RedisModule_StringToDouble)(const RedisModuleString *str, double *d);
void REDISMODULE_API_FUNC(RedisModule_AutoMemory)(RedisModuleCtx *ctx);
int REDISMODULE_API_FUNC(RedisModule_Replicate)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...);
int REDISMODULE_API_FUNC(RedisModule_ReplicateVerbatim)(RedisModuleCtx *ctx);
const char *REDISMODULE_API_FUNC(RedisModule_CallReplyStringPtr)(RedisModuleCallReply *reply, size_t *len);
RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromCallReply)(RedisModuleCallReply *reply);
int REDISMODULE_API_FUNC(RedisModule_DeleteKey)(RedisModuleKey *key);
int REDISMODULE_API_FUNC(RedisModule_StringSet)(RedisModuleKey *key, RedisModuleString *str);
char *REDISMODULE_API_FUNC(RedisModule_StringDMA)(RedisModuleKey *key, size_t *len, int mode);
int REDISMODULE_API_FUNC(RedisModule_StringTruncate)(RedisModuleKey *key, size_t newlen);
mstime_t REDISMODULE_API_FUNC(RedisModule_GetExpire)(RedisModuleKey *key);
int REDISMODULE_API_FUNC(RedisModule_SetExpire)(RedisModuleKey *key, mstime_t expire);
int REDISMODULE_API_FUNC(RedisModule_ZsetAdd)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr);
int REDISMODULE_API_FUNC(RedisModule_ZsetIncrby)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr, double *newscore);
int REDISMODULE_API_FUNC(RedisModule_ZsetScore)(RedisModuleKey *key, RedisModuleString *ele, double *score);
int REDISMODULE_API_FUNC(RedisModule_ZsetRem)(RedisModuleKey *key, RedisModuleString *ele, int *deleted);
void REDISMODULE_API_FUNC(RedisModule_ZsetRangeStop)(RedisModuleKey *key);
int REDISMODULE_API_FUNC(RedisModule_ZsetFirstInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex);
int REDISMODULE_API_FUNC(RedisModule_ZsetLastInScoreRange)(RedisModuleKey *key, double min, double max, int minex, int maxex);
int REDISMODULE_API_FUNC(RedisModule_ZsetFirstInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max);
int REDISMODULE_API_FUNC(RedisModule_ZsetLastInLexRange)(RedisModuleKey *key, RedisModuleString *min, RedisModuleString *max);
RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ZsetRangeCurrentElement)(RedisModuleKey *key, double *score);
int REDISMODULE_API_FUNC(RedisModule_ZsetRangeNext)(RedisModuleKey *key);
int REDISMODULE_API_FUNC(RedisModule_ZsetRangePrev)(RedisModuleKey *key);
int REDISMODULE_API_FUNC(RedisModule_ZsetRangeEndReached)(RedisModuleKey *key);
int REDISMODULE_API_FUNC(RedisModule_HashSet)(RedisModuleKey *key, int flags, ...);
int REDISMODULE_API_FUNC(RedisModule_HashGet)(RedisModuleKey *key, int flags, ...);
int REDISMODULE_API_FUNC(RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx);
void REDISMODULE_API_FUNC(RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos);
PORT_ULONGLONG REDISMODULE_API_FUNC(RedisModule_GetClientId)(RedisModuleCtx *ctx);
void *REDISMODULE_API_FUNC(RedisModule_PoolAlloc)(RedisModuleCtx *ctx, size_t bytes);
RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx *ctx, const char *name, int encver, RedisModuleTypeMethods *typemethods);
int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value);
RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key);
void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key);
void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value);
uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io);
void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value);
int64_t REDISMODULE_API_FUNC(RedisModule_LoadSigned)(RedisModuleIO *io);
void REDISMODULE_API_FUNC(RedisModule_EmitAOF)(RedisModuleIO *io, const char *cmdname, const char *fmt, ...);
void REDISMODULE_API_FUNC(RedisModule_SaveString)(RedisModuleIO *io, RedisModuleString *s);
void REDISMODULE_API_FUNC(RedisModule_SaveStringBuffer)(RedisModuleIO *io, const char *str, size_t len);
RedisModuleString *REDISMODULE_API_FUNC(RedisModule_LoadString)(RedisModuleIO *io);
char *REDISMODULE_API_FUNC(RedisModule_LoadStringBuffer)(RedisModuleIO *io, size_t *lenptr);
void REDISMODULE_API_FUNC(RedisModule_SaveDouble)(RedisModuleIO *io, double value);
double REDISMODULE_API_FUNC(RedisModule_LoadDouble)(RedisModuleIO *io);
void REDISMODULE_API_FUNC(RedisModule_SaveFloat)(RedisModuleIO *io, float value);
float REDISMODULE_API_FUNC(RedisModule_LoadFloat)(RedisModuleIO *io);
void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...);
void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...);
int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len);
void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str);
int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b);
RedisModuleCtx *REDISMODULE_API_FUNC(RedisModule_GetContextFromIO)(RedisModuleIO *io);
PORT_LONGLONG REDISMODULE_API_FUNC(RedisModule_Milliseconds)(void);
void REDISMODULE_API_FUNC(RedisModule_DigestAddStringBuffer)(RedisModuleDigest *md, unsigned char *ele, size_t len);
void REDISMODULE_API_FUNC(RedisModule_DigestAddLongLong)(RedisModuleDigest *md, PORT_LONGLONG ele);
void REDISMODULE_API_FUNC(RedisModule_DigestEndSequence)(RedisModuleDigest *md);
/* Experimental APIs */
#ifdef REDISMODULE_EXPERIMENTAL_API
RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClient)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(void*), PORT_LONGLONG timeout_ms);
int REDISMODULE_API_FUNC(RedisModule_UnblockClient)(RedisModuleBlockedClient *bc, void *privdata);
int REDISMODULE_API_FUNC(RedisModule_IsBlockedReplyRequest)(RedisModuleCtx *ctx);
int REDISMODULE_API_FUNC(RedisModule_IsBlockedTimeoutRequest)(RedisModuleCtx *ctx);
void *REDISMODULE_API_FUNC(RedisModule_GetBlockedClientPrivateData)(RedisModuleCtx *ctx);
int REDISMODULE_API_FUNC(RedisModule_AbortBlock)(RedisModuleBlockedClient *bc);
RedisModuleCtx *REDISMODULE_API_FUNC(RedisModule_GetThreadSafeContext)(RedisModuleBlockedClient *bc);
void REDISMODULE_API_FUNC(RedisModule_FreeThreadSafeContext)(RedisModuleCtx *ctx);
void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextLock)(RedisModuleCtx *ctx);
void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextUnlock)(RedisModuleCtx *ctx);
#endif
/* This is included inline inside each Redis module. */
static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) __attribute__((unused));
static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) {
void *getapifuncptr = ((void**)ctx)[0];
RedisModule_GetApi = (int (*)(const char *, void *)) (PORT_ULONG)getapifuncptr;
REDISMODULE_GET_API(Alloc);
REDISMODULE_GET_API(Calloc);
REDISMODULE_GET_API(Free);
REDISMODULE_GET_API(Realloc);
REDISMODULE_GET_API(Strdup);
REDISMODULE_GET_API(CreateCommand);
REDISMODULE_GET_API(SetModuleAttribs);
REDISMODULE_GET_API(WrongArity);
REDISMODULE_GET_API(ReplyWithLongLong);
REDISMODULE_GET_API(ReplyWithError);
REDISMODULE_GET_API(ReplyWithSimpleString);
REDISMODULE_GET_API(ReplyWithArray);
REDISMODULE_GET_API(ReplySetArrayLength);
REDISMODULE_GET_API(ReplyWithStringBuffer);
REDISMODULE_GET_API(ReplyWithString);
REDISMODULE_GET_API(ReplyWithNull);
REDISMODULE_GET_API(ReplyWithCallReply);
REDISMODULE_GET_API(ReplyWithDouble);
REDISMODULE_GET_API(ReplySetArrayLength);
REDISMODULE_GET_API(GetSelectedDb);
REDISMODULE_GET_API(SelectDb);
REDISMODULE_GET_API(OpenKey);
REDISMODULE_GET_API(CloseKey);
REDISMODULE_GET_API(KeyType);
REDISMODULE_GET_API(ValueLength);
REDISMODULE_GET_API(ListPush);
REDISMODULE_GET_API(ListPop);
REDISMODULE_GET_API(StringToLongLong);
REDISMODULE_GET_API(StringToDouble);
REDISMODULE_GET_API(Call);
REDISMODULE_GET_API(CallReplyProto);
REDISMODULE_GET_API(FreeCallReply);
REDISMODULE_GET_API(CallReplyInteger);
REDISMODULE_GET_API(CallReplyType);
REDISMODULE_GET_API(CallReplyLength);
REDISMODULE_GET_API(CallReplyArrayElement);
REDISMODULE_GET_API(CallReplyStringPtr);
REDISMODULE_GET_API(CreateStringFromCallReply);
REDISMODULE_GET_API(CreateString);
REDISMODULE_GET_API(CreateStringFromLongLong);
REDISMODULE_GET_API(CreateStringFromString);
REDISMODULE_GET_API(CreateStringPrintf);
REDISMODULE_GET_API(FreeString);
REDISMODULE_GET_API(StringPtrLen);
REDISMODULE_GET_API(AutoMemory);
REDISMODULE_GET_API(Replicate);
REDISMODULE_GET_API(ReplicateVerbatim);
REDISMODULE_GET_API(DeleteKey);
REDISMODULE_GET_API(StringSet);
REDISMODULE_GET_API(StringDMA);
REDISMODULE_GET_API(StringTruncate);
REDISMODULE_GET_API(GetExpire);
REDISMODULE_GET_API(SetExpire);
REDISMODULE_GET_API(ZsetAdd);
REDISMODULE_GET_API(ZsetIncrby);
REDISMODULE_GET_API(ZsetScore);
REDISMODULE_GET_API(ZsetRem);
REDISMODULE_GET_API(ZsetRangeStop);
REDISMODULE_GET_API(ZsetFirstInScoreRange);
REDISMODULE_GET_API(ZsetLastInScoreRange);
REDISMODULE_GET_API(ZsetFirstInLexRange);
REDISMODULE_GET_API(ZsetLastInLexRange);
REDISMODULE_GET_API(ZsetRangeCurrentElement);
REDISMODULE_GET_API(ZsetRangeNext);
REDISMODULE_GET_API(ZsetRangePrev);
REDISMODULE_GET_API(ZsetRangeEndReached);
REDISMODULE_GET_API(HashSet);
REDISMODULE_GET_API(HashGet);
REDISMODULE_GET_API(IsKeysPositionRequest);
REDISMODULE_GET_API(KeyAtPos);
REDISMODULE_GET_API(GetClientId);
REDISMODULE_GET_API(PoolAlloc);
REDISMODULE_GET_API(CreateDataType);
REDISMODULE_GET_API(ModuleTypeSetValue);
REDISMODULE_GET_API(ModuleTypeGetType);
REDISMODULE_GET_API(ModuleTypeGetValue);
REDISMODULE_GET_API(SaveUnsigned);
REDISMODULE_GET_API(LoadUnsigned);
REDISMODULE_GET_API(SaveSigned);
REDISMODULE_GET_API(LoadSigned);
REDISMODULE_GET_API(SaveString);
REDISMODULE_GET_API(SaveStringBuffer);
REDISMODULE_GET_API(LoadString);
REDISMODULE_GET_API(LoadStringBuffer);
REDISMODULE_GET_API(SaveDouble);
REDISMODULE_GET_API(LoadDouble);
REDISMODULE_GET_API(SaveFloat);
REDISMODULE_GET_API(LoadFloat);
REDISMODULE_GET_API(EmitAOF);
REDISMODULE_GET_API(Log);
REDISMODULE_GET_API(LogIOError);
REDISMODULE_GET_API(StringAppendBuffer);
REDISMODULE_GET_API(RetainString);
REDISMODULE_GET_API(StringCompare);
REDISMODULE_GET_API(GetContextFromIO);
REDISMODULE_GET_API(Milliseconds);
REDISMODULE_GET_API(DigestAddStringBuffer);
REDISMODULE_GET_API(DigestAddLongLong);
REDISMODULE_GET_API(DigestEndSequence);
#ifdef REDISMODULE_EXPERIMENTAL_API
REDISMODULE_GET_API(GetThreadSafeContext);
REDISMODULE_GET_API(FreeThreadSafeContext);
REDISMODULE_GET_API(ThreadSafeContextLock);
REDISMODULE_GET_API(ThreadSafeContextUnlock);
REDISMODULE_GET_API(BlockClient);
REDISMODULE_GET_API(UnblockClient);
REDISMODULE_GET_API(IsBlockedReplyRequest);
REDISMODULE_GET_API(IsBlockedTimeoutRequest);
REDISMODULE_GET_API(GetBlockedClientPrivateData);
REDISMODULE_GET_API(AbortBlock);
#endif
RedisModule_SetModuleAttribs(ctx,name,ver,apiver);
return REDISMODULE_OK;
}
#else
/* Things only defined for the modules core, not exported to modules
* including this file. */
#define RedisModuleString robj
#endif /* REDISMODULE_CORE */
#endif /* REDISMOUDLE_H */
src/replication.c
+488 -163
View File
File diff suppressed because it is too large Load Diff
src/rio.h
+3
View File
@@ -138,6 +138,9 @@ size_t rioWriteBulkString(rio *r, const char *buf, size_t len);
size_t rioWriteBulkLongLong(rio *r, PORT_LONGLONG l);
size_t rioWriteBulkDouble(rio *r, double d);
struct redisObject;
int rioWriteBulkObject(rio *r, struct redisObject *obj);
void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len);
void rioSetAutoSync(rio *r, off_t bytes);
src/scripting.c
+7 -6
View File
@@ -446,6 +446,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
if (j == 10) {
cmdlog = sdscatprintf(cmdlog," ... (%d more)",
c->argc-j-1);
break;
} else {
cmdlog = sdscatlen(cmdlog," ",1);
cmdlog = sdscatsds(cmdlog,c->argv[j]->ptr);
@@ -569,9 +570,9 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) {
reply = sdsnewlen(c->buf,c->bufpos);
c->bufpos = 0;
while(listLength(c->reply)) {
robj *o = listNodeValue(listFirst(c->reply));
sds o = listNodeValue(listFirst(c->reply));
reply = sdscatlen(reply,o->ptr,sdslen(o->ptr));
reply = sdscatsds(reply,o);
listDelNode(c->reply,listFirst(c->reply));
}
}
@@ -872,7 +873,7 @@ void scriptingEnableGlobalsProtection(lua_State *lua) {
s[j++]="end\n";
s[j++]="mt.__index = function (t, n)\n";
s[j++]=" if dbg.getinfo(2) and dbg.getinfo(2, \"S\").what ~= \"C\" then\n";
s[j++]=" error(\"Script attempted to access unexisting global variable '\"..tostring(n)..\"'\", 2)\n";
s[j++]=" error(\"Script attempted to access nonexistent global variable '\"..tostring(n)..\"'\", 2)\n";
s[j++]=" end\n";
s[j++]=" return rawget(t, n)\n";
s[j++]="end\n";
@@ -903,7 +904,6 @@ void scriptingInit(int setup) {
server.lua_caller = NULL;
server.lua_timedout = 0;
server.lua_always_replicate_commands = 0; /* Only DEBUG can change it.*/
server.lua_time_limit = LUA_SCRIPT_TIME_LIMIT;
ldbInit();
}
@@ -1137,7 +1137,7 @@ int redis_math_randomseed (lua_State *L) {
* ------------------------------------------------------------------------- */
/* Define a lua function with the specified function name and body.
* The function name musts be a 42 characters PORT_LONG string, since all the
* The function name musts be a 42 characters long string, since all the
* functions we defined in the Lua context are in the form:
*
* f_<hex sha1 sum>
@@ -2280,7 +2280,7 @@ ldbLog(sdsnew("[e]eval <code> Execute some Lua code (in a different callfr
ldbLog(sdsnew("[r]edis <cmd> Execute a Redis command."));
ldbLog(sdsnew("[m]axlen [len] Trim logged Redis replies and Lua var dumps to len."));
ldbLog(sdsnew(" Specifying zero as <len> means unlimited."));
ldbLog(sdsnew("[a]abort Stop the execution of the script. In sync"));
ldbLog(sdsnew("[a]bort Stop the execution of the script. In sync"));
ldbLog(sdsnew(" mode dataset changes will be retained."));
ldbLog(sdsnew(""));
ldbLog(sdsnew("Debugger functions you can call from Lua scripts:"));
@@ -2394,3 +2394,4 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) {
server.lua_time_start = mstime();
}
}
src/sds.c
+5 -2
View File
@@ -40,6 +40,7 @@
#include <string.h>
#include <ctype.h>
#include <assert.h>
#include <limits.h>
#include "sds.h"
#include "sdsalloc.h"
@@ -66,8 +67,10 @@ static inline char sdsReqType(size_t string_size) {
return SDS_TYPE_8;
if (string_size < 1<<16)
return SDS_TYPE_16;
#if (PORT_LONG_MAX == LLONG_MAX)
if (string_size < 1ll<<32)
return SDS_TYPE_32;
#endif
return SDS_TYPE_64;
}
@@ -511,7 +514,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
/* We try to start using a static buffer for speed.
* If not possible we revert to heap allocation. */
if (buflen > sizeof(staticbuf)) {
buf = IF_WIN32(zcalloc,s_malloc)(buflen);
buf = s_malloc(buflen);
if (buf == NULL) return NULL;
} else {
buflen = sizeof(staticbuf);
@@ -528,7 +531,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
if (buf[buflen-2] != '\0') {
if (buf != staticbuf) s_free(buf);
buflen *= 2;
buf = IF_WIN32(zcalloc,s_malloc)(buflen);
buf = s_malloc(buflen);
if (buf == NULL) return NULL;
continue;
}
src/sds.h
-11
View File
@@ -127,17 +127,6 @@ struct __attribute__ ((__packed__)) sdshdr64 {
#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
#define SDS_TYPE_5 0
#define SDS_TYPE_8 1
#define SDS_TYPE_16 2
#define SDS_TYPE_32 3
#define SDS_TYPE_64 4
#define SDS_TYPE_MASK 7
#define SDS_TYPE_BITS 3
#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (void*)((s)-(sizeof(struct sdshdr##T)));
#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
static inline size_t sdslen(const sds s) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
src/sentinel.c
+29 -16
View File
@@ -431,7 +431,7 @@ void sentinelSimFailureCrash(void);
/* ========================= Dictionary types =============================== */
unsigned int dictSdsHash(const void *key);
uint64_t dictSdsHash(const void *key);
int dictSdsKeyCompare(void *privdata, const void *key1, const void *key2);
void releaseSentinelRedisInstance(sentinelRedisInstance *ri);
@@ -535,7 +535,7 @@ void sentinelIsRunning(void) {
} else if (access(server.configfile,W_OK) == -1) {
serverLog(LL_WARNING,
"Sentinel config file %s is not writable: %s. Exiting...",
server.configfile,strerror(errno));
server.configfile, IF_WIN32(wsa_strerror(errno), strerror(errno)));
exit(1);
}
@@ -1213,11 +1213,18 @@ int sentinelUpdateSentinelAddressInAllMasters(sentinelRedisInstance *ri) {
sentinelRedisInstance *master = dictGetVal(de), *match;
match = getSentinelRedisInstanceByAddrAndRunID(master->sentinels,
NULL,0,ri->runid);
if (match->link->disconnected == 0) {
/* If there is no match, this master does not know about this
* Sentinel, try with the next one. */
if (match == NULL) continue;
/* Disconnect the old links if connected. */
if (match->link->cc != NULL)
instanceLinkCloseConnection(match->link,match->link->cc);
if (match->link->pc != NULL)
instanceLinkCloseConnection(match->link,match->link->pc);
}
if (match == ri) continue; /* Address already updated for it. */
/* Update the address of the matching Sentinel by copying the address
* of the Sentinel object that received the address update. */
releaseSentinelAddr(match->addr);
@@ -1996,14 +2003,14 @@ void sentinelFlushConfig(void) {
server.hz = saved_hz;
if (rewrite_status == -1) goto werr;
if ((fd = open(server.configfile,O_RDONLY,0)) == -1) goto werr; WIN_PORT_FIX /* %lu -> %Iu */
if ((fd = open(server.configfile,O_RDONLY,IF_WIN32(_S_IREAD|_S_IWRITE,0644))) == -1) goto werr;
POSIX_ONLY(if (fsync(fd) == -1) goto werr;)
if (close(fd) == EOF) goto werr;
return;
werr:
if (fd != -1) close(fd);
serverLog(LL_WARNING,"WARNING: Sentinel was not able to save the new configuration on disk!!!: %s", strerror(errno));
serverLog(LL_WARNING,"WARNING: Sentinel was not able to save the new configuration on disk!!!: %s", IF_WIN32(wsa_strerror(errno), strerror(errno)));
}
/* ====================== hiredis connection handling ======================= */
@@ -2205,7 +2212,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) {
if (sdslen(l) >= 32 &&
!memcmp(l,"master_link_down_since_seconds",30))
{
ri->master_link_down_time = strtol(l+31,NULL,10)*1000;
ri->master_link_down_time = strtoll(l+31,NULL,10)*1000;
}
/* role:<role> */
@@ -2723,9 +2730,15 @@ void sentinelSendPeriodicCommands(sentinelRedisInstance *ri) {
/* If this is a slave of a master in O_DOWN condition we start sending
* it INFO every second, instead of the usual SENTINEL_INFO_PERIOD
* period. In this state we want to closely monitor slaves in case they
* are turned into masters by another Sentinel, or by the sysadmin. */
* are turned into masters by another Sentinel, or by the sysadmin.
*
* Similarly we monitor the INFO output more often if the slave reports
* to be disconnected from the master, so that we can have a fresh
* disconnection time figure. */
if ((ri->flags & SRI_SLAVE) &&
(ri->master->flags & (SRI_O_DOWN|SRI_FAILOVER_IN_PROGRESS))) {
((ri->master->flags & (SRI_O_DOWN|SRI_FAILOVER_IN_PROGRESS)) ||
(ri->master_link_down_time != 0)))
{
info_period = 1000;
} else {
info_period = SENTINEL_INFO_PERIOD;
@@ -3380,8 +3393,8 @@ void sentinelInfoCommand(client *c) {
"sentinel_masters:%Iu\r\n" WIN_PORT_FIX /* %lu -> %Iu */
"sentinel_tilt:%d\r\n"
"sentinel_running_scripts:%d\r\n"
"sentinel_scripts_queue_length:%ld\r\n"
"sentinel_simulate_failure_flags:%lu\r\n",
"sentinel_scripts_queue_length:%Id\r\n" WIN_PORT_FIX /* %ld -> %Id */
"sentinel_simulate_failure_flags:%Iu\r\n", WIN_PORT_FIX /* %lu -> %Iu */
dictSize(sentinel.masters),
sentinel.tilt,
sentinel.running_scripts,
@@ -3778,15 +3791,15 @@ struct sentinelLeader {
/* Helper function for sentinelGetLeader, increment the counter
* relative to the specified runid. */
int sentinelLeaderIncr(dict *counters, char *runid) {
dictEntry *de = dictFind(counters,runid);
dictEntry *existing, *de;
uint64_t oldval;
if (de) {
oldval = dictGetUnsignedIntegerVal(de);
dictSetUnsignedIntegerVal(de,oldval+1);
de = dictAddRaw(counters,runid,&existing);
if (existing) {
oldval = dictGetUnsignedIntegerVal(existing);
dictSetUnsignedIntegerVal(existing,oldval+1);
return (int)oldval+1; WIN_PORT_FIX /* cast (int) */
} else {
de = dictAddRaw(counters,runid);
serverAssert(de != NULL);
dictSetUnsignedIntegerVal(de,1);
return 1;
src/server.c
+412 -604
View File
File diff suppressed because it is too large Load Diff
src/server.h
+521 -155
View File
File diff suppressed because it is too large Load Diff
src/siphash.c
+360
View File
@@ -0,0 +1,360 @@
/*
SipHash reference C implementation
Copyright (c) 2012-2016 Jean-Philippe Aumasson
<jeanphilippe.aumasson@gmail.com>
Copyright (c) 2012-2014 Daniel J. Bernstein <djb@cr.yp.to>
Copyright (c) 2017 Salvatore Sanfilippo <antirez@gmail.com>
To the extent possible under law, the author(s) have dedicated all copyright
and related and neighboring rights to this software to the public domain
worldwide. This software is distributed without any warranty.
You should have received a copy of the CC0 Public Domain Dedication along
with this software. If not, see
<http://creativecommons.org/publicdomain/zero/1.0/>.
----------------------------------------------------------------------------
This version was modified by Salvatore Sanfilippo <antirez@gmail.com>
in the following ways:
1. We use SipHash 1-2. This is not believed to be as strong as the
suggested 2-4 variant, but AFAIK there are not trivial attacks
against this reduced-rounds version, and it runs at the same speed
as Murmurhash2 that we used previously, why the 2-4 variant slowed
down Redis by a 4% figure more or less.
2. Hard-code rounds in the hope the compiler can optimize it more
in this raw from. Anyway we always want the standard 2-4 variant.
3. Modify the prototype and implementation so that the function directly
returns an uint64_t value, the hash itself, instead of receiving an
output buffer. This also means that the output size is set to 8 bytes
and the 16 bytes output code handling was removed.
4. Provide a case insensitive variant to be used when hashing strings that
must be considered identical by the hash table regardless of the case.
If we don't have directly a case insensitive hash function, we need to
perform a text transformation in some temporary buffer, which is costly.
5. Remove debugging code.
6. Modified the original test.c file to be a stand-alone function testing
the function in the new form (returing an uint64_t) using just the
relevant test vector.
*/
#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
/* Fast tolower() alike function that does not care about locale
* but just returns a-z insetad of A-Z. */
int siptlw(int c) {
if (c >= 'A' && c <= 'Z') {
return c+('a'-'A');
} else {
return c;
}
}
/* Test of the CPU is Little Endian and supports not aligned accesses.
* Two interesting conditions to speedup the function that happen to be
* in most of x86 servers. */
#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__)
#define UNALIGNED_LE_CPU
#endif
#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
#define U32TO8_LE(p, v) \
(p)[0] = (uint8_t)((v)); \
(p)[1] = (uint8_t)((v) >> 8); \
(p)[2] = (uint8_t)((v) >> 16); \
(p)[3] = (uint8_t)((v) >> 24);
#define U64TO8_LE(p, v) \
U32TO8_LE((p), (uint32_t)((v))); \
U32TO8_LE((p) + 4, (uint32_t)((v) >> 32));
#ifdef UNALIGNED_LE_CPU
#define U8TO64_LE(p) (*((uint64_t*)(p)))
#else
#define U8TO64_LE(p) \
(((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \
((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \
((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \
((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56))
#endif
#define U8TO64_LE_NOCASE(p) \
(((uint64_t)(siptlw((p)[0]))) | \
((uint64_t)(siptlw((p)[1])) << 8) | \
((uint64_t)(siptlw((p)[2])) << 16) | \
((uint64_t)(siptlw((p)[3])) << 24) | \
((uint64_t)(siptlw((p)[4])) << 32) | \
((uint64_t)(siptlw((p)[5])) << 40) | \
((uint64_t)(siptlw((p)[6])) << 48) | \
((uint64_t)(siptlw((p)[7])) << 56))
#define SIPROUND \
do { \
v0 += v1; \
v1 = ROTL(v1, 13); \
v1 ^= v0; \
v0 = ROTL(v0, 32); \
v2 += v3; \
v3 = ROTL(v3, 16); \
v3 ^= v2; \
v0 += v3; \
v3 = ROTL(v3, 21); \
v3 ^= v0; \
v2 += v1; \
v1 = ROTL(v1, 17); \
v1 ^= v2; \
v2 = ROTL(v2, 32); \
} while (0)
uint64_t siphash(const uint8_t *in, const size_t inlen, const uint8_t *k) {
#ifndef UNALIGNED_LE_CPU
uint64_t hash;
uint8_t *out = (uint8_t*) &hash;
#endif
uint64_t v0 = 0x736f6d6570736575ULL;
uint64_t v1 = 0x646f72616e646f6dULL;
uint64_t v2 = 0x6c7967656e657261ULL;
uint64_t v3 = 0x7465646279746573ULL;
uint64_t k0 = U8TO64_LE(k);
uint64_t k1 = U8TO64_LE(k + 8);
uint64_t m;
const uint8_t *end = in + inlen - (inlen % sizeof(uint64_t));
const int left = inlen & 7;
uint64_t b = ((uint64_t)inlen) << 56;
v3 ^= k1;
v2 ^= k0;
v1 ^= k1;
v0 ^= k0;
for (; in != end; in += 8) {
m = U8TO64_LE(in);
v3 ^= m;
SIPROUND;
v0 ^= m;
}
switch (left) {
case 7: b |= ((uint64_t)in[6]) << 48;
case 6: b |= ((uint64_t)in[5]) << 40;
case 5: b |= ((uint64_t)in[4]) << 32;
case 4: b |= ((uint64_t)in[3]) << 24;
case 3: b |= ((uint64_t)in[2]) << 16;
case 2: b |= ((uint64_t)in[1]) << 8;
case 1: b |= ((uint64_t)in[0]); break;
case 0: break;
}
v3 ^= b;
SIPROUND;
v0 ^= b;
v2 ^= 0xff;
SIPROUND;
SIPROUND;
b = v0 ^ v1 ^ v2 ^ v3;
#ifndef UNALIGNED_LE_CPU
U64TO8_LE(out, b);
return hash;
#else
return b;
#endif
}
uint64_t siphash_nocase(const uint8_t *in, const size_t inlen, const uint8_t *k)
{
#ifndef UNALIGNED_LE_CPU
uint64_t hash;
uint8_t *out = (uint8_t*) &hash;
#endif
uint64_t v0 = 0x736f6d6570736575ULL;
uint64_t v1 = 0x646f72616e646f6dULL;
uint64_t v2 = 0x6c7967656e657261ULL;
uint64_t v3 = 0x7465646279746573ULL;
uint64_t k0 = U8TO64_LE(k);
uint64_t k1 = U8TO64_LE(k + 8);
uint64_t m;
const uint8_t *end = in + inlen - (inlen % sizeof(uint64_t));
const int left = inlen & 7;
uint64_t b = ((uint64_t)inlen) << 56;
v3 ^= k1;
v2 ^= k0;
v1 ^= k1;
v0 ^= k0;
for (; in != end; in += 8) {
m = U8TO64_LE_NOCASE(in);
v3 ^= m;
SIPROUND;
v0 ^= m;
}
switch (left) {
case 7: b |= ((uint64_t)siptlw(in[6])) << 48;
case 6: b |= ((uint64_t)siptlw(in[5])) << 40;
case 5: b |= ((uint64_t)siptlw(in[4])) << 32;
case 4: b |= ((uint64_t)siptlw(in[3])) << 24;
case 3: b |= ((uint64_t)siptlw(in[2])) << 16;
case 2: b |= ((uint64_t)siptlw(in[1])) << 8;
case 1: b |= ((uint64_t)siptlw(in[0])); break;
case 0: break;
}
v3 ^= b;
SIPROUND;
v0 ^= b;
v2 ^= 0xff;
SIPROUND;
SIPROUND;
b = v0 ^ v1 ^ v2 ^ v3;
#ifndef UNALIGNED_LE_CPU
U64TO8_LE(out, b);
return hash;
#else
return b;
#endif
}
/* --------------------------------- TEST ------------------------------------ */
#ifdef SIPHASH_TEST
const uint8_t vectors_sip64[64][8] = {
{ 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, },
{ 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, },
{ 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, },
{ 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, },
{ 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, },
{ 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, },
{ 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, },
{ 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, },
{ 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, },
{ 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, },
{ 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, },
{ 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, },
{ 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, },
{ 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, },
{ 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, },
{ 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, },
{ 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, },
{ 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, },
{ 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, },
{ 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, },
{ 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, },
{ 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, },
{ 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, },
{ 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, },
{ 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, },
{ 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, },
{ 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, },
{ 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, },
{ 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, },
{ 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, },
{ 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, },
{ 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, },
{ 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, },
{ 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, },
{ 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, },
{ 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, },
{ 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, },
{ 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, },
{ 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, },
{ 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, },
{ 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, },
{ 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, },
{ 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, },
{ 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, },
{ 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, },
{ 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, },
{ 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, },
{ 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, },
{ 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, },
{ 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, },
{ 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, },
{ 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, },
{ 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, },
{ 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, },
{ 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, },
{ 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, },
{ 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, },
{ 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, },
{ 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, },
{ 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, },
{ 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, },
{ 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, },
{ 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, },
{ 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, },
};
/* Test siphash using a test vector. Returns 0 if the function passed
* all the tests, otherwise 1 is returned.
*
* IMPORTANT: The test vector is for SipHash 2-4. Before running
* the test revert back the siphash() function to 2-4 rounds since
* now it uses 1-2 rounds. */
int siphash_test(void) {
uint8_t in[64], k[16];
int i;
int fails = 0;
for (i = 0; i < 16; ++i)
k[i] = i;
for (i = 0; i < 64; ++i) {
in[i] = i;
uint64_t hash = siphash(in, i, k);
const uint8_t *v = NULL;
v = (uint8_t *)vectors_sip64;
if (memcmp(&hash, v + (i * 8), 8)) {
/* printf("fail for %d bytes\n", i); */
fails++;
}
}
/* Run a few basic tests with the case insensitive version. */
uint64_t h1, h2;
h1 = siphash((uint8_t*)"hello world",11,(uint8_t*)"1234567812345678");
h2 = siphash_nocase((uint8_t*)"hello world",11,(uint8_t*)"1234567812345678");
if (h1 != h2) fails++;
h1 = siphash((uint8_t*)"hello world",11,(uint8_t*)"1234567812345678");
h2 = siphash_nocase((uint8_t*)"HELLO world",11,(uint8_t*)"1234567812345678");
if (h1 != h2) fails++;
h1 = siphash((uint8_t*)"HELLO world",11,(uint8_t*)"1234567812345678");
h2 = siphash_nocase((uint8_t*)"HELLO world",11,(uint8_t*)"1234567812345678");
if (h1 == h2) fails++;
if (!fails) return 0;
return 1;
}
int main(void) {
if (siphash_test() == 0) {
printf("SipHash test: OK\n");
return 0;
} else {
printf("SipHash test: FAILED\n");
return 1;
}
}
#endif
src/slowlog.c
+20 -6
View File
@@ -45,7 +45,7 @@
/* Create a new slowlog entry.
* Incrementing the ref count of all the objects retained is up to
* this function. */
slowlogEntry *slowlogCreateEntry(robj **argv, int argc, PORT_LONGLONG duration) {
slowlogEntry *slowlogCreateEntry(client *c, robj **argv, int argc, PORT_LONGLONG duration) {
slowlogEntry *se = zmalloc(sizeof(*se));
int j, slargc = argc;
@@ -72,15 +72,24 @@ slowlogEntry *slowlogCreateEntry(robj **argv, int argc, PORT_LONGLONG duration)
(PORT_ULONG)
sdslen(argv[j]->ptr) - SLOWLOG_ENTRY_MAX_STRING);
se->argv[j] = createObject(OBJ_STRING,s);
} else {
} else if (argv[j]->refcount == OBJ_SHARED_REFCOUNT) {
se->argv[j] = argv[j];
incrRefCount(argv[j]);
} else {
/* Here we need to dupliacate the string objects composing the
* argument vector of the command, because those may otherwise
* end shared with string objects stored into keys. Having
* shared objects between any part of Redis, and the data
* structure holding the data, is a problem: FLUSHALL ASYNC
* may release the shared string object and create a race. */
se->argv[j] = dupStringObject(argv[j]);
}
}
}
se->time = time(NULL);
se->duration = duration;
se->id = server.slowlog_entry_id++;
se->peerid = sdsnew(getClientPeerId(c));
se->cname = c->name ? sdsnew(c->name->ptr) : sdsempty();
return se;
}
@@ -95,6 +104,8 @@ void slowlogFreeEntry(void *septr) {
for (j = 0; j < se->argc; j++)
decrRefCount(se->argv[j]);
zfree(se->argv);
sdsfree(se->peerid);
sdsfree(se->cname);
zfree(se);
}
@@ -109,10 +120,11 @@ void slowlogInit(void) {
/* Push a new entry into the slow log.
* This function will make sure to trim the slow log accordingly to the
* configured max length. */
void slowlogPushEntryIfNeeded(robj **argv, int argc, PORT_LONGLONG duration) {
void slowlogPushEntryIfNeeded(client *c, robj **argv, int argc, PORT_LONGLONG duration) {
if (server.slowlog_log_slower_than < 0) return; /* Slowlog disabled */
if (duration >= server.slowlog_log_slower_than)
listAddNodeHead(server.slowlog,slowlogCreateEntry(argv,argc,duration));
listAddNodeHead(server.slowlog,
slowlogCreateEntry(c,argv,argc,duration));
/* Remove old entries if needed. */
while (listLength(server.slowlog) > server.slowlog_max_len)
@@ -152,13 +164,15 @@ void slowlogCommand(client *c) {
int j;
se = ln->value;
addReplyMultiBulkLen(c,4);
addReplyMultiBulkLen(c,6);
addReplyLongLong(c,se->id);
addReplyLongLong(c,se->time);
addReplyLongLong(c,se->duration);
addReplyMultiBulkLen(c,se->argc);
for (j = 0; j < se->argc; j++)
addReplyBulk(c,se->argv[j]);
addReplyBulkCBuffer(c,se->peerid,sdslen(se->peerid));
addReplyBulkCBuffer(c,se->cname,sdslen(se->cname));
sent++;
}
setDeferredMultiBulkLength(c,totentries,sent);
src/slowlog.h
+4 -2
View File
@@ -35,13 +35,15 @@ typedef struct slowlogEntry {
robj **argv;
int argc;
PORT_LONGLONG id; /* Unique entry identifier. */
PORT_LONGLONG duration; /* Time spent by the query, in nanoseconds. */
PORT_LONGLONG duration; /* Time spent by the query, in microseconds. */
time_t time; /* Unix time at which the query was executed. */
sds cname; /* Client name. */
sds peerid; /* Client network address. */
} slowlogEntry;
/* Exported API */
void slowlogInit(void);
void slowlogPushEntryIfNeeded(robj **argv, int argc, PORT_LONGLONG duration);
void slowlogPushEntryIfNeeded(client *c, robj **argv, int argc, PORT_LONGLONG duration);
/* Exported commands */
void slowlogCommand(client *c);
src/sort.c
+15 -13
View File
@@ -112,9 +112,9 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) {
if (fieldobj) {
if (o->type != OBJ_HASH) goto noobj;
/* Retrieve value from hash by the field name. This operation
* already increases the refcount of the returned object. */
o = hashTypeGetObject(o, fieldobj);
/* Retrieve value from hash by the field name. The returend object
* is a new object with refcount already incremented. */
o = hashTypeGetValueObject(o, fieldobj->ptr);
} else {
if (o->type != OBJ_STRING) goto noobj;
@@ -380,9 +380,9 @@ void sortCommand(client *c) {
listTypeReleaseIterator(li);
} else if (sortval->type == OBJ_SET) {
setTypeIterator *si = setTypeInitIterator(sortval);
robj *ele;
while((ele = setTypeNextObject(si)) != NULL) {
vector[j].obj = ele;
sds sdsele;
while((sdsele = setTypeNextObject(si)) != NULL) {
vector[j].obj = createObject(OBJ_STRING,sdsele);
vector[j].u.score = 0;
vector[j].u.cmpobj = NULL;
j++;
@@ -399,7 +399,7 @@ void sortCommand(client *c) {
zset *zs = sortval->ptr;
zskiplist *zsl = zs->zsl;
zskiplistNode *ln;
robj *ele;
sds sdsele;
int rangelen = vectorlen;
/* Check if starting point is trivial, before doing log(N) lookup. */
@@ -417,8 +417,8 @@ void sortCommand(client *c) {
while(rangelen--) {
serverAssertWithInfo(c,sortval,ln != NULL);
ele = ln->obj;
vector[j].obj = ele;
sdsele = ln->ele;
vector[j].obj = createStringObject(sdsele,sdslen(sdsele));
vector[j].u.score = 0;
vector[j].u.cmpobj = NULL;
j++;
@@ -431,9 +431,11 @@ void sortCommand(client *c) {
dict *set = ((zset*)sortval->ptr)->dict;
dictIterator *di;
dictEntry *setele;
sds sdsele;
di = dictGetIterator(set);
while((setele = dictNext(di)) != NULL) {
vector[j].obj = dictGetKey(setele);
sdsele = dictGetKey(setele);
vector[j].obj = createStringObject(sdsele,sdslen(sdsele));
vector[j].u.score = 0;
vector[j].u.cmpobj = NULL;
j++;
@@ -577,9 +579,9 @@ void sortCommand(client *c) {
}
/* Cleanup */
if (sortval->type == OBJ_LIST || sortval->type == OBJ_SET)
for (j = 0; j < vectorlen; j++)
decrRefCount(vector[j].obj);
for (j = 0; j < vectorlen; j++)
decrRefCount(vector[j].obj);
decrRefCount(sortval);
listRelease(operations);
for (j = 0; j < vectorlen; j++) {
src/t_hash.c
+212 -189
View File
@@ -52,17 +52,9 @@ void hashTypeTryConversion(robj *o, robj **argv, int start, int end) {
}
}
/* Encode given objects in-place when the hash uses a dict. */
void hashTypeTryObjectEncoding(robj *subject, robj **o1, robj **o2) {
if (subject->encoding == OBJ_ENCODING_HT) {
if (o1) *o1 = tryObjectEncoding(*o1);
if (o2) *o2 = tryObjectEncoding(*o2);
}
}
/* Get the value from a ziplist encoded hash, identified by field.
* Returns -1 when the field cannot be found. */
int hashTypeGetFromZiplist(robj *o, robj *field,
int hashTypeGetFromZiplist(robj *o, sds field,
unsigned char **vstr,
unsigned int *vlen,
PORT_LONGLONG *vll)
@@ -72,12 +64,10 @@ int hashTypeGetFromZiplist(robj *o, robj *field,
serverAssert(o->encoding == OBJ_ENCODING_ZIPLIST);
field = getDecodedObject(field);
zl = o->ptr;
fptr = ziplistIndex(zl, ZIPLIST_HEAD);
if (fptr != NULL) {
fptr = ziplistFind(fptr, field->ptr, (unsigned int)sdslen(field->ptr), 1); WIN_PORT_FIX /* cast (unsigned int) */
fptr = ziplistFind(fptr, (unsigned char*)field, (unsigned int)sdslen(field), 1); WIN_PORT_FIX /* cast (unsigned int) */
if (fptr != NULL) {
/* Grab pointer to the value (fptr points to the field) */
vptr = ziplistNext(zl, fptr);
@@ -85,8 +75,6 @@ int hashTypeGetFromZiplist(robj *o, robj *field,
}
}
decrRefCount(field);
if (vptr != NULL) {
ret = ziplistGet(vptr, vstr, vlen, vll);
serverAssert(ret);
@@ -97,56 +85,63 @@ int hashTypeGetFromZiplist(robj *o, robj *field,
}
/* Get the value from a hash table encoded hash, identified by field.
* Returns -1 when the field cannot be found. */
int hashTypeGetFromHashTable(robj *o, robj *field, robj **value) {
* Returns NULL when the field cannot be found, otherwise the SDS value
* is returned. */
sds hashTypeGetFromHashTable(robj *o, sds field) {
dictEntry *de;
serverAssert(o->encoding == OBJ_ENCODING_HT);
de = dictFind(o->ptr, field);
if (de == NULL) return -1;
*value = dictGetVal(de);
return 0;
if (de == NULL) return NULL;
return dictGetVal(de);
}
/* Higher level function of hashTypeGet*() that always returns a Redis
* object (either new or with refcount incremented), so that the caller
* can retain a reference or call decrRefCount after the usage.
/* Higher level function of hashTypeGet*() that returns the hash value
* associated with the specified field. If the field is found C_OK
* is returned, otherwise C_ERR. The returned object is returned by
* reference in either *vstr and *vlen if it's returned in string form,
* or stored in *vll if it's returned as a number.
*
* The lower level function can prevent copy on write so it is
* the preferred way of doing read operations. */
robj *hashTypeGetObject(robj *o, robj *field) {
robj *value = NULL;
* If *vll is populated *vstr is set to NULL, so the caller
* can always check the function return by checking the return value
* for C_OK and checking if vll (or vstr) is NULL. */
int hashTypeGetValue(robj *o, sds field, unsigned char **vstr, unsigned int *vlen, PORT_LONGLONG *vll) {
if (o->encoding == OBJ_ENCODING_ZIPLIST) {
unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX;
PORT_LONGLONG vll = LLONG_MAX;
if (hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll) == 0) {
if (vstr) {
value = createStringObject((char*)vstr, vlen);
} else {
value = createStringObjectFromLongLong(vll);
}
}
*vstr = NULL;
if (hashTypeGetFromZiplist(o, field, vstr, vlen, vll) == 0)
return C_OK;
} else if (o->encoding == OBJ_ENCODING_HT) {
robj *aux;
if (hashTypeGetFromHashTable(o, field, &aux) == 0) {
incrRefCount(aux);
value = aux;
sds value;
if ((value = hashTypeGetFromHashTable(o, field)) != NULL) {
*vstr = (unsigned char*) value;
*vlen = sdslen(value);
return C_OK;
}
} else {
serverPanic("Unknown hash encoding");
}
return value;
return C_ERR;
}
/* Like hashTypeGetValue() but returns a Redis object, which is useful for
* interaction with the hash type outside t_hash.c.
* The function returns NULL if the field is not found in the hash. Otherwise
* a newly allocated string object with the value is returned. */
robj *hashTypeGetValueObject(robj *o, sds field) {
unsigned char *vstr;
unsigned int vlen;
PORT_LONGLONG vll;
if (hashTypeGetValue(o,field,&vstr,&vlen,&vll) == C_ERR) return NULL;
if (vstr) return createStringObject((char*)vstr,vlen);
else return createStringObjectFromLongLong(vll);
}
/* Higher level function using hashTypeGet*() to return the length of the
* object associated with the requested field, or 0 if the field does not
* exist. */
size_t hashTypeGetValueLength(robj *o, robj *field) {
size_t hashTypeGetValueLength(robj *o, sds field) {
size_t len = 0;
if (o->encoding == OBJ_ENCODING_ZIPLIST) {
unsigned char *vstr = NULL;
@@ -156,10 +151,10 @@ size_t hashTypeGetValueLength(robj *o, robj *field) {
if (hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll) == 0)
len = vstr ? vlen : sdigits10(vll);
} else if (o->encoding == OBJ_ENCODING_HT) {
robj *aux;
sds aux;
if (hashTypeGetFromHashTable(o, field, &aux) == 0)
len = stringObjectLen(aux);
if ((aux = hashTypeGetFromHashTable(o, field)) != NULL)
len = sdslen(aux);
} else {
serverPanic("Unknown hash encoding");
}
@@ -168,7 +163,7 @@ size_t hashTypeGetValueLength(robj *o, robj *field) {
/* Test if the specified field exists in the given hash. Returns 1 if the field
* exists, and 0 when it doesn't. */
int hashTypeExists(robj *o, robj *field) {
int hashTypeExists(robj *o, sds field) {
if (o->encoding == OBJ_ENCODING_ZIPLIST) {
unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX;
@@ -176,32 +171,44 @@ int hashTypeExists(robj *o, robj *field) {
if (hashTypeGetFromZiplist(o, field, &vstr, &vlen, &vll) == 0) return 1;
} else if (o->encoding == OBJ_ENCODING_HT) {
robj *aux;
if (hashTypeGetFromHashTable(o, field, &aux) == 0) return 1;
if (hashTypeGetFromHashTable(o, field) != NULL) return 1;
} else {
serverPanic("Unknown hash encoding");
}
return 0;
}
/* Add an element, discard the old if the key already exists.
/* Add a new field, overwrite the old with the new value if it already exists.
* Return 0 on insert and 1 on update.
* This function will take care of incrementing the reference count of the
* retained fields and value objects. */
int hashTypeSet(robj *o, robj *field, robj *value) {
*
* By default, the key and value SDS strings are copied if needed, so the
* caller retains ownership of the strings passed. However this behavior
* can be effected by passing appropriate flags (possibly bitwise OR-ed):
*
* HASH_SET_TAKE_FIELD -- The SDS field ownership passes to the function.
* HASH_SET_TAKE_VALUE -- The SDS value ownership passes to the function.
*
* When the flags are used the caller does not need to release the passed
* SDS string(s). It's up to the function to use the string to create a new
* entry or to free the SDS string before returning to the caller.
*
* HASH_SET_COPY corresponds to no flags passed, and means the default
* semantics of copying the values if needed.
*
*/
#define HASH_SET_TAKE_FIELD (1<<0)
#define HASH_SET_TAKE_VALUE (1<<1)
#define HASH_SET_COPY 0
int hashTypeSet(robj *o, sds field, sds value, int flags) {
int update = 0;
if (o->encoding == OBJ_ENCODING_ZIPLIST) {
unsigned char *zl, *fptr, *vptr;
field = getDecodedObject(field);
value = getDecodedObject(value);
zl = o->ptr;
fptr = ziplistIndex(zl, ZIPLIST_HEAD);
if (fptr != NULL) {
fptr = ziplistFind(fptr, field->ptr, (unsigned int)sdslen(field->ptr), 1); WIN_PORT_FIX /* cast (unsigned int) */
fptr = ziplistFind(fptr, (unsigned char*)field, (unsigned int)sdslen(field), 1); WIN_PORT_FIX /* cast (unsigned int) */
if (fptr != NULL) {
/* Grab pointer to the value (fptr points to the field) */
vptr = ziplistNext(zl, fptr);
@@ -212,49 +219,73 @@ int hashTypeSet(robj *o, robj *field, robj *value) {
zl = ziplistDelete(zl, &vptr);
/* Insert new value */
zl = ziplistInsert(zl, vptr, value->ptr, (unsigned int)sdslen(value->ptr)); WIN_PORT_FIX /* cast (unsigned int) */
zl = ziplistInsert(zl, vptr, (unsigned char*)value,
(unsigned int)sdslen(value)); WIN_PORT_FIX /* cast (unsigned int) */
}
}
if (!update) {
/* Push new field/value pair onto the tail of the ziplist */
zl = ziplistPush(zl, field->ptr, (unsigned int)sdslen(field->ptr), ZIPLIST_TAIL); WIN_PORT_FIX /* cast (unsigned int) */
zl = ziplistPush(zl, value->ptr, (unsigned int)sdslen(value->ptr), ZIPLIST_TAIL); WIN_PORT_FIX /* cast (unsigned int) */
zl = ziplistPush(zl, (unsigned char*)field, (unsigned int)sdslen(field), WIN_PORT_FIX /* cast (unsigned int) */
ZIPLIST_TAIL);
zl = ziplistPush(zl, (unsigned char*)value, (unsigned int)sdslen(value), WIN_PORT_FIX /* cast (unsigned int) */
ZIPLIST_TAIL);
}
o->ptr = zl;
decrRefCount(field);
decrRefCount(value);
/* Check if the ziplist needs to be converted to a hash table */
if (hashTypeLength(o) > server.hash_max_ziplist_entries)
hashTypeConvert(o, OBJ_ENCODING_HT);
} else if (o->encoding == OBJ_ENCODING_HT) {
if (dictReplace(o->ptr, field, value)) { /* Insert */
incrRefCount(field);
} else { /* Update */
dictEntry *de = dictFind(o->ptr,field);
if (de) {
sdsfree(dictGetVal(de));
if (flags & HASH_SET_TAKE_VALUE) {
dictGetVal(de) = value;
value = NULL;
} else {
dictGetVal(de) = sdsdup(value);
}
update = 1;
} else {
sds f,v;
if (flags & HASH_SET_TAKE_FIELD) {
f = field;
field = NULL;
} else {
f = sdsdup(field);
}
if (flags & HASH_SET_TAKE_VALUE) {
v = value;
value = NULL;
} else {
v = sdsdup(value);
}
dictAdd(o->ptr,f,v);
}
incrRefCount(value);
} else {
serverPanic("Unknown hash encoding");
}
/* Free SDS strings we did not referenced elsewhere if the flags
* want this function to be responsible. */
if (flags & HASH_SET_TAKE_FIELD && field) sdsfree(field);
if (flags & HASH_SET_TAKE_VALUE && value) sdsfree(value);
return update;
}
/* Delete an element from a hash.
* Return 1 on deleted and 0 on not found. */
int hashTypeDelete(robj *o, robj *field) {
int hashTypeDelete(robj *o, sds field) {
int deleted = 0;
if (o->encoding == OBJ_ENCODING_ZIPLIST) {
unsigned char *zl, *fptr;
field = getDecodedObject(field);
zl = o->ptr;
fptr = ziplistIndex(zl, ZIPLIST_HEAD);
if (fptr != NULL) {
fptr = ziplistFind(fptr, field->ptr, (unsigned int)sdslen(field->ptr), 1); WIN_PORT_FIX /* cast (unsigned int) */
fptr = ziplistFind(fptr, (unsigned char*)field, (unsigned int)sdslen(field), 1); WIN_PORT_FIX /* cast (unsigned int) */
if (fptr != NULL) {
zl = ziplistDelete(zl,&fptr);
zl = ziplistDelete(zl,&fptr);
@@ -262,9 +293,6 @@ int hashTypeDelete(robj *o, robj *field) {
deleted = 1;
}
}
decrRefCount(field);
} else if (o->encoding == OBJ_ENCODING_HT) {
if (dictDelete((dict*)o->ptr, field) == C_OK) {
deleted = 1;
@@ -276,22 +304,20 @@ int hashTypeDelete(robj *o, robj *field) {
} else {
serverPanic("Unknown hash encoding");
}
return deleted;
}
/* Return the number of elements in a hash. */
PORT_ULONG hashTypeLength(robj *o) {
PORT_ULONG length = PORT_ULONG_MAX;
PORT_ULONG hashTypeLength(const robj *o) {
PORT_ULONG length = ULONG_MAX;
if (o->encoding == OBJ_ENCODING_ZIPLIST) {
length = ziplistLen(o->ptr) / 2;
} else if (o->encoding == OBJ_ENCODING_HT) {
length = (PORT_ULONG)dictSize((dict*)o->ptr); WIN_PORT_FIX /* cast (PORT_ULONG) */
length = (PORT_ULONG)dictSize((const dict*)o->ptr); WIN_PORT_FIX /* cast (PORT_ULONG) */
} else {
serverPanic("Unknown hash encoding");
}
return length;
}
@@ -308,15 +334,12 @@ hashTypeIterator *hashTypeInitIterator(robj *subject) {
} else {
serverPanic("Unknown hash encoding");
}
return hi;
}
void hashTypeReleaseIterator(hashTypeIterator *hi) {
if (hi->encoding == OBJ_ENCODING_HT) {
if (hi->encoding == OBJ_ENCODING_HT)
dictReleaseIterator(hi->di);
}
zfree(hi);
}
@@ -378,41 +401,51 @@ void hashTypeCurrentFromZiplist(hashTypeIterator *hi, int what,
}
/* Get the field or value at iterator cursor, for an iterator on a hash value
* encoded as a ziplist. Prototype is similar to `hashTypeGetFromHashTable`. */
void hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what, robj **dst) {
* encoded as a hash table. Prototype is similar to
* `hashTypeGetFromHashTable`. */
sds hashTypeCurrentFromHashTable(hashTypeIterator *hi, int what) {
serverAssert(hi->encoding == OBJ_ENCODING_HT);
if (what & OBJ_HASH_KEY) {
*dst = dictGetKey(hi->de);
return dictGetKey(hi->de);
} else {
*dst = dictGetVal(hi->de);
return dictGetVal(hi->de);
}
}
/* A non copy-on-write friendly but higher level version of hashTypeCurrent*()
* that returns an object with incremented refcount (or a new object). It is up
* to the caller to decrRefCount() the object if no reference is retained. */
robj *hashTypeCurrentObject(hashTypeIterator *hi, int what) {
robj *dst;
/* Higher level function of hashTypeCurrent*() that returns the hash value
* at current iterator position.
*
* The returned element is returned by reference in either *vstr and *vlen if
* it's returned in string form, or stored in *vll if it's returned as
* a number.
*
* If *vll is populated *vstr is set to NULL, so the caller
* can always check the function return by checking the return value
* type checking if vstr == NULL. */
void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, PORT_LONGLONG *vll) {
if (hi->encoding == OBJ_ENCODING_ZIPLIST) {
unsigned char *vstr = NULL;
unsigned int vlen = UINT_MAX;
PORT_LONGLONG vll = LLONG_MAX;
hashTypeCurrentFromZiplist(hi, what, &vstr, &vlen, &vll);
if (vstr) {
dst = createStringObject((char*)vstr, vlen);
} else {
dst = createStringObjectFromLongLong(vll);
}
*vstr = NULL;
hashTypeCurrentFromZiplist(hi, what, vstr, vlen, vll);
} else if (hi->encoding == OBJ_ENCODING_HT) {
hashTypeCurrentFromHashTable(hi, what, &dst);
incrRefCount(dst);
sds ele = hashTypeCurrentFromHashTable(hi, what);
*vstr = (unsigned char*) ele;
*vlen = sdslen(ele);
} else {
serverPanic("Unknown hash encoding");
}
return dst;
}
/* Return the key or value at the current iterator position as a new
* SDS string. */
sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what) {
unsigned char *vstr;
unsigned int vlen;
PORT_LONGLONG vll;
hashTypeCurrentObject(hi,what,&vstr,&vlen,&vll);
if (vstr) return sdsnewlen(vstr,vlen);
return sdsfromlonglong(vll);
}
robj *hashTypeLookupWriteOrCreate(client *c, robj *key) {
@@ -444,26 +477,21 @@ void hashTypeConvertZiplist(robj *o, int enc) {
dict = dictCreate(&hashDictType, NULL);
while (hashTypeNext(hi) != C_ERR) {
robj *field, *value;
sds key, value;
field = hashTypeCurrentObject(hi, OBJ_HASH_KEY);
field = tryObjectEncoding(field);
value = hashTypeCurrentObject(hi, OBJ_HASH_VALUE);
value = tryObjectEncoding(value);
ret = dictAdd(dict, field, value);
key = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_KEY);
value = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE);
ret = dictAdd(dict, key, value);
if (ret != DICT_OK) {
serverLogHexDump(LL_WARNING,"ziplist with dup elements dump",
o->ptr,ziplistBlobLen(o->ptr));
serverAssert(ret == DICT_OK);
serverPanic("Ziplist corruption detected");
}
}
hashTypeReleaseIterator(hi);
zfree(o->ptr);
o->encoding = OBJ_ENCODING_HT;
o->ptr = dict;
} else {
serverPanic("Unknown hash encoding");
}
@@ -483,30 +511,15 @@ void hashTypeConvert(robj *o, int enc) {
* Hash type commands
*----------------------------------------------------------------------------*/
void hsetCommand(client *c) {
int update;
robj *o;
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
hashTypeTryConversion(o,c->argv,2,3);
hashTypeTryObjectEncoding(o,&c->argv[2], &c->argv[3]);
update = hashTypeSet(o,c->argv[2],c->argv[3]);
addReply(c, update ? shared.czero : shared.cone);
signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
server.dirty++;
}
void hsetnxCommand(client *c) {
robj *o;
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
hashTypeTryConversion(o,c->argv,2,3);
if (hashTypeExists(o, c->argv[2])) {
if (hashTypeExists(o, c->argv[2]->ptr)) {
addReply(c, shared.czero);
} else {
hashTypeTryObjectEncoding(o,&c->argv[2], &c->argv[3]);
hashTypeSet(o,c->argv[2],c->argv[3]);
hashTypeSet(o,c->argv[2]->ptr,c->argv[3]->ptr,HASH_SET_COPY);
addReply(c, shared.cone);
signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
@@ -514,8 +527,8 @@ void hsetnxCommand(client *c) {
}
}
void hmsetCommand(client *c) {
int i;
void hsetCommand(client *c) {
int i, created = 0;
robj *o;
if ((c->argc % 2) == 1) {
@@ -525,11 +538,19 @@ void hmsetCommand(client *c) {
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
hashTypeTryConversion(o,c->argv,2,c->argc-1);
for (i = 2; i < c->argc; i += 2) {
hashTypeTryObjectEncoding(o,&c->argv[i], &c->argv[i+1]);
hashTypeSet(o,c->argv[i],c->argv[i+1]);
for (i = 2; i < c->argc; i += 2)
created += !hashTypeSet(o,c->argv[i]->ptr,c->argv[i+1]->ptr,HASH_SET_COPY);
/* HMSET (deprecated) and HSET return value is different. */
char *cmdname = c->argv[0]->ptr;
if (cmdname[1] == 's' || cmdname[1] == 'S') {
/* HSET */
addReplyLongLong(c, created);
} else {
/* HMSET */
addReply(c, shared.ok);
}
addReply(c, shared.ok);
signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id);
server.dirty++;
@@ -537,17 +558,20 @@ void hmsetCommand(client *c) {
void hincrbyCommand(client *c) {
PORT_LONGLONG value, incr, oldvalue;
robj *o, *current, *new;
robj *o;
sds new;
unsigned char *vstr;
unsigned int vlen;
if (getLongLongFromObjectOrReply(c,c->argv[3],&incr,NULL) != C_OK) return;
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
if ((current = hashTypeGetObject(o,c->argv[2])) != NULL) {
if (getLongLongFromObjectOrReply(c,current,&value,
"hash value is not an integer") != C_OK) {
decrRefCount(current);
return;
}
decrRefCount(current);
if (hashTypeGetValue(o,c->argv[2]->ptr,&vstr,&vlen,&value) == C_OK) {
if (vstr) {
if (string2ll((char*)vstr,vlen,&value) == 0) {
addReplyError(c,"hash value is not an integer");
return;
}
} /* Else hashTypeGetValue() already stored it into &value */
} else {
value = 0;
}
@@ -559,10 +583,8 @@ void hincrbyCommand(client *c) {
return;
}
value += incr;
new = createStringObjectFromLongLong(value);
hashTypeTryObjectEncoding(o,&c->argv[2],NULL);
hashTypeSet(o,c->argv[2],new);
decrRefCount(new);
new = sdsfromlonglong(value);
hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE);
addReplyLongLong(c,value);
signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hincrby",c->argv[1],c->db->id);
@@ -571,26 +593,34 @@ void hincrbyCommand(client *c) {
void hincrbyfloatCommand(client *c) {
PORT_LONGDOUBLE value, incr;
robj *o, *current, *new, *aux;
PORT_LONGLONG ll;
robj *o;
sds new;
unsigned char *vstr;
unsigned int vlen;
if (getLongDoubleFromObjectOrReply(c,c->argv[3],&incr,NULL) != C_OK) return;
if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return;
if ((current = hashTypeGetObject(o,c->argv[2])) != NULL) {
if (getLongDoubleFromObjectOrReply(c,current,&value,
"hash value is not a valid float") != C_OK) {
decrRefCount(current);
return;
if (hashTypeGetValue(o,c->argv[2]->ptr,&vstr,&vlen,&ll) == C_OK) {
if (vstr) {
if (string2ld((char*)vstr,vlen,&value) == 0) {
addReplyError(c,"hash value is not a float");
return;
}
} else {
value = (PORT_LONGDOUBLE)ll;
}
decrRefCount(current);
} else {
value = 0;
}
value += incr;
new = createStringObjectFromLongDouble(value,1);
hashTypeTryObjectEncoding(o,&c->argv[2],NULL);
hashTypeSet(o,c->argv[2],new);
addReplyBulk(c,new);
char buf[256];
int len = ld2string(buf,sizeof(buf),value,1);
new = sdsnewlen(buf,len);
hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE);
addReplyBulkCBuffer(c,buf,len);
signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(NOTIFY_HASH,"hincrbyfloat",c->argv[1],c->db->id);
server.dirty++;
@@ -598,14 +628,16 @@ void hincrbyfloatCommand(client *c) {
/* Always replicate HINCRBYFLOAT as an HSET command with the final value
* in order to make sure that differences in float pricision or formatting
* will not create differences in replicas or after an AOF restart. */
robj *aux, *newobj;
aux = createStringObject("HSET",4);
newobj = createRawStringObject(buf,len);
rewriteClientCommandArgument(c,0,aux);
decrRefCount(aux);
rewriteClientCommandArgument(c,3,new);
decrRefCount(new);
rewriteClientCommandArgument(c,3,newobj);
decrRefCount(newobj);
}
static void addHashFieldToReply(client *c, robj *o, robj *field) {
static void addHashFieldToReply(client *c, robj *o, sds field) {
int ret;
if (o == NULL) {
@@ -630,15 +662,11 @@ static void addHashFieldToReply(client *c, robj *o, robj *field) {
}
} else if (o->encoding == OBJ_ENCODING_HT) {
robj *value;
ret = hashTypeGetFromHashTable(o, field, &value);
if (ret < 0) {
sds value = hashTypeGetFromHashTable(o, field);
if (value == NULL)
addReply(c, shared.nullbulk);
} else {
addReplyBulk(c, value);
}
else
addReplyBulkCBuffer(c, value, sdslen(value));
} else {
serverPanic("Unknown hash encoding");
}
@@ -650,7 +678,7 @@ void hgetCommand(client *c) {
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.nullbulk)) == NULL ||
checkType(c,o,OBJ_HASH)) return;
addHashFieldToReply(c, o, c->argv[2]);
addHashFieldToReply(c, o, c->argv[2]->ptr);
}
void hmgetCommand(client *c) {
@@ -667,7 +695,7 @@ void hmgetCommand(client *c) {
addReplyMultiBulkLen(c, c->argc-2);
for (i = 2; i < c->argc; i++) {
addHashFieldToReply(c, o, c->argv[i]);
addHashFieldToReply(c, o, c->argv[i]->ptr);
}
}
@@ -679,7 +707,7 @@ void hdelCommand(client *c) {
checkType(c,o,OBJ_HASH)) return;
for (j = 2; j < c->argc; j++) {
if (hashTypeDelete(o,c->argv[j])) {
if (hashTypeDelete(o,c->argv[j]->ptr)) {
deleted++;
if (hashTypeLength(o) == 0) {
dbDelete(c->db,c->argv[1]);
@@ -713,7 +741,7 @@ void hstrlenCommand(client *c) {
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,o,OBJ_HASH)) return;
addReplyLongLong(c,hashTypeGetValueLength(o,c->argv[2]));
addReplyLongLong(c,hashTypeGetValueLength(o,c->argv[2]->ptr));
}
static void addHashIteratorCursorToReply(client *c, hashTypeIterator *hi, int what) {
@@ -723,18 +751,13 @@ static void addHashIteratorCursorToReply(client *c, hashTypeIterator *hi, int wh
PORT_LONGLONG vll = LLONG_MAX;
hashTypeCurrentFromZiplist(hi, what, &vstr, &vlen, &vll);
if (vstr) {
if (vstr)
addReplyBulkCBuffer(c, vstr, vlen);
} else {
else
addReplyBulkLongLong(c, vll);
}
} else if (hi->encoding == OBJ_ENCODING_HT) {
robj *value;
hashTypeCurrentFromHashTable(hi, what, &value);
addReplyBulk(c, value);
sds value = hashTypeCurrentFromHashTable(hi, what);
addReplyBulkCBuffer(c, value, sdslen(value));
} else {
serverPanic("Unknown hash encoding");
}
@@ -788,7 +811,7 @@ void hexistsCommand(client *c) {
if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL ||
checkType(c,o,OBJ_HASH)) return;
addReply(c, hashTypeExists(o,c->argv[2]) ? shared.cone : shared.czero);
addReply(c, hashTypeExists(o,c->argv[2]->ptr) ? shared.cone : shared.czero);
}
void hscanCommand(client *c) {

Some files were not shown because too many files have changed in this diff Show More