Redis/redis
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Compare commits

base: Redis/redis:4.0.3
Branches Tags
Redis/redis:6.2 Redis/redis:unstable Redis/redis:7.2 Redis/redis:7.4 Redis/redis:8.0
Redis/redis:7.2.10 Redis/redis:6.2.19 Redis/redis:7.4.5 Redis/redis:8.0.3 Redis/redis:8.2-rc1 Redis/redis:8.2-rc1-int Redis/redis:8.2-m01 Redis/redis:8.2-m01-int2 Redis/redis:8.2-m01-int Redis/redis:8.0.2 Redis/redis:7.2.9 Redis/redis:7.4.4 Redis/redis:8.0.1 Redis/redis:8.0.1-int Redis/redis:8.0.0 Redis/redis:8.0-rc2-int Redis/redis:7.4.3 Redis/redis:7.2.8 Redis/redis:6.2.18 Redis/redis:8.0-rc1 Redis/redis:8.0-rc1-int2 Redis/redis:8.0-rc1-int Redis/redis:8.0-m04 Redis/redis:8.0-m04-int Redis/redis:8.0-m03 Redis/redis:7.2.7 Redis/redis:7.4.2 Redis/redis:6.2.17 Redis/redis:8.0-m02 Redis/redis:6.2.16 Redis/redis:7.2.6 Redis/redis:7.4.1 Redis/redis:8.0-m01 Redis/redis:7.4.0 Redis/redis:7.4-rc2 Redis/redis:7.4-rc1 Redis/redis:7.2.5 Redis/redis:7.2.4 Redis/redis:7.0.15 Redis/redis:7.2.3 Redis/redis:7.2.2 Redis/redis:7.0.14 Redis/redis:6.2.15 Redis/redis:6.2.14 Redis/redis:7.2.1 Redis/redis:7.0.13 Redis/redis:7.2.0 Redis/redis:7.2-rc3 Redis/redis:7.0.12 Redis/redis:6.2.13 Redis/redis:6.0.20 Redis/redis:7.2-rc2 Redis/redis:6.0.19 Redis/redis:6.2.12 Redis/redis:7.0.11 Redis/redis:7.2-rc1 Redis/redis:7.0.10 Redis/redis:7.0.9 Redis/redis:6.2.11 Redis/redis:6.0.18 Redis/redis:6.2.10 Redis/redis:6.0.17 Redis/redis:6.2.9 Redis/redis:7.0.8 Redis/redis:7.0.7 Redis/redis:7.0.6 Redis/redis:6.2.8 Redis/redis:7.0.5 Redis/redis:7.0.4 Redis/redis:7.0.3 Redis/redis:7.0.2 Redis/redis:7.0.1 Redis/redis:7.0.0 Redis/redis:6.2.7 Redis/redis:7.0-rc3 Redis/redis:7.0-rc2 Redis/redis:7.0-rc1 Redis/redis:6.2.6 Redis/redis:6.0.16 Redis/redis:5.0.14 Redis/redis:6.2.5 Redis/redis:5.0.13 Redis/redis:6.0.15 Redis/redis:6.0.14 Redis/redis:6.2.4 Redis/redis:6.2.3 Redis/redis:6.0.13 Redis/redis:6.2.2 Redis/redis:6.2.1 Redis/redis:6.0.12 Redis/redis:5.0.12 Redis/redis:6.0.11 Redis/redis:6.2.0 Redis/redis:5.0.11 Redis/redis:6.2-rc3 Redis/redis:6.0.10 Redis/redis:6.2-rc2 Redis/redis:6.2-rc1 Redis/redis:6.0.9 Redis/redis:5.0.10 Redis/redis:6.0.8 Redis/redis:6.0.7 Redis/redis:6.0.6 Redis/redis:6.0.5 Redis/redis:6.0.4 Redis/redis:6.0.3 Redis/redis:6.0.2 Redis/redis:6.0.1 Redis/redis:6.0.0 Redis/redis:5.0.9 Redis/redis:6.0-rc4 Redis/redis:6.0-rc3 Redis/redis:5.0.8 Redis/redis:6.0-rc2 Redis/redis:6.0-rc1 Redis/redis:5.0.7 Redis/redis:5.0.6 Redis/redis:5.0.5 Redis/redis:3.2.13 Redis/redis:4.0.14 Redis/redis:5.0.4 Redis/redis:4.0.13 Redis/redis:5.0.3 Redis/redis:4.0.12 Redis/redis:5.0.2 Redis/redis:5.0.1 Redis/redis:5.0.0 Redis/redis:5.0-rc6 Redis/redis:5.0-rc5 Redis/redis:4.0.11 Redis/redis:5.0-rc4 Redis/redis:5.0-rc3 Redis/redis:5.0-rc2 Redis/redis:4.0.10 Redis/redis:3.2.12 Redis/redis:5.0-rc1 Redis/redis:4.0.9 Redis/redis:4.0.8 Redis/redis:4.0.7 Redis/redis:4.0.6 Redis/redis:4.0.5 Redis/redis:4.0.4 Redis/redis:4.0.3 Redis/redis:3.2.11 Redis/redis:4.0.2 Redis/redis:3.2.10 Redis/redis:4.0.1 Redis/redis:4.0.0 Redis/redis:3.2.9 Redis/redis:4.0-rc3 Redis/redis:3.2.8 Redis/redis:3.2.7 Redis/redis:3.2.6 Redis/redis:4.0-rc2 Redis/redis:4.0-rc1 Redis/redis:3.2.5 Redis/redis:3.2.4 Redis/redis:3.2.3 Redis/redis:3.2.2 Redis/redis:3.2.1 Redis/redis:3.2.0 Redis/redis:3.2.0-rc3 Redis/redis:3.0.7 Redis/redis:3.2.0-rc2 Redis/redis:3.2-rc1 Redis/redis:3.0.6 Redis/redis:2.8.24 Redis/redis:3.0.5 Redis/redis:2.8.23 Redis/redis:2.8.22 Redis/redis:3.0.4 Redis/redis:3.0.3 Redis/redis:3.0.2 Redis/redis:2.8.21 Redis/redis:2.8.20 Redis/redis:3.0.1 Redis/redis:3.0.0 Redis/redis:3.0.0-rc6 Redis/redis:3.0.0-rc5 Redis/redis:3.0.0-rc4 Redis/redis:3.0.0-rc3 Redis/redis:3.0.0-rc2 Redis/redis:2.8.19 Redis/redis:2.8.18 Redis/redis:3.0.0-rc1 Redis/redis:2.8.17 Redis/redis:2.8.16 Redis/redis:2.8.15 Redis/redis:2.8.14 Redis/redis:3.0.0-beta8 Redis/redis:2.8.13 Redis/redis:3.0.0-beta7 Redis/redis:2.8.12 Redis/redis:2.8.11 Redis/redis:3.0.0-beta6 Redis/redis:2.8.10 Redis/redis:3.0.0-beta5 Redis/redis:3.0.0-beta4 Redis/redis:3.0.0-beta3 Redis/redis:2.8.9 Redis/redis:2.8.8 Redis/redis:3.0.0-beta2 Redis/redis:2.8.7 Redis/redis:2.8.6 Redis/redis:3.0.0-beta1 Redis/redis:2.8.5 Redis/redis:2.8.4 Redis/redis:2.6.17 Redis/redis:2.8.3 Redis/redis:2.8.2 Redis/redis:2.8.1 Redis/redis:2.8.0 Redis/redis:2.8.0-rc6 Redis/redis:2.8.0-rc5 Redis/redis:2.8.0-rc4 Redis/redis:2.6.16 Redis/redis:2.6.15 Redis/redis:2.8.0-rc3 Redis/redis:2.8.0-rc2 Redis/redis:2.8.0-rc1 Redis/redis:2.6.14-2 Redis/redis:2.6.14-1 Redis/redis:2.6.14 Redis/redis:2.6.13 Redis/redis:2.6.12 Redis/redis:2.6.10-3 Redis/redis:2.6.10-2 Redis/redis:2.6.11 Redis/redis:2.6.10-1 Redis/redis:2.6.10 Redis/redis:2.6.9-1 Redis/redis:2.6.9 Redis/redis:2.6.8-1 Redis/redis:2.6.8 Redis/redis:2.6.7-1 Redis/redis:2.6.7 Redis/redis:2.6.6 Redis/redis:2.4.18 Redis/redis:2.6.5 Redis/redis:2.6.4 Redis/redis:2.6.3 Redis/redis:2.6.2 Redis/redis:2.6.1 Redis/redis:2.6.0 Redis/redis:2.6.0-rc8 Redis/redis:2.6.0-rc7 Redis/redis:2.4.17 Redis/redis:2.6.0-rc6 Redis/redis:2.4.16 Redis/redis:2.4.15 Redis/redis:2.6.0-rc5 Redis/redis:2.6.0-rc4 Redis/redis:2.4.14 Redis/redis:2.6.0-rc3 Redis/redis:2.6.0-rc2 Redis/redis:2.4.13 Redis/redis:2.4.12 Redis/redis:2.6.0-rc1 Redis/redis:2.4.11 Redis/redis:2.4.10 Redis/redis:2.4.9 Redis/redis:2.4.8 Redis/redis:2.4.7 Redis/redis:2.4.6 Redis/redis:2.4.5 Redis/redis:2.4.4 Redis/redis:2.4.3 Redis/redis:2.4.2 Redis/redis:2.2.15 Redis/redis:2.4.1 Redis/redis:2.4.0 Redis/redis:2.2.14 Redis/redis:2.4.0-rc8 Redis/redis:2.2.13 Redis/redis:2.4.0-rc7 Redis/redis:2.4.0-rc6 Redis/redis:2.4.0-rc5 Redis/redis:2.4.0-rc4 Redis/redis:2.2.12 Redis/redis:2.4.0-rc3 Redis/redis:with-deprecated-diskstore Redis/redis:2.2.111-scripting Redis/redis:2.2.11 Redis/redis:2.2.110-scripting Redis/redis:2.2.10 Redis/redis:2.2.9 Redis/redis:2.4.0-rc2 Redis/redis:2.4.0-rc1 Redis/redis:2.2.8 Redis/redis:2.2.107-scripting Redis/redis:2.2.7 Redis/redis:2.2.106-scripting Redis/redis:2.2.6 Redis/redis:2.2.105-scripting Redis/redis:2.2.5 Redis/redis:2.2.4 Redis/redis:2.2.3 Redis/redis:3.0-alpha0 Redis/redis:2.0.5 Redis/redis:2.2.2 Redis/redis:2.2.1 Redis/redis:2.2.0 Redis/redis:2.2.0-rc4 Redis/redis:2.3-alpha0 Redis/redis:2.2.0-rc3 Redis/redis:2.2.0-rc2 Redis/redis:2.2.0-rc1 Redis/redis:2.2-alpha6 Redis/redis:2.2-alpha5 Redis/redis:v2.0.4-stable Redis/redis:2.0.4 Redis/redis:2.2-alpha4 Redis/redis:2.2-alpha3 Redis/redis:v2.0.3-stable Redis/redis:2.0.3 Redis/redis:v2.0.2-stable Redis/redis:2.0.2 Redis/redis:2.2-alpha2 Redis/redis:v2.0.1-stable Redis/redis:2.0.1 Redis/redis:v2.0.0-stable Redis/redis:2.0.0 Redis/redis:2.2-alpha1 Redis/redis:twitter-20100825 Redis/redis:twitter-20100804 Redis/redis:2.2-alpha0 Redis/redis:v2.0.0-rc4 Redis/redis:v2.0.0-rc3 Redis/redis:v2.0.0-rc2 Redis/redis:v2.0.0-rc1b Redis/redis:v2.1.1-watch Redis/redis:v2.0.0-rc1 Redis/redis:v1.3.12 Redis/redis:v1.3.11 Redis/redis:v1.3.10 Redis/redis:v1.3.9 Redis/redis:v1.3.8 Redis/redis:v1.3.7 Redis/redis:1.3.6 Redis/redis:vm-playpen
...
compare: Redis/redis:5.0.3
Branches Tags
Redis/redis:unstable Redis/redis:7.2 Redis/redis:6.2 Redis/redis:7.4 Redis/redis:8.0
Redis/redis:7.2.10 Redis/redis:6.2.19 Redis/redis:7.4.5 Redis/redis:8.0.3 Redis/redis:8.2-rc1 Redis/redis:8.2-rc1-int Redis/redis:8.2-m01 Redis/redis:8.2-m01-int2 Redis/redis:8.2-m01-int Redis/redis:8.0.2 Redis/redis:7.2.9 Redis/redis:7.4.4 Redis/redis:8.0.1 Redis/redis:8.0.1-int Redis/redis:8.0.0 Redis/redis:8.0-rc2-int Redis/redis:7.4.3 Redis/redis:7.2.8 Redis/redis:6.2.18 Redis/redis:8.0-rc1 Redis/redis:8.0-rc1-int2 Redis/redis:8.0-rc1-int Redis/redis:8.0-m04 Redis/redis:8.0-m04-int Redis/redis:8.0-m03 Redis/redis:7.2.7 Redis/redis:7.4.2 Redis/redis:6.2.17 Redis/redis:8.0-m02 Redis/redis:6.2.16 Redis/redis:7.2.6 Redis/redis:7.4.1 Redis/redis:8.0-m01 Redis/redis:7.4.0 Redis/redis:7.4-rc2 Redis/redis:7.4-rc1 Redis/redis:7.2.5 Redis/redis:7.2.4 Redis/redis:7.0.15 Redis/redis:7.2.3 Redis/redis:7.2.2 Redis/redis:7.0.14 Redis/redis:6.2.15 Redis/redis:6.2.14 Redis/redis:7.2.1 Redis/redis:7.0.13 Redis/redis:7.2.0 Redis/redis:7.2-rc3 Redis/redis:7.0.12 Redis/redis:6.2.13 Redis/redis:6.0.20 Redis/redis:7.2-rc2 Redis/redis:6.0.19 Redis/redis:6.2.12 Redis/redis:7.0.11 Redis/redis:7.2-rc1 Redis/redis:7.0.10 Redis/redis:7.0.9 Redis/redis:6.2.11 Redis/redis:6.0.18 Redis/redis:6.2.10 Redis/redis:6.0.17 Redis/redis:6.2.9 Redis/redis:7.0.8 Redis/redis:7.0.7 Redis/redis:7.0.6 Redis/redis:6.2.8 Redis/redis:7.0.5 Redis/redis:7.0.4 Redis/redis:7.0.3 Redis/redis:7.0.2 Redis/redis:7.0.1 Redis/redis:7.0.0 Redis/redis:6.2.7 Redis/redis:7.0-rc3 Redis/redis:7.0-rc2 Redis/redis:7.0-rc1 Redis/redis:6.2.6 Redis/redis:6.0.16 Redis/redis:5.0.14 Redis/redis:6.2.5 Redis/redis:5.0.13 Redis/redis:6.0.15 Redis/redis:6.0.14 Redis/redis:6.2.4 Redis/redis:6.2.3 Redis/redis:6.0.13 Redis/redis:6.2.2 Redis/redis:6.2.1 Redis/redis:6.0.12 Redis/redis:5.0.12 Redis/redis:6.0.11 Redis/redis:6.2.0 Redis/redis:5.0.11 Redis/redis:6.2-rc3 Redis/redis:6.0.10 Redis/redis:6.2-rc2 Redis/redis:6.2-rc1 Redis/redis:6.0.9 Redis/redis:5.0.10 Redis/redis:6.0.8 Redis/redis:6.0.7 Redis/redis:6.0.6 Redis/redis:6.0.5 Redis/redis:6.0.4 Redis/redis:6.0.3 Redis/redis:6.0.2 Redis/redis:6.0.1 Redis/redis:6.0.0 Redis/redis:5.0.9 Redis/redis:6.0-rc4 Redis/redis:6.0-rc3 Redis/redis:5.0.8 Redis/redis:6.0-rc2 Redis/redis:6.0-rc1 Redis/redis:5.0.7 Redis/redis:5.0.6 Redis/redis:5.0.5 Redis/redis:3.2.13 Redis/redis:4.0.14 Redis/redis:5.0.4 Redis/redis:4.0.13 Redis/redis:5.0.3 Redis/redis:4.0.12 Redis/redis:5.0.2 Redis/redis:5.0.1 Redis/redis:5.0.0 Redis/redis:5.0-rc6 Redis/redis:5.0-rc5 Redis/redis:4.0.11 Redis/redis:5.0-rc4 Redis/redis:5.0-rc3 Redis/redis:5.0-rc2 Redis/redis:4.0.10 Redis/redis:3.2.12 Redis/redis:5.0-rc1 Redis/redis:4.0.9 Redis/redis:4.0.8 Redis/redis:4.0.7 Redis/redis:4.0.6 Redis/redis:4.0.5 Redis/redis:4.0.4 Redis/redis:4.0.3 Redis/redis:3.2.11 Redis/redis:4.0.2 Redis/redis:3.2.10 Redis/redis:4.0.1 Redis/redis:4.0.0 Redis/redis:3.2.9 Redis/redis:4.0-rc3 Redis/redis:3.2.8 Redis/redis:3.2.7 Redis/redis:3.2.6 Redis/redis:4.0-rc2 Redis/redis:4.0-rc1 Redis/redis:3.2.5 Redis/redis:3.2.4 Redis/redis:3.2.3 Redis/redis:3.2.2 Redis/redis:3.2.1 Redis/redis:3.2.0 Redis/redis:3.2.0-rc3 Redis/redis:3.0.7 Redis/redis:3.2.0-rc2 Redis/redis:3.2-rc1 Redis/redis:3.0.6 Redis/redis:2.8.24 Redis/redis:3.0.5 Redis/redis:2.8.23 Redis/redis:2.8.22 Redis/redis:3.0.4 Redis/redis:3.0.3 Redis/redis:3.0.2 Redis/redis:2.8.21 Redis/redis:2.8.20 Redis/redis:3.0.1 Redis/redis:3.0.0 Redis/redis:3.0.0-rc6 Redis/redis:3.0.0-rc5 Redis/redis:3.0.0-rc4 Redis/redis:3.0.0-rc3 Redis/redis:3.0.0-rc2 Redis/redis:2.8.19 Redis/redis:2.8.18 Redis/redis:3.0.0-rc1 Redis/redis:2.8.17 Redis/redis:2.8.16 Redis/redis:2.8.15 Redis/redis:2.8.14 Redis/redis:3.0.0-beta8 Redis/redis:2.8.13 Redis/redis:3.0.0-beta7 Redis/redis:2.8.12 Redis/redis:2.8.11 Redis/redis:3.0.0-beta6 Redis/redis:2.8.10 Redis/redis:3.0.0-beta5 Redis/redis:3.0.0-beta4 Redis/redis:3.0.0-beta3 Redis/redis:2.8.9 Redis/redis:2.8.8 Redis/redis:3.0.0-beta2 Redis/redis:2.8.7 Redis/redis:2.8.6 Redis/redis:3.0.0-beta1 Redis/redis:2.8.5 Redis/redis:2.8.4 Redis/redis:2.6.17 Redis/redis:2.8.3 Redis/redis:2.8.2 Redis/redis:2.8.1 Redis/redis:2.8.0 Redis/redis:2.8.0-rc6 Redis/redis:2.8.0-rc5 Redis/redis:2.8.0-rc4 Redis/redis:2.6.16 Redis/redis:2.6.15 Redis/redis:2.8.0-rc3 Redis/redis:2.8.0-rc2 Redis/redis:2.8.0-rc1 Redis/redis:2.6.14-2 Redis/redis:2.6.14-1 Redis/redis:2.6.14 Redis/redis:2.6.13 Redis/redis:2.6.12 Redis/redis:2.6.10-3 Redis/redis:2.6.10-2 Redis/redis:2.6.11 Redis/redis:2.6.10-1 Redis/redis:2.6.10 Redis/redis:2.6.9-1 Redis/redis:2.6.9 Redis/redis:2.6.8-1 Redis/redis:2.6.8 Redis/redis:2.6.7-1 Redis/redis:2.6.7 Redis/redis:2.6.6 Redis/redis:2.4.18 Redis/redis:2.6.5 Redis/redis:2.6.4 Redis/redis:2.6.3 Redis/redis:2.6.2 Redis/redis:2.6.1 Redis/redis:2.6.0 Redis/redis:2.6.0-rc8 Redis/redis:2.6.0-rc7 Redis/redis:2.4.17 Redis/redis:2.6.0-rc6 Redis/redis:2.4.16 Redis/redis:2.4.15 Redis/redis:2.6.0-rc5 Redis/redis:2.6.0-rc4 Redis/redis:2.4.14 Redis/redis:2.6.0-rc3 Redis/redis:2.6.0-rc2 Redis/redis:2.4.13 Redis/redis:2.4.12 Redis/redis:2.6.0-rc1 Redis/redis:2.4.11 Redis/redis:2.4.10 Redis/redis:2.4.9 Redis/redis:2.4.8 Redis/redis:2.4.7 Redis/redis:2.4.6 Redis/redis:2.4.5 Redis/redis:2.4.4 Redis/redis:2.4.3 Redis/redis:2.4.2 Redis/redis:2.2.15 Redis/redis:2.4.1 Redis/redis:2.4.0 Redis/redis:2.2.14 Redis/redis:2.4.0-rc8 Redis/redis:2.2.13 Redis/redis:2.4.0-rc7 Redis/redis:2.4.0-rc6 Redis/redis:2.4.0-rc5 Redis/redis:2.4.0-rc4 Redis/redis:2.2.12 Redis/redis:2.4.0-rc3 Redis/redis:with-deprecated-diskstore Redis/redis:2.2.111-scripting Redis/redis:2.2.11 Redis/redis:2.2.110-scripting Redis/redis:2.2.10 Redis/redis:2.2.9 Redis/redis:2.4.0-rc2 Redis/redis:2.4.0-rc1 Redis/redis:2.2.8 Redis/redis:2.2.107-scripting Redis/redis:2.2.7 Redis/redis:2.2.106-scripting Redis/redis:2.2.6 Redis/redis:2.2.105-scripting Redis/redis:2.2.5 Redis/redis:2.2.4 Redis/redis:2.2.3 Redis/redis:3.0-alpha0 Redis/redis:2.0.5 Redis/redis:2.2.2 Redis/redis:2.2.1 Redis/redis:2.2.0 Redis/redis:2.2.0-rc4 Redis/redis:2.3-alpha0 Redis/redis:2.2.0-rc3 Redis/redis:2.2.0-rc2 Redis/redis:2.2.0-rc1 Redis/redis:2.2-alpha6 Redis/redis:2.2-alpha5 Redis/redis:v2.0.4-stable Redis/redis:2.0.4 Redis/redis:2.2-alpha4 Redis/redis:2.2-alpha3 Redis/redis:v2.0.3-stable Redis/redis:2.0.3 Redis/redis:v2.0.2-stable Redis/redis:2.0.2 Redis/redis:2.2-alpha2 Redis/redis:v2.0.1-stable Redis/redis:2.0.1 Redis/redis:v2.0.0-stable Redis/redis:2.0.0 Redis/redis:2.2-alpha1 Redis/redis:twitter-20100825 Redis/redis:twitter-20100804 Redis/redis:2.2-alpha0 Redis/redis:v2.0.0-rc4 Redis/redis:v2.0.0-rc3 Redis/redis:v2.0.0-rc2 Redis/redis:v2.0.0-rc1b Redis/redis:v2.1.1-watch Redis/redis:v2.0.0-rc1 Redis/redis:v1.3.12 Redis/redis:v1.3.11 Redis/redis:v1.3.10 Redis/redis:v1.3.9 Redis/redis:v1.3.8 Redis/redis:v1.3.7 Redis/redis:1.3.6 Redis/redis:vm-playpen
1531 Commits
4.0.3 ... 5.0.3
Author SHA1 Message Date
antirez b4867130f4 Redis 5.0.3 2018-12-12 13:25:58 +01:00
antirez 2c6ee0f9b3 freeMemoryIfNeeded() small refactoring. 
Related to issue #5686 and PR #5689.
 2018-12-12 11:53:15 +01:00
zhaozhao.zz 107e93e75a evict: don't care about mem if loading 
When loading data, we call processEventsWhileBlocked
to process events and execute commands.
But if we are loading AOF it's dangerous, because
processCommand would call freeMemoryIfNeeded to evict,
and that will break data consistency, see issue #5686.
 2018-12-12 11:53:15 +01:00
antirez 9edb893c7d Crashing is too much in addReplyErrorLength(). 
See #5663.
 2018-12-12 11:53:15 +01:00
hdmg c55254a5f4 fix comments fault discription 2018-12-11 18:01:36 +01:00
lsytj0413 dfd250132d fix a typo: craeted -> created 2018-12-11 18:01:30 +01:00
antirez 392a2566b6 stringmatchlen() fuzz test added. 
Verified to be able to trigger at least #5632. Does not report other
issues.
 2018-12-11 17:59:16 +01:00
antirez 7602f69530 Fix stringmatchlen() read past buffer bug. 
See #5632.
 2018-12-11 17:59:16 +01:00
zhaozhao.zz c4f3585e1f multi: ignore multiState's cmd_flags when loading AOF 2018-12-11 17:59:16 +01:00
antirez d037e98711 Reject EXEC containing write commands against RO replica. 
Thanks to @soloestoy for discovering this issue in #5667.
This is an alternative fix in order to avoid both cycling the clients
and also disconnecting clients just having valid read-only transactions
pending.
 2018-12-11 17:59:16 +01:00
artix e00ab32454 Cluster Manager: 
- Multiple owners checking in 'fix'/'check' commands is now
  optional (using --cluster-search-multiple-owners).
- Updated help.
 2018-12-11 17:59:16 +01:00
artix 94f64de35c Cluster Manager: 
- FixOpenSlot now correctly updates in-memory cluster configuration.
    - Improved output messages.
 2018-12-11 17:59:16 +01:00
artix 752d636fa6 Cluster Manager: 'fix' command now handles open slots with migrating state 
in one node and importing state in multiple nodes.
 2018-12-11 17:59:16 +01:00
artix 552091f990 Cluster Manager: setting new slot owner is now handled atomically 
in 'fix' command.
 2018-12-11 17:59:16 +01:00
artix 2280f4f7c4 Cluster Manager: code cleanup. 2018-12-11 17:59:16 +01:00
artix e084b8ccbf Cluster Manager: check/fix commands now handle multiple owners even if 
all slots are covered and not open.
 2018-12-11 17:59:16 +01:00
zhaozhao.zz fa726e2af5 remove useless tryObjectEncoding in debug assert 2018-12-11 17:59:16 +01:00
Oran Agra 40244b10f0 fix #5580, display fragmentation and rss overhead bytes as signed 
these metrics become negative when RSS is smaller than the used_memory.
This can easily happen when the program allocated a lot of memory and haven't
written to it yet, in which case the kernel doesn't allocate any pages to the process
 2018-12-11 17:58:50 +01:00
zhaozhao.zz beab31512e networking: current_client should not be NULL when trim qb_pos 2018-12-11 17:58:50 +01:00
antirez 07ccb642b7 Remove no longer relevant comment in processCommand(). 2018-12-11 17:58:50 +01:00
antirez 60fdaf072d DEBUG DIGEST-VALUE implemented. 2018-12-11 17:58:50 +01:00
antirez 48b31b0dce DEBUG DIGEST refactoring: extract function to digest a value. 2018-12-11 17:58:50 +01:00
yura ef3ff40206 redis-cli reshard/rebalance: ability to force replacement on existing keys 2018-12-11 17:58:50 +01:00
Thomas Orozco ee223fb8c3 cli: pass auth through REDISCLI_AUTH 
This adds support for passing a password through a REDISCLI_AUTH
environment variable (which is safer than the CLI), which might often be
safer than passing it through a CLI argument.

Passing a password this way does not trigger the warning about passing a
password through CLI arguments, and CLI arguments take precedence over
it.
 2018-12-11 17:58:50 +01:00
yongman 41295e5595 Fix cluster call reply format readable 2018-12-11 17:58:50 +01:00
Oran Agra 0ed3970f2c fix small test suite race conditions 2018-12-11 17:58:50 +01:00
zhaozhao.zz 605dddbbc0 MEMORY command: make USAGE more accurate 
In MEMORY USAGE command, we count the key argv[2] into usage,
but the argument in command may contains free spaces because of
sdsMakeRoomFor. But the key in db never contains free spaces
because we use sdsdup when dbAdd, so using the real key to
count the usage is more accurate.
 2018-12-11 17:58:19 +01:00
yongman 1f43bf29a3 Fix choose a random master node for slot assignment 2018-12-11 17:58:19 +01:00
Weiliang Li 69f0c6788f fix comment typo in util.c 
fix comment typo in util.c
 2018-12-11 17:58:19 +01:00
Chris Lamb bc53a3abb9 Clarify the "Creating Server TCP listening socket" error. 
This really helps spot it in the logs, otherwise it does not look like a
warning/error. For example:

  Creating Server TCP listening socket ::1:6379: bind: Cannot assign requested address

... is not nearly as clear as:

  Could not create server TCP listening listening socket ::1:6379: bind: Cannot assign requested address
 2018-12-11 17:58:19 +01:00
Chris Lamb fefe546068 Don't treat unsupported protocols as fatal errors 
If we encounter an unsupported protocol in the "bind" list, don't
ipso-facto consider it a fatal error. We continue to abort startup if
there are no listening sockets at all.

This ensures that the lack of IPv6 support does not prevent Redis from
starting on Debian where we try to bind to the ::1 interface by default
(via "bind 127.0.0.1 ::1"). A machine with IPv6 disabled (such as some
container systems) would simply fail to start Redis after the initiall
call to apt(8).

This is similar to the case for where "bind" is not specified:

  https://github.com/antirez/redis/issues/3894

... and was based on the corresponding PR:

  https://github.com/antirez/redis/pull/4108

... but also adds EADDRNOTAVAIL to the list of errors to catch which I
believe is missing from there.

This issue was raised in Debian as both <https://bugs.debian.org/900284>
& <https://bugs.debian.org/914354>.
 2018-12-11 17:58:19 +01:00
David Carlier a8862972b6 OpenBSD support. 
Special treatment here as backtrace support is optional,
cannot be found via pkg-config and similar neither.
 2018-12-11 17:58:19 +01:00
David Carlier 5e86daf947 Backtrace/register dump on BSD. 
FreeBSD/DragonFlyBSD does have backtrace only it does not
belong to libc.
 2018-12-11 17:58:19 +01:00
Guy Benoish 7c8cf5acdd Don't call sdscmp() with shared.maxstring or shared.minstring 2018-12-11 17:58:19 +01:00
Qu Chen 39e9eda377 Add unit test for stream XCLAIM command. 2018-12-11 17:57:54 +01:00
antirez 62485232e8 Abort instead of crashing when loading bad stream master key. 
See #5612.
 2018-12-11 17:57:50 +01:00
Madelyn Olson a54873092e Fixed a serverPanic when sending an invalid command to a monitor client 2018-12-04 18:10:22 +01:00
antirez 1637522f00 Redis 5.0.2. 2018-11-22 11:26:38 +01:00
David Carlier e8b4291a0f DragonFlyBSD little build fix 2018-11-22 11:21:47 +01:00
yongman 8fcfd374d7 skip slave nodes when sending cluster setslot command 2018-11-22 11:21:47 +01:00
yongman d7089ddddc Fix pointer access and memory leak in redis-cli. 2018-11-22 11:21:47 +01:00
antirez 17b4cd83f4 Test: regression test for #5570. 2018-11-20 08:32:50 +01:00
antirez 45123169bc Stream: fix XREADGROUP history reading of deleted messages. 
This commit fixes #5570. It is a similar bug to one fixed a few weeks
ago and is due to the range API to be called with NULL as "end ID"
parameter instead of repeating again the start ID, to be sure that we
selectively issue the entry with a given ID, or we get zero returned
(and we know we should emit a NULL reply).
 2018-11-20 08:32:47 +01:00
David Carlier 5ad588f0f4 only FreeBSD change/little warning addressing 2018-11-20 08:32:41 +01:00
David Carlier 11801e1a78 tweak form feedback 2018-11-20 08:32:31 +01:00
David Carlier c1f13575d7 allow flavors 2018-11-20 08:32:31 +01:00
David Carlier 275a2d49cc Fix clang build. 
Some math functions require c11 standard.
 2018-11-20 08:32:31 +01:00
antirez 44ad514185 Test: regression test for #5577. 2018-11-20 08:32:31 +01:00
antirez c7951f4304 Streams: fix XREADGROUP history reading when CG last_id is low. 
This fixes the issue reported in #5570.
This was fixed the hard way, that is, propagating more information to
the lower level API about this being a request to read just the history,
so that the code is simpler and less likely to regress.
 2018-11-20 08:32:31 +01:00
antirez a69bc5befe t_stream.c comment resized to 80 cols. 2018-11-20 08:32:31 +01:00
antirez 5314099d04 Redis 5 changelog: don't expect Lua replies to be ordered. 
Related to #5538.
 2018-11-07 17:00:13 +01:00
antirez c595050012 Redis 5.0.1. 2018-11-07 13:23:06 +01:00
antirez c801283fd2 Fix cluster-replica-no-failover option name. 
Thanks to @NicolasLM, see issue #5537.
 2018-11-07 13:08:05 +01:00
antirez 4c4f50e1c1 MEMORY command: make strcasecmp() conditional like the following. 2018-11-07 13:08:02 +01:00
Itamar Haber a7b46e0e64 Uppercases subcommands in MEMORY HELP 2018-11-07 13:05:49 +01:00
Itamar Haber 80e129d93e Standardizes MEMORY HELP subcommand 2018-11-07 13:05:49 +01:00
valentino 88805cbb3e fix short period of server.hz being uninitialized 
server.hz was uninitialized between initServerConfig and initServer.
this can lead to someone (e.g. queued modules) doing createObject,
and accessing an uninitialized variable, that can potentially be 0,
and lead to a crash.
 2018-11-07 13:05:49 +01:00
Itamar Haber 6b402733e1 Adds HELP to LATENCY 
Signed-off-by: Itamar Haber <itamar@redislabs.com>
 2018-11-07 13:05:49 +01:00
yongman 1c637de98c fix malloc in clusterManagerComputeReshardTable 2018-11-07 13:05:49 +01:00
artix 90b52fde57 Cluster Manager: removed unused var. 2018-11-07 13:05:49 +01:00
artix 89cbb5df06 Cluster Manager: further improvements to "fix": 
- clusterManagerFixOpenSlot: ensure that the
  slot is unassigned before ADDSLOTS
- clusterManagerFixSlotsCoverage: after cold
  migration, the slot configuration
  is now updated on all the nodes.
 2018-11-07 13:05:49 +01:00
artix 175515c944 Cluster Manager: fixed string parsing issue in clusterManagerGetConfigSignature 2018-11-07 13:05:29 +01:00
artix 3997dd6eaa Cluster Manager: better fix subcommand. 2018-11-07 13:05:29 +01:00
artix bd80291c36 Cluster Manager: fixed typos in comments. 2018-11-07 13:05:29 +01:00
artix 4369cbce08 Cluster Manager: fixed 'DELSLOT' subcommand typo. 2018-11-07 13:05:29 +01:00
antirez 1ed821e28d Fix XCLAIM missing entry bug. 
This bug had a double effect:

1. Sometimes entries may not be emitted, producing broken protocol where
the array length was greater than the emitted entires, blocking the
client waiting for more data.

2. Some other time the right entry was claimed, but a wrong entry was
returned to the client.

This fix should correct both the instances.
 2018-11-05 17:12:37 +01:00
michael-grunder b49bcd01d0 Use typedef'd mstime_t instead of time_t 
This fixes an overflow on 32-bit systems.
 2018-11-05 17:12:37 +01:00
antirez 09d1849ed5 Improve streamReplyWithRange() top comment. 2018-11-05 17:12:37 +01:00
antirez bdf6306f29 Add support for Sentinel authentication. 
So far it was not possible to setup Sentinel with authentication
enabled. This commit introduces this feature: every Sentinel will try to
authenticate with other sentinels using the same password it is
configured to accept clients with.

So for instance if a Sentinel has a "requirepass" configuration
statemnet set to "foo", it will use the "foo" password to authenticate
with every other Sentinel it connects to. So basically to add the
"requirepass" to all the Sentinels configurations is enough in order to
make sure that:

1) Clients will require the password to access the Sentinels instances.
2) Each Sentinel will use the same password to connect and authenticate
   with every other Sentinel in the group.

Related to #3279 and #3329.
 2018-11-05 17:12:37 +01:00
antirez 50222af5f4 Disable protected mode in Sentinel mode. 
Sentinel must be exposed, so protected mode is just an issue for users
in case Redis was started in Sentinel mode.

Related to #3279 and #3329.
 2018-11-05 17:12:37 +01:00
antirez 643ee6e38c When replica kills a pending RDB save during SYNC, log it. 
This logs what happens in the context of the fix in PR #5367.
 2018-11-05 17:12:37 +01:00
Andrey Bugaevskiy 8b609c9986 Move child termination to readSyncBulkPayload 2018-11-05 17:12:37 +01:00
Andrey Bugaevskiy 2710260594 Prevent RDB autosave from overwriting full resync results 
During the full database resync we may still have unsaved changes
on the receiving side. This causes a race condition between
synced data rename/load and the rename of rdbSave tempfile.
 2018-11-05 17:12:37 +01:00
antirez a677923d7c asyncCloseClientOnOutputBufferLimitReached(): don't free fake clients. 
Fake clients are used in special situations and are not linked to the
normal clients list, freeing them will always result in Redis crashing
in one way or the other.

It's not common to send replies to fake clients, but we have one usage
in the modules API. When a client is blocked, we associate to the
blocked client object (that is safe to manipulate in a thread), a fake
client that accumulates replies. So because of this bug there was
the problem described in issue #5443.

The fix was verified to work with the provided example module. To write
a regression is very hard and unlikely to be triggered in the future.
 2018-11-05 17:12:37 +01:00
David Carlier 427e440a58 needs it for the global 2018-11-05 17:12:37 +01:00
David Carlier 28f9ca4e1d Fix non Linux build. 
timezone global is a linux-ism whereas it is a function under BSD.
Here a helper to get the timezone value in a more portable manner.
 2018-11-05 17:12:37 +01:00
zhaozhao.zz 4bf9efe20f MULTI: OOM err if cannot free enough memory in MULTI/EXEC context 2018-11-05 17:12:37 +01:00
antirez 4fbd7a39f8 Add command fingerprint comment for XSETID. 2018-11-05 17:12:37 +01:00
Itamar Haber 2480db53f1 Plugs a potential underflow 2018-11-05 17:12:37 +01:00
Itamar Haber e5e4d2ef08 Corrects inline documentation of syntax 2018-11-05 17:12:37 +01:00
zhaozhao.zz 713800d20a if we read a expired key, misses++ 2018-11-05 17:12:37 +01:00
antirez e79ee263cf Fix XRANGE COUNT option for value of 0. 2018-11-05 17:12:37 +01:00
antirez 505cc70ff8 Fix typo in streamReplyWithRange() top comment. 2018-11-05 17:12:37 +01:00
Damien Tournoud 3c36561d27 Overhead is the allocated size of the AOF buffer, not its length 2018-11-05 17:12:37 +01:00
antirez 3761582ffc Simplify part of the #5470 patch. 2018-11-05 17:12:37 +01:00
zhaozhao.zz edc47a3ad2 do not delete expired keys in KEYS command 2018-11-05 17:12:37 +01:00
antirez 9872af6d52 Use guide comments to make changes in #5462 more obvious. 2018-10-22 17:45:39 +02:00
youjiali1995 3f399c3bff migrate: fix mismatch of RESTORE reply when some keys have expired. 2018-10-22 17:45:39 +02:00
hujie eaaff6211b fix typo in config.c 2018-10-22 17:45:39 +02:00
hujiecs 43ebb7ee01 several typos fixed, optimize MSETNX to avoid unnecessary loop 2018-10-22 17:45:31 +02:00
antirez de8fdaacfc Remove useless complexity from MSET implementation. 2018-10-22 17:44:26 +02:00
antirez dc8f111251 Fix again stack generation on the Raspberry Pi. 
The fix was removed by c8ca71d40 attempting to fix the stack generation
on ARM64, without testing if it would still work on ARM32.
Now it should work both sides.
 2018-10-19 10:40:42 +02:00
antirez 83a6e81db3 Get rid of the word slave in the release note of Redis 5. 2018-10-17 18:11:49 +02:00
antirez 882ca6962f Redis 5.0.0. 2018-10-17 17:31:39 +02:00
antirez 1eb0994ecc Streams: use bulk replies instead of status replies. 
They play better with Lua scripting, otherwise Lua will see status
replies as "ok" = "string" which is very odd, and actually as @oranagra
reasoned in issue #5456 in the rest of the Redis code base there was no
such concern as saving a few bytes when the protocol is emitted.
 2018-10-17 17:20:05 +02:00
antirez bcc0916d08 Fix conditional in XGROUP. 2018-10-17 13:00:55 +02:00
antirez 1b2f23f3c4 Update help.h for redis-cli. 2018-10-17 12:57:15 +02:00
antirez de0ae56c84 Tests for XGROUP CREATE MKSTREAM. 2018-10-17 12:12:04 +02:00
antirez 56c3dfa195 Fix XGROUP CREATE MKSTREAM handling of . 2018-10-17 12:11:48 +02:00
antirez 2687f2283c Process MKSTREAM option of XGROUP CREATE at a later time. 
This avoids issues with having to replicate a command that produced
errors.
 2018-10-17 12:11:48 +02:00
zhaozhao.zz cfbaf8f1f3 Scripting & Streams: some commands need right flags 
xadd with id * generates random stream id

xadd & xtrim with approximate maxlen count may
trim stream randomly

xinfo may get random radix-tree-keys/nodes

xpending may get random idletime

xclaim: master and slave may have different
idletime in stream
 2018-10-17 12:11:48 +02:00
antirez 4e4099b95d XGROUP CREATE: MKSTREAM option for automatic stream creation. 2018-10-17 11:27:09 +02:00
zhaozhao.zz 6dd4d864a9 Streams: Tests modified XSTREAM -> XSETID 2018-10-17 11:02:34 +02:00
zhaozhao.zz 3aff0e8cb5 Streams: rewrite empty streams with certain lastid 2018-10-17 11:02:34 +02:00
antirez 880b563e2d Tests modified to use XADD MAXLEN 0 + XSETID. 
See #5426.
 2018-10-17 11:02:34 +02:00
antirez 83c8783598 Streams: rewrite empty streams with XADD MAXLEN 0. Use XSETID. 
Related to #5426.
 2018-10-17 11:02:34 +02:00
antirez fd22e3acae XSETID: accept IDs based on last entry. 
Related to #5426.
 2018-10-17 11:02:34 +02:00
antirez dfab3cba2a Streams: XSTREAM SETID -> XSETID. 
Keep vanilla stream commands at toplevel, see #5426.
 2018-10-17 11:02:34 +02:00
zhaozhao.zz a3fb28edce Streams: rewrite id in XSTREAM CREATE * 2018-10-17 11:02:34 +02:00
zhaozhao.zz f4b4db1387 Streams: add tests for aof rewrite 2018-10-17 11:02:34 +02:00
zhaozhao.zz d22f1ef032 Stream & AOF: rewrite stream in correct way 2018-10-17 11:02:34 +02:00
zhaozhao.zz 6455274d1c Streams: add tests for XSTREAM command 2018-10-17 11:02:34 +02:00
zhaozhao.zz 0edbe953ea Streams: add a new command XTREAM 
XSTREAM CREATE <key> <id or *> -- Create a new empty stream.
XSTREAM SETID <key> <id or $>  -- Set the current stream ID.
 2018-10-17 11:02:34 +02:00
Hamid Alaei 9714bba266 fix timer context selected database 2018-10-15 13:05:42 +02:00
antirez eb53f15a3e Make comment about nack->consumer test for minidle more obvious. 
Related to #5437.
 2018-10-15 13:05:35 +02:00
antirez a77f836e0d Streams: use propagate_last_id itself as streamPropagateGroupID trigger. 
Avoid storing the dirty value. See #5437.
 2018-10-15 13:05:35 +02:00
antirez 0f0610eb01 Streams: better naming: lastid_updated -> propagate_last_id. 
See #5437 but also I updated a previous usage of the same var name.
 2018-10-15 13:05:35 +02:00
zhaozhao.zz a745e4234d Streams: panic if streamID invalid after check, should not be possible. 2018-10-15 13:05:35 +02:00
zhaozhao.zz 9974be13a4 Streams: propagate lastid in XCLAIM when it has effect 2018-10-15 13:05:35 +02:00
zhaozhao.zz 69a628d0f0 Streams: XCLAIM ignore minidle if NACK is created by FORCE 
Because the NACK->consumer is NULL, if idletime < minidle
the NACK does not belong to any consumer, then redis will crash
in XPENDING.
 2018-10-15 13:05:35 +02:00
zhaozhao.zz a04b43c7ea Streams: bugfix XCLAIM should propagate group name not consumer name 2018-10-15 13:05:35 +02:00
Sergey Chupov 8977a90c50 fixed typos in readme 2018-10-15 13:05:35 +02:00
antirez 3a745674cf redis.conf typo fixed: ingore -> ignore. 2018-10-15 13:05:35 +02:00
antirez 22770d762c Rax: radix tree updated to latest version from antirez/rax. 2018-10-15 13:05:17 +02:00
antirez fbac534fd3 Test: avoid time related false positive in RESTORE test. 2018-10-15 13:05:17 +02:00
antirez 4987233795 LOLWUT: capitalize Nees. 2018-10-15 13:05:17 +02:00
antirez 80c471f58e Test: cgroup propagation test also for NOACK variant. 
Related to #5433.
 2018-10-15 13:05:17 +02:00
antirez 8defa5da3c Test: consumer group last ID slave propagation test. 
This is a regression for #5433.
 2018-10-15 13:05:17 +02:00
zhaozhao.zz e1e3eacae3 Avoid recreate write handler for protected client. 2018-10-15 13:05:17 +02:00
antirez b501fd5d3e Fix propagation of consumer groups last ID. 
Issue #5433.
 2018-10-15 13:05:17 +02:00
antirez bedc3dee24 Redis 5.0 RC6. 2018-10-10 11:11:06 +02:00
antirez 9a6fa7d082 changelog.tcl: get optional argument for number of commits. 2018-10-10 11:03:25 +02:00
antirez 101e419ffc Free protected clients asynchronously. 
Related to #4840.

Note that when we re-enter the event loop with aeProcessEvents() we
don't process timers, nor before/after sleep callbacks, so we should
never end calling freeClientsInAsyncFreeQueue() when re-entering the
loop.
 2018-10-09 18:30:59 +02:00
antirez 726debb835 Actually use the protectClient() API where needed. 
Related to #4804.
 2018-10-09 18:30:49 +02:00
antirez 0b87f78a5f Introduce protectClient() + some refactoring. 
The idea is to have an API for the cases like -BUSY state and DEBUG
RELOAD where we have to manually deinstall the read handler.
See #4804.
 2018-10-09 18:30:49 +02:00
zhaozhao.zz 6aa8ac70a4 debug: avoid free client unexpectedly when reload & loadaof 2018-10-09 18:30:49 +02:00
antirez 48040b0266 aof.c: improve indentation and change warning message. 
Related to #5201.

I removed the !!! Warning part since compared to the other errors, a
missing EXEC is in theory a normal happening in the AOF file, at least
in theory: may happen in a differnet number of situations, and it's
probably better to don't give the user the feeling that something really
bad happened.
 2018-10-09 18:30:49 +02:00
zhaozhao.zz 7cc2056965 AOF: discard if we lost EXEC when loading aof 2018-10-09 18:30:49 +02:00
antirez 2007d30c9d Refactoring of XADD / XTRIM MAXLEN rewriting. 
See #5141.
 2018-10-09 18:30:49 +02:00
zhaozhao.zz 6a2981101b Streams: add test cases for XADD/XTRIM maxlen 2018-10-09 18:30:34 +02:00
zhaozhao.zz 041161b7ce Streams: propagate specified MAXLEN instead of approximated 
Slaves and rebooting redis may have different radix tree struct,
by different stream* config options. So propagating approximated
MAXLEN to AOF/slaves may lead to date inconsistency.
 2018-10-09 18:30:34 +02:00
zhaozhao.zz f04d799bb9 Streams: reset approx_maxlen in every maxlen loop 2018-10-09 18:30:34 +02:00
zhaozhao.zz affd93654f Streams: XTRIM will return an error if MAXLEN with a count < 0 2018-10-09 18:30:34 +02:00
zhaozhao.zz 4c405ad07f Streams: propagate original MAXLEN argument in XADD context 
If we rewrite the MAXLEN argument as zero when no trimming
was performed, date between master and slave and aof will
be inconsistent, because `xtrim maxlen 0` means delete all
entries in stream.
 2018-10-09 18:30:34 +02:00
antirez 5c6d4b4ad4 Fix typo in replicationCron() comment. 2018-10-09 18:30:22 +02:00
antirez a67a8dbf2f Fix typo in design comment of bio.c. 2018-10-09 18:30:18 +02:00
antirez c4ab5a05a7 xclaimCommand(): fix comment typos. 2018-10-09 18:30:13 +02:00
antirez dc0b628a8e streamAppendItem(): Update the radix tree pointer only if changed. 2018-10-03 11:19:32 +02:00
antirez 4566fbc79d Listpack: optionally force reallocation on inserts. 
This is useful in order to spot bugs where we fail
at updating the pointer returned by the insertion
function. Normally often the same pointer is returned,
making it harder than needed to spot bugs.

Related to #5210.
 2018-10-03 11:19:32 +02:00
antirez 5eca170c5b Fix printf type mismatch in genRedisInfoString(). 2018-10-03 11:19:32 +02:00
antirez 260b53a284 streamIteratorRemoveEntry(): set back lp only if pointer changed. 
Most of the times the pointer will remain the same since integers of
similar size don't take more space in listpacks.

Related to #5210.
 2018-10-03 11:19:32 +02:00
zhaozhao.zz 5d12f9d98b Streams: update listpack with new pointer in XDEL 2018-10-03 11:19:32 +02:00
zhaozhao.zz 6b7ad8381a bugfix: replace lastcmd with cmd when rewrite BRPOPLPUSH as RPOPLPUSH 
There are two problems if we use lastcmd:

1. BRPOPLPUSH cannot be rewrited as RPOPLPUSH in multi/exec
    In mulit/exec context, the lastcmd is exec.
2. Redis will crash when execute RPOPLPUSH loading from AOF
    In fakeClient, the lastcmd is NULL.
 2018-10-03 11:19:32 +02:00
Oran Agra 3454a04399 script cache memory in INFO and MEMORY includes both script code and overheads 2018-10-03 11:19:32 +02:00
Oran Agra d6aeca862c fix #5024 - commandstats for multi-exec were logged as EXEC. 
this was broken a while back by ba9154d7e7
the purpose of which was to fix commandstats for GEOADD
 2018-10-03 11:19:32 +02:00
antirez a996b2a285 Fix XINFO comment for consistency. 2018-10-03 11:19:32 +02:00
Bruce Merry 1a8447b6c1 Fix invalid use of sdsZmallocSize on an embedded string 
sdsZmallocSize assumes a dynamically allocated SDS. When given a string
object created by createEmbeddedStringObject, it calls zmalloc_size on a
pointer that isn't the one returned by zmalloc
 2018-10-03 11:19:12 +02:00
Bruce Merry 8dde46ad16 Fix incorrect memory usage accounting in zrealloc 
When HAVE_MALLOC_SIZE is false, each call to zrealloc causes used_memory
to increase by PREFIX_SIZE more than it should, due to mis-matched
accounting between the original zmalloc (which includes PREFIX size in
its increment) and zrealloc (which misses it from its decrement).

I've also supplied a command-line test to easily demonstrate the
problem. It's not wired into the test framework, because I don't know
TCL so I'm not sure how to automate it.
 2018-10-03 11:19:12 +02:00
Hamid Alaei b362a1b758 fix dict get on not found 2018-10-03 11:19:12 +02:00
antirez 55e9df8a41 Try to avoid issues with GCC pragmas and older compilers. 
See issue #5394.
 2018-10-03 11:19:12 +02:00
antirez b0d2270290 Modules: hellodict example WIP #3: KEYRANGE. 2018-10-03 11:19:12 +02:00
antirez af2f668264 Modules: Modules: dictionary API WIP #13: Compare API exported. 2018-10-03 11:18:55 +02:00
antirez f9a3e6ef79 Modules: Modules: dictionary API WIP #12: DictCompare API. 2018-10-03 11:18:55 +02:00
antirez 01e0341a5f Modules: Modules: dictionary API WIP #11: DictCompareC API. 2018-10-03 11:18:55 +02:00
antirez f9b3ce9a56 Modules: hellodict example WIP #1: GET command. 2018-10-03 11:18:55 +02:00
antirez 36e66d861f Modules: hellodict example WIP #1: SET command. 2018-10-03 11:18:55 +02:00
antirez e33fdbe8e3 Modules: remove useless defines in hellotimer.c 2018-10-03 11:18:55 +02:00
antirez 1c8b22486d Modules: fix top comment of hellotimer.c 2018-10-03 11:18:55 +02:00
Guy Korland 7ded552d6d add missing argument to function doc 2018-10-03 11:18:55 +02:00
Pavel Skuratovich f92b3273fe Fix typo in comment 2018-10-03 11:18:55 +02:00
antirez 57b6c34308 Modules: dictionary API WIP #10: export API to modules. 2018-10-03 11:18:37 +02:00
antirez 3f82e59c0d Modules: dictionary API WIP #9: iterator returning string object. 2018-10-03 11:18:37 +02:00
antirez 6a73aca3fe Modules: dictionary API WIP #8: Iterator next/prev. 2018-10-03 11:18:37 +02:00
antirez ef8413db74 Modules: dictionary API WIP #7: don't store the context. 
Storing the context is useless, because we can't really reuse that
later. For instance in the API RM_DictNext() that returns a
RedisModuleString for the next key iterated, the user should pass the
new context, because we may run the keys of the dictionary in a
different context of the one where the dictionary was created. Also the
dictionary may be created without a context, but we may still demand
automatic memory management for the returned strings while iterating.
 2018-10-03 11:18:37 +02:00
antirez 05579e38d7 Modules: dictionary API WIP #6: implement automatic memory management. 2018-10-03 11:18:37 +02:00
antirez 11c53f8ca1 Modules: dictionary API work in progress #5: rename API for consistency. 
By using the "C" suffix for functions getting pointer/len, we can do the
same in the future for other modules APIs that need a variant with
pointer/len and that are now accepting a RedisModuleString.
 2018-10-03 11:18:37 +02:00
antirez 0bd7091b88 Modules: change RedisModuleString API to allow NULL context. 
The burden of having to always create RedisModuleString objects within a
module context was too much, especially now that we have threaded
operations and modules are doing more interesting things. The context in
the string API is currently only used for automatic memory managemnet,
so now the API was modified so that the user can opt-out this feature by
passing a NULL context.
 2018-10-03 11:18:36 +02:00
antirez 5fc16f173b Modules: dictionary API work in progress #4: reseek API. 2018-10-03 11:18:36 +02:00
antirez 45b7f77970 Modules: dictionary API work in progress #3: Iterator creation. 2018-10-03 11:18:36 +02:00
antirez 8576b0aef7 Modules: dictionary API work in progress #2: Del API. 2018-10-03 11:18:36 +02:00
antirez 4b0fa7a71f Modules: dictionary API work in progress #1. 2018-10-03 11:18:36 +02:00
antirez 282107609f Module cluster flags: use RM_SetClusterFlags() in the example. 2018-10-03 11:18:17 +02:00
antirez 18c5ab930d Module cluster flags: add RM_SetClusterFlags() API. 2018-10-03 11:18:17 +02:00
antirez 4ce6bff28f Module cluster flags: add hooks for NO_FAILOVER flag. 2018-10-03 11:18:17 +02:00
antirez 2ba52889b5 Module cluster flags: add hooks for NO_REDIRECTION flag. 2018-10-03 11:18:17 +02:00
antirez 6a39ece652 Module cluster flags: initial vars / defines added. 2018-10-03 11:18:17 +02:00
antirez 0ff35370d2 Modules: rename the reused static client to something more general. 2018-10-03 11:18:17 +02:00
antirez 2d11ee956f Modules: associate a fake client to timer context callback. 2018-10-03 11:18:17 +02:00
antirez 851b2ed30f Modules: associate a fake client to cluster message context callback. 
Fixes #5354.
 2018-10-03 11:18:17 +02:00
artix 148e491148 Cluster Manager: clusterManagerFixOpenSlot now counts node's keys in slot 
if node is neither migrating nor importing.
 2018-10-03 11:17:59 +02:00
Guy Korland 8afca145b9 No need to return "OK" 
No need to return "+OK" in this case since the result is an Array of all the nodes
 2018-10-03 11:17:59 +02:00
Guy Korland 9a278db2c0 typo fix 2018-10-03 11:17:59 +02:00
antirez 2647903616 Revert "fix repeat argument issue and reduce unnessary loop times for redis-cli." 
Reverts commit 9505dd2011
since the commit introduced the very serious bug issue #5286.
 2018-10-03 11:17:59 +02:00
Guy Korland 27b7fb5a93 Fix few typos 2018-10-03 11:17:59 +02:00
Guy Korland 233aa2d331 RedisModule_HashSet call must end with NULL 
Extended the RedisModule_HashSet doc to mark that each call must end with NULL
 2018-10-03 11:17:59 +02:00
antirez a84940721f Sentinel: document how to undo a renamed command. 2018-10-03 11:17:44 +02:00
antirez 6c8a8f2eb7 LOLWUT: split the command from version-specific implementations. 2018-09-14 12:38:07 +02:00
antirez 5c75840641 Slave removal: add a few forgotten aliases for CONFIG SET. 2018-09-14 12:38:07 +02:00
antirez 2da823c43d LOLWUT: add Redis version in the output. 
This creates an incentive to run the command and as a side effect
experience the art piece inside.
 2018-09-14 12:38:07 +02:00
antirez bfcba42066 LOLWUT: Ness -> Nees. 2018-09-14 12:38:07 +02:00
antirez efed898a4c LOLWUT: Limit maximum CPU effort. 2018-09-14 12:38:07 +02:00
antirez eb0fbd71b4 LOLWUT: change padding conditional to a more direct one. 2018-09-14 12:38:07 +02:00
Slobodan Mišković ed08feb70b Fix spelling descrive -> describe 2018-09-14 12:38:07 +02:00
antirez 2ffb4413dd LOLWUT: fix crash when col < 2. 
Close #5345.
 2018-09-14 12:38:07 +02:00
antirez 55dae693bf LOLWUT: fix structure typo in comment. 2018-09-14 12:38:07 +02:00
antirez 9b3098b93a LOLWUT: Fix license copyright year. 2018-09-14 12:38:07 +02:00
antirez 263dbadcbe LOLWUT: increase the translation factor. 2018-09-14 12:38:07 +02:00
antirez a622f6c018 LOLWUT: change default size to fit a normal terminal better. 2018-09-14 12:38:07 +02:00
antirez 38b0d25af2 LOLWUT: wrap it into a proper command. 2018-09-14 12:38:07 +02:00
antirez 34ebd8985a LOLWUT: draw Schotter by Georg Nees. 2018-09-14 12:38:07 +02:00
antirez 46286e6411 LOLWUT: draw rotated squares using trivial trigonometry. 2018-09-14 12:38:07 +02:00
antirez 2d4143fd42 LOLWUT: draw lines using Bresenham algorithm. 2018-09-14 12:38:07 +02:00
antirez 3546d9ce5f LOLWUT: Rendering of the virtual canvas to a string. 2018-09-14 12:38:07 +02:00
antirez b404a6cef7 LOLWUT: show the output verbatim in redis-cli. 2018-09-14 12:38:07 +02:00
antirez e30ba94fe8 LOLWUT: canvas structure and BSD license on top. 2018-09-14 12:38:07 +02:00
antirez 9c771145bc LOLWUT: Emit Braille unicode according to pixel pattern. 2018-09-14 12:38:06 +02:00
Jakub Vrana 4a1d6c7df5 Slave removal: capitalize Replica 2018-09-14 12:37:28 +02:00
antirez 72e0368a36 Slave removal: remove slave from integration tests descriptions. 2018-09-14 12:37:28 +02:00
antirez c7841c2b71 Slave removal: remove slave from top-level tests descriptions. 2018-09-14 12:37:28 +02:00
antirez 1b9b19ba8f Slave removal: remove slave from object.c. 2018-09-14 12:37:28 +02:00
antirez 7da266e6c4 Slave removal: remove slave from the README. 2018-09-14 12:37:28 +02:00
antirez 93d803c9a6 Slave removal: server.c logs fixed. 2018-09-14 12:37:28 +02:00
antirez 8943403208 Slave removal: remove slave from sentinel.conf when possible. 
All the occurrences translated, but the ones referring to SLAVEOF
must be intact because that describe the actual Sentinel -> Redis
protocol. In theory we could send REPLICAOF to Redis instances, but
actually this would prevent Sentinel to be compatible with older Redis
instances.
 2018-09-14 12:37:28 +02:00
antirez 7673d88df7 Slave removal: replace very few things in Sentinel. 
SENTINEL REPLICAS was added as an alias, in the configuration rewriting
now it uses known-replica, however all the rest is basically at API
level of logged events and messages having to do with the protocol, so
there is very little to do here compared to the Redis core itself, to
preserve compatibility.
 2018-09-14 12:37:28 +02:00
antirez f1de29b34e Slave removal: scripting.c logs and other stuff fixed. 2018-09-14 12:37:27 +02:00
antirez 53fe558eaf Slave removal: replication.c logs fixed. 2018-09-14 12:37:27 +02:00
antirez c92b02ddc2 Slave removal: networking.c logs fixed. 2018-09-14 12:36:59 +02:00
antirez be76ed0c60 Slave removal: blocked.c logs fixed. 2018-09-14 12:36:59 +02:00
antirez 3fd7315149 Slave removal: Make obvious in redis.conf what a replica is. 2018-09-14 12:36:59 +02:00
antirez a22168e49e Slave removal: slave mode -> replica mode text in redis-cli. 2018-09-14 12:36:59 +02:00
antirez 0e222fbec2 Slave removal: fix typo of replicaof. 2018-09-14 12:36:59 +02:00
antirez 34a5615e1a Slave removal: slave -> replica in redis.conf and output buffer option. 2018-09-14 12:36:59 +02:00
antirez 1d2fcf6f45 Slave removal: Convert cluster.c log messages and command names. 2018-09-14 12:36:59 +02:00
antirez 2546158d2d Slave removal: config.c converted + config rewriting hacks. 
Aliases added for all the commands mentioning slave. Moreover CONFIG
REWRITE will use the new names, and will be able to reuse the old lines
mentioning the old options.
 2018-09-14 12:36:44 +02:00
antirez c0952c0d15 Slave removal: redis-cli --slave -> --replica. 
--slave alias remains but is undocumented, just for backward
compatibiltiy.
 2018-09-14 12:36:44 +02:00
antirez 1f37f1dd53 Slave removal: SLAVEOF -> REPLICAOF. SLAVEOF is now an alias. 2018-09-14 12:36:44 +02:00
Amin Mesbah 7928f578e2 Use geohash limit defines in constraint check 
Slight robustness improvement, especially if the limit values are
changed, as was suggested in antires/redis#4291 [1].

[1] https://github.com/antirez/redis/pull/4291
 2018-09-14 12:36:34 +02:00
Jeffrey Lovitz bb2bed7866 CLI Help text loop verifies arg count 2018-09-14 12:36:34 +02:00
youjiali1995 246980d091 sentinel: fix randomized sentinelTimer. 2018-09-14 12:36:34 +02:00
youjiali1995 fa7de8c499 bio: fix bioWaitStepOfType. 2018-09-14 12:36:34 +02:00
Weiliang Li 7642f9d517 fix usage typo in redis-cli 2018-09-14 12:36:33 +02:00
antirez a72af0eac6 Redis 5.0 RC5. 2018-09-06 13:04:23 +02:00
antirez 1d1bf7f032 Document that effects replication is Redis 5 default. 2018-09-05 19:58:25 +02:00
antirez cfd969c777 Fix scripting tests now that we default to commands repl. 2018-09-05 19:55:55 +02:00
antirez 3e1fb5ff4c Use commands (effects) replication by default in scripts. 
See issue #5250 and issue #5292 for more info.
 2018-09-05 19:55:52 +02:00
antirez c6c71abe55 Safer script stop condition on OOM. 
Here the idea is that we do not want freeMemoryIfNeeded() to propagate a
DEL command before the script and change what happens in the script
execution once it reaches the slave. For example see this potential
issue (in the words of @soloestoy):

On master, we run the following script:

    if redis.call('get','key')
    then
        redis.call('set','xxx','yyy')
    end
    redis.call('set','c','d')

Then when redis attempts to execute redis.call('set','xxx','yyy'), we call freeMemoryIfNeeded(), and the key may get deleted, and because redis.call('set','xxx','yyy') has already been executed on master, this script will be replicated to slave.

But the slave received "DEL key" before the script, and will ignore maxmemory, so after that master has xxx and c, slave has only one key c.

Note that this patch (and other related work) was authored collaboratively in
issue #5250 with the help of @soloestoy and @oranagra.

Related to issue #5250.
 2018-09-05 19:55:49 +02:00
antirez dfbce91a14 Propagate read-only scripts as SCRIPT LOAD. 
See issue #5250 and the new comments added to the code in this commit
for details.
 2018-09-05 19:55:45 +02:00
antirez 1705e42eb2 Don't perform eviction when re-entering the event loop. 
Related to #5250.
 2018-09-05 19:55:42 +02:00
antirez a0dd6f822b Clarify why remaining may be zero in readQueryFromClient(). 
See #5304.
 2018-09-05 19:55:38 +02:00
zhaozhao.zz 2eed31a59e networking: fix unexpected negative or zero readlen 
To avoid copying buffers to create a large Redis Object which
exceeding PROTO_IOBUF_LEN 32KB, we just read the remaining data
we need, which may less than PROTO_IOBUF_LEN. But the remaining
len may be zero, if the bulklen+2 equals sdslen(c->querybuf),
in client pause context.

For example:

Time1:

python
>>> import os, socket
>>> server="127.0.0.1"
>>> port=6379
>>> data1="*3\r\n$3\r\nset\r\n$1\r\na\r\n$33000\r\n"
>>> data2="".join("x" for _ in range(33000)) + "\r\n"
>>> data3="\n\n"
>>> s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
>>> s.settimeout(10)
>>> s.connect((server, port))
>>> s.send(data1)
28

Time2:

redis-cli client pause 10000

Time3:

>>> s.send(data2)
33002
>>> s.send(data3)
2
>>> s.send(data3)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
socket.error: [Errno 104] Connection reset by peer

To fix that, we should check if remaining is greater than zero.
 2018-09-05 19:55:34 +02:00
antirez 37fb606cdc Merge branch '5.0' of github.com:/antirez/redis into 5.0 2018-09-04 13:13:47 +02:00
zhaozhao.zz 1898e6ce7f networking: optimize parsing large bulk greater than 32k 
If we are going to read a large object from network
try to make it likely that it will start at c->querybuf
boundary so that we can optimize object creation
avoiding a large copy of data.

But only when the data we have not parsed is less than
or equal to ll+2. If the data length is greater than
ll+2, trimming querybuf is just a waste of time, because
at this time the querybuf contains not only our bulk.

It's easy to reproduce the that:

Time1: call `client pause 10000` on slave.

Time2: redis-benchmark -t set -r 10000 -d 33000 -n 10000.

Then slave hung after 10 seconds.
 2018-09-04 12:54:16 +02:00
antirez 82fc63d151 Unblocked clients API refactoring. See #4418. 2018-09-04 12:54:14 +02:00
zhaozhao.zz 839bb52cc2 if master is already unblocked, do not unblock it twice 2018-09-04 12:54:09 +02:00
zhaozhao.zz 2e1cd82d96 fix multiple unblock for clientsArePaused() 2018-09-04 12:54:02 +02:00
antirez 17233080c3 Make pending buffer processing safe for CLIENT_MASTER client. 
Related to #5305.
 2018-09-04 12:53:56 +02:00
antirez 42bce87a83 Test: processing of master stream in slave -BUSY state. 
See #5297.
 2018-09-04 12:53:53 +02:00
antirez 8bf42f6031 After slave Lua script leaves busy state, re-process the master buffer. 
Technically speaking we don't really need to put the master client in
the clients that need to be processed, since in practice the PING
commands from the master will take care, however it is conceptually more
sane to do so.
 2018-09-04 12:53:48 +02:00
antirez c2b104c73c While the slave is busy, just accumulate master input. 
Processing command from the master while the slave is in busy state is
not correct, however we cannot, also, just reply -BUSY to the
replication stream commands from the master. The correct solution is to
stop processing data from the master, but just accumulate the stream
into the buffers and resume the processing later.

Related to #5297.
 2018-09-04 12:53:41 +02:00
antirez 7b75f4ae7e Allow scripts to timeout even if from the master instance. 
However the master scripts will be impossible to kill.

Related to #5297.
 2018-09-04 12:53:38 +02:00
antirez adc4e031bf Allow scripts to timeout on slaves as well. 
See reasoning in #5297.
 2018-09-04 12:53:35 +02:00
dejun.xdj 20ec1f0ced Revise the comments of latency command. 2018-09-04 12:53:32 +02:00
Chris Lamb 8e5423eb85 Correct "did not received" -> "did not receive" typos/grammar. 2018-09-04 12:53:28 +02:00
zhaozhao.zz 395063d74e remove duplicate bind in sentinel.conf 2018-09-04 12:53:21 +02:00
Salvatore Sanfilippo b221ca41da Merge pull request #5300 from SaschaRoland/xread-block-5299 
#5299 Fix blocking XREAD for streams that ran dry
 2018-08-31 18:39:46 +02:00
Sascha Roland eea0d3c50a #5299 Fix blocking XREAD for streams that ran dry 
The conclusion, that a xread request can be answered syncronously in
case that the stream's last_id is larger than the passed last-received-id
parameter, assumes, that there must be entries present, which could be
returned immediately.
This assumption fails for empty streams that actually contained some
entries which got removed by xdel, ... .

As result, the client is answered synchronously with an empty result,
instead of blocking for new entries to arrive.
An additional check for a non-empty stream is required.
 2018-08-29 19:12:29 +02:00
antirez 4cb9ee111e Add maxmemory slave behavior change in the change log. 2018-08-29 12:31:52 +02:00
zhaozhao.zz 5ad888ba17 Supplement to PR #4835, just take info/memory/command as random commands 2018-08-29 12:29:28 +02:00
zhaozhao.zz d928487f2b some commands' flags should be set correctly, issue #4834 2018-08-29 12:29:26 +02:00
Oran Agra af675f0a58 Fix unstable tests on slow machines. 
Few tests had borderline thresholds that were adjusted.

The slave buffers test had two issues, preventing the slave buffer from growing:
1) the slave didn't necessarily go to sleep on time, or woke up too early,
   now using SIGSTOP to make sure it goes to sleep exactly when we want.
2) the master disconnected the slave on timeout
 2018-08-29 12:29:23 +02:00
antirez f2cd16bea5 Document slave-ignore-maxmemory in redis.conf. 2018-08-29 12:29:18 +02:00
antirez 02d729b492 Make slave-ignore-maxmemory configurable. 2018-08-29 12:29:16 +02:00
antirez 447da44d96 Introduce repl_slave_ignore_maxmemory flag internally. 
Note: this breaks backward compatibility with Redis 4, since now slaves
by default are exact copies of masters and do not try to evict keys
independently.
 2018-08-29 12:29:14 +02:00
antirez 868b29252b Better variable meaning in processCommand(). 2018-08-29 12:29:00 +02:00
antirez 319f2ee659 Re-apply rebased #2358. 2018-08-29 12:28:55 +02:00
zhaozhao.zz 22c166da5a block: format code 2018-08-29 12:28:43 +02:00
zhaozhao.zz c03c591330 block: rewrite BRPOPLPUSH as RPOPLPUSH to propagate 2018-08-29 12:28:39 +02:00
zhaozhao.zz fcd5ef1624 networking: make setProtocolError simple and clear 
Function setProtocolError just records proctocol error
details in server log, set client as CLIENT_CLOSE_AFTER_REPLY.
It doesn't care about querybuf sdsrange, because we
will do it after procotol parsing.
 2018-08-29 12:28:35 +02:00
zhaozhao.zz 656e4b2f9d networking: just move qb_pos instead of sdsrange in processInlineBuffer 2018-08-29 12:28:31 +02:00
zhaozhao.zz 2c7972cec8 networking: just return C_OK if multibulk processing saw a <= 0 length. 2018-08-29 12:28:25 +02:00
zhaozhao.zz 1203a04f5e adjust qbuf to 26 in test case for client list 2018-08-29 12:28:22 +02:00
zhaozhao.zz aff86fa1f5 pipeline: do not sdsrange querybuf unless all commands processed 
This is an optimization for processing pipeline, we discussed a
problem in issue #5229: clients may be paused if we apply `CLIENT
PAUSE` command, and then querybuf may grow too large, the cost of
memmove in sdsrange after parsing a completed command will be
horrible. The optimization is that parsing all commands in queyrbuf
, after that we can just call sdsrange only once.
 2018-08-29 12:28:19 +02:00
Chris Lamb 45a6c5be2a Use SOURCE_DATE_EPOCH over unreproducible uname + date calls. 
See <https://reproducible-builds.org/specs/source-date-epoch/> for more
details.

Signed-off-by: Chris Lamb <chris@chris-lamb.co.uk>
 2018-08-29 12:28:16 +02:00
Chris Lamb 186df14811 Make some defaults explicit in the sentinel.conf for package maintainers 
This may look a little pointless (and it is a complete no-op change here)
but as package maintainers need to modify these lines to actually
daemonize (etc. etc) but it's far preferable if the diff is restricted to
actually changing just that bit, not adding docs, etc. The less diff the
better, in general.

Signed-off-by: Chris Lamb <chris@chris-lamb.co.uk>
 2018-08-29 12:28:12 +02:00
dejun.xdj b59f04a099 Streams: ID of xclaim command starts from the sixth argument. 2018-08-29 12:28:09 +02:00
shenlongxing a3f2437bdd Fix stream command paras 2018-08-29 12:28:06 +02:00
antirez df9112354b Fix AOF comment to report the current behavior. 
Realted to #5201.
 2018-08-29 12:28:04 +02:00
antirez 5b06bdf457 Redis 5.0 RC4. 2018-08-03 16:46:06 +02:00
antirez a4d1201eba Test suite: add --loop option. 
Very useful with --stop in order to catch heisenbugs.
 2018-08-03 12:28:04 +02:00
antirez 273d8191e1 Test suite: new --stop option. 
It pauses the test execution once the first failure is found.
 2018-08-03 12:28:04 +02:00
antirez fbbcc6a657 Streams IDs parsing refactoring. 
Related to #5184.
 2018-08-02 18:34:42 +02:00
antirez 70c4bcb7bc Test: new sorted set skiplist order consistency. 
This should be able to find new bugs and regressions about the new
sorted set update function when ZADD is used to update an element
already existing.

The test is able to find the bug fixed at 2f282aee immediately.
 2018-08-02 18:34:34 +02:00
antirez 63addc5c1e Fix zslUpdateScore() edge case. 
When the element new score is the same of prev/next node, the
lexicographical order kicks in, so we can safely update the node in
place only when the new score is strictly between the adjacent nodes
but never equal to one of them.

Technically speaking we could do extra checks to make sure that even if the
score is the same as one of the adjacent nodes, we can still update on
place, but this rarely happens, so probably not a good deal to make it
more complex.

Related to #5179.
 2018-08-02 18:34:34 +02:00
antirez 724740cc19 More commenting of zslUpdateScore(). 2018-08-02 18:34:34 +02:00
antirez ddc87eef4f Explain what's the point of zslUpdateScore() in top comment. 2018-08-02 18:34:34 +02:00
antirez 741f29ea52 Remove old commented zslUpdateScore() from source. 2018-08-02 18:34:34 +02:00
antirez 201168368a Optimize zslUpdateScore() as asked in #5179. 2018-08-02 18:34:34 +02:00
antirez 8c297e8b43 zsetAdd() refactored adding zslUpdateScore(). 2018-08-02 18:34:34 +02:00
dejun.xdj bd2f3f6bb1 Streams: rearrange the usage of '-' and '+' IDs in stream commands. 2018-08-02 18:34:34 +02:00
dejun.xdj c0c06b8456 Streams: add mmid_supp argument in streamParseIDOrReply(). 
If 'mmid_supp' is set to 0, "-" and "+" will be
treated as an invalid ID.
 2018-08-02 18:34:34 +02:00
antirez ab237a8e26 Minor improvements to PR #5187. 2018-08-02 18:34:34 +02:00
Oran Agra 1ce3cf7a8f test suite conveniency improvements 
* allowing --single to be repeated
* adding --only so that only a specific test inside a unit can be run
* adding --skiptill useful to resume a test that crashed passed the problematic unit.
  useful together with --clients 1
* adding --skipfile to use a file containing list of tests names to skip
* printing the names of the tests that are skiped by skipfile or denytags
* adding --config to add config file options from command line
 2018-08-02 18:34:34 +02:00
Oran Agra 3662289995 add DEBUG LOG, to to assist test suite debugging 2018-08-02 18:34:34 +02:00
antirez 83d4311acd Cluster cron announce IP minor refactoring. 2018-08-02 18:34:04 +02:00
shenlongxing a633f8e130 Fix cluster-announce-ip memory leak 2018-08-02 18:34:04 +02:00
antirez 24c455381a Tranfer -> transfer typo fixed. 2018-08-02 18:34:04 +02:00
zhaozhao.zz c609f240a5 refactor dbOverwrite to make lazyfree work 2018-08-02 18:34:04 +02:00
antirez 9e97173988 Refactoring: replace low-level checks with writeCommandsDeniedByDiskError(). 2018-08-02 18:34:04 +02:00
antirez 0e77cef096 Fix writeCommandsDeniedByDiskError() inverted return value. 2018-08-02 18:34:04 +02:00
antirez acfe9d138a Better top comment for writeCommandsDeniedByDiskError(). 2018-08-02 18:34:04 +02:00
antirez 4e933e0059 Introduce writeCommandsDeniedByDiskError(). 2018-08-02 18:34:04 +02:00
WuYunlong 41607dfd25 Consider aof write error as well as rdb in lua script. 2018-08-02 18:34:04 +02:00
Salvatore Sanfilippo 1d073a64f7 Merge pull request #5168 from rpv-tomsk/issue-5033 
INFO CPU: higher precision of reported values
 2018-07-30 18:03:15 +02:00
Guy Korland 2db31fd4bb Few typo fixes 2018-07-30 18:00:32 +02:00
antirez 64242757d7 Add year in log. 
User: "is there a reason why redis server logs  are missing the year in
the "date time"?"

Me: "I guess I did not imagine it would be stable enough to run for
several years".
 2018-07-30 18:00:30 +02:00
antirez 50be4a1f5c Document dynamic-hz in the example redis.conf. 2018-07-30 18:00:27 +02:00
antirez 9a76472d71 Make dynamic hz actually configurable. 2018-07-30 18:00:24 +02:00
antirez a330d06c82 Control dynamic HZ via server configuration. 2018-07-30 18:00:21 +02:00
antirez d42602ffc8 Dynamic HZ: adapt cron frequency to number of clients. 2018-07-30 18:00:19 +02:00
antirez 7b5f0223f8 Dynamic HZ: separate hz from the configured hz. 
This way we can remember what the user configured HZ is, but change the
actual HZ dynamically if needed in the dynamic HZ feature
implementation.
 2018-07-30 18:00:16 +02:00
antirez 037b00dece Remove useless conditional from emptyDb(). 
Related to #4852.
 2018-07-30 17:59:52 +02:00
antirez 0e97ae79b0 Make emptyDb() change introduced in #4852 simpler to read. 2018-07-30 17:59:49 +02:00
zhaozhao.zz f7740fafbd optimize flushdb, avoid useless loops 2018-07-30 17:59:46 +02:00
zhaozhao.zz 0c00837669 Streams: fix xdel memory leak 2018-07-30 17:59:43 +02:00
antirez dc600a25cd Example the magic +1 in migrateCommand(). 
Related to #5154.
 2018-07-30 17:59:40 +02:00
antirez d6827ab638 Make changes of PR #5154 hopefully simpler. 2018-07-30 17:59:37 +02:00
WuYunlong 89ec14531b Do not migrate already expired keys. 2018-07-30 17:59:35 +02:00
Pavel Rochnyack cd25ed17b9 INFO CPU: higher precision of reported values 
Closes: #5033
 2018-07-25 21:50:24 +07:00
antirez 6bfb4745fa Streams: refactoring of next entry seek in the iterator. 
After #5161 the code could be made a bit more obvious for newcomers.
 2018-07-24 11:16:25 +02:00
zhaozhao.zz 4724548e07 Streams: skip master fileds only when we are going forward in streamIteratorGetID 2018-07-24 11:16:21 +02:00
Oran Agra 4b79fdf1d6 fix slave buffer test suite false positives 
it looks like on slow machines we're getting:
[err]: slave buffer are counted correctly in tests/unit/maxmemory.tcl
Expected condition '$slave_buf > 2*1024*1024' to be true (16914 > 2*1024*1024)

this is a result of the slave waking up too early and eating the
slave buffer before the traffic and the test ends.
 2018-07-24 11:16:15 +02:00
antirez a1e081f719 string2ll(): better commenting. 2018-07-24 11:16:11 +02:00
dsomeshwar 8b4fe752ec removing redundant check 2018-07-24 11:16:07 +02:00
antirez 9e5bf0471a Restore string2ll() to original version. 
See PR #5157.
 2018-07-24 11:16:05 +02:00
Oran Agra c2ecdcde98 fix recursion typo in zmalloc_usable 2018-07-24 11:16:00 +02:00
antirez 4f742bd6bf string2ll(): remove duplicated check for special case. 
Related to #5157. The PR author correctly indentified that the check was
duplicated, but removing the second one introduces a bug that was fixed
in the past (hence the duplication). Instead we can remove the first
instance of the check without issues.
 2018-07-24 11:15:57 +02:00
antirez a4efac0067 string2ll(): test for NULL pointer in all the cases. 2018-07-24 11:15:54 +02:00
antirez 2c07c107cb Change 42 to 1000 as warning level for cached scripts. 
Related to #4883.
 2018-07-23 18:46:12 +02:00
Itamar Haber 270903d6b2 Adds Lua overheads to MEMORY STATS, smartens the MEMORY DOCTOR 2018-07-23 18:46:12 +02:00
Itamar Haber faf3dbfcf9 Adds memory information about the script's cache to INFO 
Implementation notes: as INFO is "already broken", I didn't want to break it further. Instead of computing the server.lua_script dict size on every call, I'm keeping a running sum of the body's length and dict overheads.

This implementation is naive as it **does not** take into consideration dict rehashing, but that inaccuracy pays off in speed ;)

Demo time:

```bash
$ redis-cli info memory | grep "script"
used_memory_scripts:96
used_memory_scripts_human:96B
number_of_cached_scripts:0
$ redis-cli eval "" 0 ; redis-cli info memory | grep "script"
(nil)
used_memory_scripts:120
used_memory_scripts_human:120B
number_of_cached_scripts:1
$ redis-cli script flush ; redis-cli info memory | grep "script"
OK
used_memory_scripts:96
used_memory_scripts_human:96B
number_of_cached_scripts:0
$ redis-cli eval "return('Hello, Script Cache :)')" 0 ; redis-cli info memory | grep "script"
"Hello, Script Cache :)"
used_memory_scripts:152
used_memory_scripts_human:152B
number_of_cached_scripts:1
$ redis-cli eval "return redis.sha1hex(\"return('Hello, Script Cache :)')\")" 0 ; redis-cli info memory | grep "script"
"1be72729d43da5114929c1260a749073732dc822"
used_memory_scripts:232
used_memory_scripts_human:232B
number_of_cached_scripts:2
✔ 19:03:54 redis [lua_scripts-in-info-memory L ✚…⚑] $ redis-cli evalsha 1be72729d43da5114929c1260a749073732dc822 0
"Hello, Script Cache :)"
```
 2018-07-23 18:46:12 +02:00
antirez 49841a54db Fix merge errors. 
For some reason I made a merge fiasco between 5.0 and unstable.
This fixes the error introduced by merging unstable.
 2018-07-23 18:36:54 +02:00
antirez 77a7ec7200 Merge branch 'unstable' into 5.0 branch 2018-07-23 18:29:33 +02:00
antirez 4ff47a0b9b Top comment clientsCron(). 2018-07-20 09:46:18 +02:00
antirez aba6855282 Clarify that clientsCronTrackExpansiveClients() indexes may jump ahead. 2018-07-20 09:36:48 +02:00
antirez be88c0b16a Rename INFO CLIENT max buffers field names for correctness. 
They are actually delayed a few seconds, so let's call them "recent".
 2018-07-19 17:38:20 +02:00
antirez 0cf3794e6e Fix wrong array index variable in getExpansiveClientsInfo(). 2018-07-19 17:34:15 +02:00
antirez ea3a20c5d0 Change INFO CLIENTS sections to report pre-computed max/min client buffers. 2018-07-19 17:16:19 +02:00
antirez 8f7e496ba5 Rename var in clientsCronTrackExpansiveClients() for clarity. 2018-07-19 13:59:13 +02:00
antirez 8d617596f1 Implement a function to retrieve the expansive clients mem usage. 2018-07-19 13:58:04 +02:00
antirez 85a1b4f807 clientsCronTrackExpansiveClients() actual implementation. 2018-07-19 13:54:20 +02:00
antirez d4c5fc57db clientsCronTrackExpansiveClients() skeleton and ideas. 2018-07-19 13:49:00 +02:00
antirez 1c95c07596 Make vars used only by INFO CLIENTS local to the block. 
Related to #4727.
 2018-07-19 12:23:05 +02:00
Salvatore Sanfilippo 16b8d364cc Merge pull request #4727 from kingpeterpaule/redis-fix-info-cli 
move getClientsMaxBuffers func into  info clients command
 2018-07-19 12:21:35 +02:00
antirez 0aca977c47 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-07-18 17:41:37 +02:00
antirez 313b2240ae In addReplyErrorLength() only panic when replying to slave. 
See #5135 for more context.
 2018-07-18 17:41:16 +02:00
antirez 6183f0590d Refine comment in addReplyErrorLength() about replying to masters/slaves. 
See #5135 for some context.
 2018-07-18 17:40:07 +02:00
Salvatore Sanfilippo 22e9321c3e Merge pull request #5138 from oranagra/improve_defrag_test 
make active defrag test more stable
 2018-07-18 11:05:03 +02:00
Oran Agra f89c93c8ad make active defrag test more stable 
on slower machines, the active defrag test tended to fail.
although the fragmentation ratio was below the treshold, the defragger was
still in the middle of a scan cycle.

this commit changes:
- the defragger uses the current fragmentation state, rather than the cache one
  that is updated by server cron every 100ms. this actually fixes a bug of
  starting one excess scan cycle
- the test lets the defragger use more CPU cycles, in hope that the defrag
  will be faster, but also give it more time before we give up.
 2018-07-18 10:16:33 +03:00
Salvatore Sanfilippo 8213f64d64 Merge pull request #5122 from trevor211/allowWritesWhenAofDisabled 
Accept write commands if persisting is disabled
 2018-07-17 18:08:46 +02:00
Salvatore Sanfilippo 46fd927800 Merge pull request #4237 from aspirewit/update-comment 
Update the comment
 2018-07-17 18:04:55 +02:00
antirez 6201f7b4e0 Streams: better error when $ is given with XREADGROUP. 2018-07-17 17:54:10 +02:00
Salvatore Sanfilippo 4bff45c7a6 Merge pull request #5136 from 0xtonyxia/fix-xread-id-parse 
Fix xreadgroup with '$' ID.
 2018-07-17 17:50:31 +02:00
antirez afc7e08a20 Panic when we are sending an error to our master/slave. 
Related to #5135, see discussion there.
 2018-07-17 17:42:30 +02:00
Salvatore Sanfilippo e03358c0d9 Merge pull request #5135 from oranagra/rare_repl_corruption 
fix rare replication stream corruption with disk-based replication
 2018-07-17 17:33:58 +02:00
dejun.xdj 846cf12ae5 Streams: remove meaningless if condition. 
It's already checked if xreadgroup is set and groupname is NULL.
 2018-07-17 18:23:47 +08:00
dejun.xdj 6501b6bb6d Streams: return an error message if using xreadgroup with '$' ID. 
Redis will always return an empty result when '$' ID is specified
with xreadgroup command, it's meaningless.
 2018-07-17 18:19:10 +08:00
Oran Agra d55598988b fix rare replication stream corruption with disk-based replication 
The slave sends \n keepalive messages to the master while parsing the rdb,
and later sends REPLCONF ACK once a second. rarely, the master recives both
a linefeed char and a REPLCONF in the same read, \n*3\r\n$8\r\nREPLCONF\r\n...
and it tries to trim two chars (\r\n) from the query buffer,
trimming the '*' from *3\r\n$8\r\nREPLCONF\r\n...

then the master tries to process a command starting with '3' and replies to
the slave a bunch of -ERR and one +OK.
although the slave silently ignores these (prints a log message), this corrupts
the replication offset at the slave since the slave increases the replication
offset, and the master did not.

other than the fix in processInlineBuffer, i did several other improvments
while hunting this very rare bug.

- when redis replies with "unknown command" it includes a portion of the
  arguments, not just the command name. so it would be easier to understand
  what was recived, in my case, on the slave side,  it was -ERR, but
  the "arguments" were the interesting part (containing info on the error).
- about a year ago i added code in addReplyErrorLength to print the error to
  the log in case of a reply to master (since this string isn't actually
  trasmitted to the master), now changed that block to print a similar log
  message to indicate an error being sent from the master to the slave.
  note that the slave is marked as CLIENT_SLAVE only after PSYNC was received,
  so this will not cause any harm for REPLCONF, and will only indicate problems
  that are gonna corrupt the replication stream anyway.
- two places were c->reply was emptied, and i wanted to reset sentlen
  this is a precaution (i did not actually see such a problem), since a
  non-zero sentlen will cause corruption to be transmitted on the socket.
 2018-07-17 12:51:49 +03:00
antirez cefe21d28a dict.c: remove a few trailing spaces. 2018-07-17 10:39:47 +02:00
Salvatore Sanfilippo 4fc2099235 Merge pull request #5128 from kingpeterpaule/remove-one-loop-in-freeMemoryIfNeeded 
remove ineffective loop in dictGetSomeKeys.
 2018-07-17 10:38:55 +02:00
Salvatore Sanfilippo 9fbd49bbaf Merge pull request #5113 from 0xtonyxia/using-compare-func-instead 
Streams: using streamCompareID() instead of direct compare.
 2018-07-16 18:34:35 +02:00
Salvatore Sanfilippo cab396761e Merge pull request #5127 from oranagra/sds_req_type 
bugfix in sdsReqType creating 64bit sds headers on 32bit systems
 2018-07-16 18:32:14 +02:00
antirez f9c84d6d39 Hopefully improve commenting of #5126. 
Reading the PR gave me the opportunity to better specify what the code
was doing in places where I was not immediately sure about what was
going on. Moreover I documented the structure in server.h so that people
reading the header file will immediately understand what the structure
is useful for.
 2018-07-16 17:56:54 +02:00
Salvatore Sanfilippo e22a1218bc Merge pull request #5126 from oranagra/slave_buf_memory_2 
slave buffers were wasteful and incorrectly counted causing eviction
 2018-07-16 17:45:50 +02:00
Salvatore Sanfilippo 28dd8dd150 Merge pull request #5132 from soloestoy/propagate-xdel-correctly 
Streams: correctly propagate xdel if needed
 2018-07-16 16:04:32 +02:00
Oran Agra bf680b6f8c slave buffers were wasteful and incorrectly counted causing eviction 
A) slave buffers didn't count internal fragmentation and sds unused space,
   this caused them to induce eviction although we didn't mean for it.

B) slave buffers were consuming about twice the memory of what they actually needed.
- this was mainly due to sdsMakeRoomFor growing to twice as much as needed each time
  but networking.c not storing more than 16k (partially fixed recently in 237a38737).
- besides it wasn't able to store half of the new string into one buffer and the
  other half into the next (so the above mentioned fix helped mainly for small items).
- lastly, the sds buffers had up to 30% internal fragmentation that was wasted,
  consumed but not used.

C) inefficient performance due to starting from a small string and reallocing many times.

what i changed:
- creating dedicated buffers for reply list, counting their size with zmalloc_size
- when creating a new reply node from, preallocate it to at least 16k.
- when appending a new reply to the buffer, first fill all the unused space of the
  previous node before starting a new one.

other changes:
- expose mem_not_counted_for_evict info field for the benefit of the test suite
- add a test to make sure slave buffers are counted correctly and that they don't cause eviction
 2018-07-16 16:43:42 +03:00
zhaozhao.zz 73306c6fed Streams: correctly propagate xdel if needed 2018-07-16 20:48:07 +08:00
antirez 103c5a1a3c Add a few comments to streamIteratorRemoveEntry(). 2018-07-16 12:41:55 +02:00
Salvatore Sanfilippo a317f55d25 Merge pull request #5131 from soloestoy/optimize-xdel 
Streams: free lp if all elements are deleted
 2018-07-16 12:39:38 +02:00
antirez 185e0d9cd8 Modify XINFO field from last-id to last-generated-id. 
Related to #5129.
 2018-07-16 12:25:26 +02:00
Salvatore Sanfilippo 4215e74be5 Merge pull request #5129 from soloestoy/xinfo-show-last-id 
Streams: show last id for streams and groups
 2018-07-16 12:24:14 +02:00
zhaozhao.zz c9324f81a2 Streams: free lp if all elements are deleted 2018-07-16 15:57:41 +08:00
paule b6ce7d5ddc Update dict.c 
change coding style.
 2018-07-16 14:29:59 +08:00
zhaozhao.zz b4ba5ac840 Streams: show last id for streams and groups 2018-07-16 13:24:54 +08:00
peterpaule 816fc6cbc6 remove one ineffective loop in dictGetSomeKeys. 2018-07-16 11:28:22 +08:00
Oran Agra b05a22e261 bugfix in sdsReqType creating 64bit sds headers on 32bit systems 2018-07-15 18:24:18 +03:00
dejun.xdj 491682a668 Streams: using streamCompareID() instead of direct compare in block.c. 2018-07-14 15:03:05 +08:00
dejun.xdj a2177cd2b6 Streams: add streamCompareID() declaration in stream.h. 2018-07-14 15:02:24 +08:00
dejun.xdj 0484dbcf68 Streams: using streamCompareID() instead of direct compare. 2018-07-14 20:34:06 +08:00
WuYunlong 2d4366c544 Accept write commands if persisting is disabled, 
event if we do have problems persisting on disk
previously.
 2018-07-14 09:06:23 +08:00
Salvatore Sanfilippo ab33bcd346 Merge pull request #5120 from andrewsensus/leap-year-comment-patch-1 
update leap year comment
 2018-07-13 18:18:45 +02:00
antirez 2352a51992 Test: XDEL fuzz testing. Remove and check stage. 2018-07-13 17:58:17 +02:00
antirez 3d7d20b7f3 Test: fix lshuffle by providing the "K" combinator. 2018-07-13 17:52:39 +02:00
antirez 967ad3643c Test: add lshuffle in the Tcl utility functions set. 2018-07-13 17:51:03 +02:00
antirez d6efd5fc1d Test: XDEL fuzz testing, stream creation. 2018-07-13 17:47:26 +02:00
andrewsensus 8dc08ae215 update leap year comment 2018-07-13 09:46:30 -06:00
antirez 699971535e Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-07-13 17:41:10 +02:00
antirez a93f8f0998 Test: XDEL basic test. 2018-07-13 17:40:48 +02:00
Salvatore Sanfilippo a44a59340c Merge pull request #5119 from trevor211/fixSlowlogConfig 
Fix slowlog config
 2018-07-13 17:35:15 +02:00
WuYunlong d6ba4fd56d Fix config set slowlog-log-slower-than and condition in createLatencyReport 2018-07-13 17:53:55 +08:00
WuYunlong b3660be8f5 Add test in slowlog.tcl 2018-07-13 17:51:06 +08:00
artix d4182a0a0d Cluster Manager: more checks on --cluster-weight option. 2018-07-13 10:51:58 +02:00
artix d222eda9f0 Redis-trib deprecated: it no longer works and it 
outputs a warning to the user.
 2018-07-13 10:51:58 +02:00
artix 513eb5728c Cluster Manager: auth support (-a argument). 2018-07-13 10:51:58 +02:00
Salvatore Sanfilippo f3980bb9fd Merge pull request #5115 from shenlongxing/patch-1 
Delete unused role checking.
 2018-07-12 19:07:40 +02:00
Shen Longxing c2a85fb3c5 Delete unused role checking. 
When check rdb file, it is unnecessary to check role.
 2018-07-12 21:21:37 +08:00
Salvatore Sanfilippo 4cb5bd4e9d Merge pull request #4820 from charpty/wip-serverc-simplify 
Remove unnecessary return statements
 2018-07-12 13:42:15 +02:00
antirez 8d6b7861a7 Add regression test for #5111. 2018-07-12 13:35:17 +02:00
antirez b6260a0270 Streams: when re-delivering because of SETID, reset deliveries counter. 
We don't want to increment the deliveries here, because the sysadmin
reset the consumer group so the desire is likely to restart processing,
and having the PEL polluted with old information is not useful but
probably confusing.

Related to #5111.
 2018-07-12 13:19:39 +02:00
antirez a7c180e559 Simplify duplicated NACK #5112 fix. 
We don't really need to distinguish between the case the consumer is the
same or is a different one.
 2018-07-12 13:15:54 +02:00
Salvatore Sanfilippo bf4def0fbc Merge pull request #5112 from 0xtonyxia/fix-xreadgroup-crash-after-setid 
Streams: fix xreadgroup crash after xgroup SETID is sent.
 2018-07-12 13:04:22 +02:00
Salvatore Sanfilippo 167705519b Merge pull request #5114 from oranagra/defrag_32 
Active defrag fixes for 32bit builds (again)
 2018-07-11 22:07:46 +02:00
Oran Agra 920158ec81 Active defrag fixes for 32bit builds (again) 
* overflow in jemalloc fragmentation hint to the defragger
 2018-07-11 16:09:00 +03:00
Salvatore Sanfilippo f45e790125 Merge pull request #4967 from JingchengLi/unstable 
fix repeat argument issue and reduce unnessary loop times for redis-cli.
 2018-07-10 15:13:20 +02:00
tengfeng 9505dd2011 fix repeat argument issue and reduce unnessary loop times for redis-cli. 2018-07-10 20:58:01 +08:00
antirez 0420c3276f Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-07-10 12:06:44 +02:00
antirez 28e95c7c52 Streams: fix typo "consumer". 2018-07-10 12:04:31 +02:00
antirez a8c1bb310e Streams: fix new XREADGROUP sync logic. 2018-07-10 12:02:03 +02:00
antirez 1a02b5f6ee Streams: make blocking for > a truly special case. 
To simplify the semantics of blocking for a group, this commit changes
the implementation to better match the description we provide of
conusmer groups: blocking for > will make the consumer waiting for new
elements in the group. However blocking for any other ID will always
serve the local history of the consumer.

However it must be noted that the > ID is actually an alias for the
special ID ms/seq of UINT64_MAX,UINT64_MAX.
 2018-07-10 11:34:17 +02:00
antirez a71e814853 Streams: send an error to consumers blocked on non-existing group. 
To detect when the group (or the whole key) is destroyed to send an
error to the consumers blocked in such group is a problem, so we leave
the consumers listening, the sysadmin is free to create or destroy
groups assuming she/he knows what to do. However a client may be blocked
in a given consumer group, that is later destroyed. Then the stream
receives new elements. In that case there is no sane behavior to serve
the consumer... but to report an error about the group no longer
existing.

More about detecting this synchronously and why it is not done:

1. Normally we don't do that, we leave clients blocked for other data
types such as lists.

2. When we free a stream object there is no longer information about
what was the key it was associated with, so while destroying the
consumer groups we miss the info to unblock the clients in that moment.

3. Objects can be reclaimed in other threads where it is no longer safe
to do client operations.
 2018-07-10 11:19:06 +02:00
antirez 09327f11dd Streams: fix unblocking logic into a consumer group. 
When a client blocks for a consumer group, we don't know the actual ID
we want to be served: other clients blocked in the same consumer group
may be served first, so the consumer group latest delivered ID changes.
This was not handled correctly, all the clients in the consumer group
were unblocked without data but the first.
 2018-07-10 11:11:41 +02:00
dejun.xdj 3f8a3efe5f Streams: fix xreadgroup crash after xgroup SETID is sent. 
For issue #5111.
 2018-07-10 16:26:13 +08:00
Salvatore Sanfilippo 7239e9ca5f Merge pull request #5095 from 0xtonyxia/fix-indentation 
Fix indentation.
 2018-07-09 13:56:39 +02:00
dejun.xdj 61f12973f7 Bugfix: PEL is incorrect when consumer is blocked using xreadgroup with NOACK option. 
Save NOACK option into client.blockingState structure.
 2018-07-09 13:40:29 +02:00
antirez b67f027699 redis-cli: fix #4990 additional argument in help. 2018-07-09 13:36:47 +02:00
antirez 18d65849fd redis-cli: fix #5096 double error message. 2018-07-09 13:25:48 +02:00
antirez 846203279c redis-cli: cliConnect() flags CC_FORCE and CC_QUIET. 
We need CC_QUIET in order to fix #5096 by silently failing if needed.
 2018-07-09 13:24:30 +02:00
Amit Dey a3a5a25fa0 fixing broken link in CONTRIBUTING 
http://help.github.com/send-pull-requests/ 
is no longer supported

this change modifies the link to the working one
https://help.github.com/articles/creating-a-pull-request/
 2018-07-09 13:14:57 +02:00
dejun.xdj 289d8d9c2c CLIENT UNBLOCK: fix client unblock help message. 2018-07-09 13:03:57 +02:00
minkikim89 62a4a8c1d5 fix whitespace in redis-cli.c 2018-07-09 12:59:13 +02:00
WuYunlong 0a5805d7f1 fix compile warning in addReplySubcommandSyntaxError 2018-07-09 12:57:12 +02:00
antirez cb78c84235 Use nolocks_localtime() for safer logging. 2018-07-04 16:50:22 +02:00
antirez 81778d91bf Cache timezone and daylight active flag for safer logging. 
With such information will be able to use a private localtime()
implementation serverLog(), which does not use any locking and is both
thread and fork() safe.
 2018-07-04 16:45:00 +02:00
antirez 18d8205bbd Localtime: clarify is_leap_year() working with comments. 2018-07-04 15:50:41 +02:00
antirez 2964414450 Localtime: fix comment about leap year. 2018-07-04 15:16:54 +02:00
antirez 0ea39aa404 Localtime: fix daylight saving adjustment. Use * not +. 2018-07-04 15:16:20 +02:00
antirez 6614d30599 Localtime: fix daylight time documentation and computation. 2018-07-04 14:09:07 +02:00
dejun.xdj 46f5a2ca07 Fix indentation. 2018-07-04 20:04:06 +08:00
antirez 243c5a7a30 Localtime: add a test main() function to check the output. 2018-07-04 13:46:31 +02:00
antirez c25ee35a8b Localtime: day of month is 1 based. Convert from 0 based "days". 2018-07-04 13:35:40 +02:00
antirez b9f3383029 Localtime: fix timezone adjustment. 2018-07-04 13:35:15 +02:00
antirez 0c12cbedbb Localtime: compute year, month and day of the month. 2018-07-04 13:25:55 +02:00
antirez 06ca400f95 Localtime: basics initial calculations. Year missing. 2018-07-04 13:16:53 +02:00
antirez 6a52906758 Localtime function skeleton and file added. 2018-07-04 12:56:12 +02:00
Jack Drogon 93238575f7 Fix typo 2018-07-03 18:19:46 +02:00
antirez 94b3ee6142 Clarify the pending_querybuf field of clients. 2018-07-03 13:25:41 +02:00
antirez 549b8b9973 Improve style of PR #5084. 2018-07-03 13:23:29 +02:00
Salvatore Sanfilippo 526b30a7ce Merge pull request #5084 from chendq8/pending-querybuf 
limit the size of pending-querybuf in masterclient
 2018-07-03 13:19:37 +02:00
antirez 677d10b2a8 Set repl_down_since to zero on state change. 
PR #5081 fixes an "interesting" bug about Redis Cluster failover but in
general about the updating of repl_down_since, that is used in order to
count the time a slave was left disconnected from its master.

While the fix provided resolves the specific issue, in general the
validity of repl_down_since is limited to states that are different
than the state CONNECTED, and the disconnected time is set when the
state is DISCONNECTED. However from CONNECTED to other states, the state
machine must always go to DISCONNECTED first. So it makes sense to set
the field to zero (since it is meaningless in that context) when the
state is set to CONNECTED.
 2018-07-03 12:42:14 +02:00
Salvatore Sanfilippo 02e385169e Merge pull request #5081 from trevor211/fixClusterFailover 
cluster failover bug
 2018-07-03 12:36:53 +02:00
chendianqiang cbb2ac0799 Merge branch 'unstable' into pending-querybuf 2018-07-03 10:07:26 +08:00
antirez 2edcafb35d addReplySubSyntaxError() renamed to addReplySubcommandSyntaxError(). 2018-07-02 18:49:34 +02:00
Salvatore Sanfilippo bc6a004588 Merge pull request #4998 from itamarhaber/module_command_help 
Module command help
 2018-07-02 18:46:56 +02:00
Salvatore Sanfilippo ee09b5ed3a Merge pull request #5071 from akshaynagpal/patch-2 
Added link to Google Group
 2018-07-02 18:41:58 +02:00
Salvatore Sanfilippo f03ad96236 Merge pull request #5068 from shenlongxing/fix-rename-command 
fix empty string for sentinel rename-command
 2018-07-02 18:40:35 +02:00
Salvatore Sanfilippo e4881cd083 Merge pull request #5090 from trevor211/test_helper_tcl 
fix tests/test_helper.tcl with --wait-server option.
 2018-07-02 17:51:04 +02:00
WuYunlong 2833cfbe39 fix tests/test_helper.tcl with --wait-server option. 
Issue #5063 added --wait-server option, but can not work.
 2018-07-02 19:26:52 +08:00
chendianqiang 7de1ada070 limit the size of pending-querybuf in masterclient 2018-07-01 14:43:53 +08:00
WuYunlong 2e167f7d0e fix server.repl_down_since resetting, so that slaves could failover 
automatically as expected.
 2018-06-30 09:39:08 +08:00
WuYunlong aeb7bc3ec6 cluster.tcl: Add master consecutively down test. 2018-06-30 09:29:35 +08:00
antirez d751d98b50 Change CLIENT LIST TYPE help string. 
Making it more similar to KILL.
 2018-06-29 18:03:00 +02:00
Salvatore Sanfilippo a0b05a0424 Merge pull request #5075 from soloestoy/client-list-types 
FEATURE: implements client list type option
 2018-06-29 18:00:32 +02:00
Salvatore Sanfilippo aa2c390ef8 Merge pull request #5074 from soloestoy/fix-compile-warning 
fix some compile warnings
 2018-06-29 17:56:47 +02:00
Salvatore Sanfilippo a4ef94d2f7 Merge pull request #5076 from 0xtonyxia/add-no-auth-warning-option 
Add no auth warning option
 2018-06-29 13:32:58 +02:00
dejun.xdj 9f1856268c Check if password is used on command line interface. 2018-06-28 19:09:22 +08:00
dejun.xdj 1139070a0f Fix trailing white space. 2018-06-28 18:47:28 +08:00
dejun.xdj bbd0ca9522 Fix code format issue. 2018-06-28 18:39:42 +08:00
dejun.xdj 7becf54e51 Don't output password warning message when --no-auth-warning is used. 2018-06-28 18:38:24 +08:00
dejun.xdj bde05e9c32 Avoid -Woverlength-strings compile warning. 
Using another fprintf call to output the rest help message.
 2018-06-28 18:22:06 +08:00
antirez 5baf50d850 Rax library updated (node callback). 2018-06-28 12:19:06 +02:00
dejun.xdj 0b74fd6766 Add --no-auth-warning help message. 2018-06-28 18:17:04 +08:00
zhaozhao.zz b9cbd04b57 clients: add type option for client list 2018-06-28 17:43:05 +08:00
zhaozhao.zz f5538642cc clients: show pubsub flag in client list 2018-06-28 17:28:38 +08:00
zhaozhao.zz 1fcf2737a6 fix some compile warnings 2018-06-28 17:22:59 +08:00
Akshay Nagpal 007e3cbd70 Added link to Google Group 2018-06-27 20:08:39 -04:00
antirez ab55f9da5e Make CLIENT HELP output nicer to the eyes. 2018-06-28 00:21:32 +02:00
antirez 75f1a7bdfc Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-06-28 00:17:21 +02:00
antirez 4a70ff7451 Add unblock in CLIENT HELP. 2018-06-28 00:17:10 +02:00
shenlongxing 3c27db1cd9 fix empty string for sentinel rename-command 2018-06-28 01:08:55 +08:00
Salvatore Sanfilippo f7b21bc71e Merge pull request #5066 from oranagra/defrag_jemalloc5_fix 
minor fix in creating a stream NACK for rdb and defrag tests
 2018-06-27 18:56:49 +02:00
Salvatore Sanfilippo 730a4cfa00 Merge pull request #5067 from mpaltun/mpaltun-doc-fix 
Streams documentation fix
 2018-06-27 18:54:27 +02:00
antirez 2214043b5c CLIENT UNBLOCK: support unblocking by error. 2018-06-27 18:51:06 +02:00
Mustafa Paltun 010dc172d4 Update t_stream.c 2018-06-27 16:35:30 +03:00
Mustafa Paltun 6d0acb330a Update help.h 2018-06-27 16:33:31 +03:00
Oran Agra de495ee7ab minor fix in creating a stream NACK for rdb and defrag tests 2018-06-27 15:34:17 +03:00
antirez 71295ee305 CLIENT UNBLOCK implemented. 2018-06-27 14:08:42 +02:00
antirez fb39bfd7af Take clients in a ID -> Client handle dictionary. 2018-06-27 14:08:42 +02:00
antirez ed65d734e7 CLIENT ID implemented. 2018-06-27 14:08:42 +02:00
Salvatore Sanfilippo 345b4809f4 Merge pull request #5063 from oranagra/test_suite_improvements 
test suite infra improvements and fix
 2018-06-27 14:07:25 +02:00
Salvatore Sanfilippo 35c5f3fa7d Merge pull request #5065 from oranagra/defrag_jemalloc5 
defrag hint support for jemalloc 5, and active defrag for streams
 2018-06-27 14:05:16 +02:00
Oran Agra 5616d4c603 add active defrag support for streams 2018-06-27 15:00:41 +03:00
Oran Agra e8099cabd1 add defrag hint support into jemalloc 5 2018-06-27 10:52:39 +03:00
Oran Agra 751eea24c4 test suite infra improvements and fix 
* fail the test (exit code) in case of timeout.
* add --wait-server to allow attaching a debugger
* add --dont-clean to keep log files when tests are done
 2018-06-26 20:23:55 +03:00
Salvatore Sanfilippo bb666d445d Merge pull request #5027 from tigertv/unstable 
Update sort.c
 2018-06-26 16:13:20 +02:00
antirez b9058c7328 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-06-26 16:09:09 +02:00
antirez 438317796b Sentinel: test command renaming feature. 2018-06-26 16:08:43 +02:00
Salvatore Sanfilippo eb052ba93a Merge pull request #5059 from guybe7/fix_restore_warning 
Fix compiler warning in restoreCommand
 2018-06-26 14:41:48 +02:00
antirez 27178a3fde Fix type of argslen in sendSynchronousCommand(). 
Related to #5037.
 2018-06-26 14:38:35 +02:00
antirez 1f1e724f47 Remove black space. 2018-06-26 14:37:22 +02:00
Salvatore Sanfilippo aa5eaad48c Merge pull request #5037 from madolson/repl-auth-fix 
Fixed replication authentication with whitespace password
 2018-06-26 14:33:20 +02:00
antirez 3cf8dd2c84 Sentinel: fix SENTINEL SET error reporting. 
Thanks to @shenlongxing for reporting the problem.
Related to #5062.
 2018-06-26 09:17:38 +02:00
Madelyn Olson 45731edc4b Addressed comments 2018-06-26 00:57:35 +00:00
Madelyn Olson e8d68b6b72 Fixed replication authentication with whitespace in password 2018-06-26 00:48:37 +00:00
antirez fc0c9c8097 Sentinel: drop the renamed-command entry in a more natural way. 
Instead of telling the user to set the renamed command to "" to remove
the renaming, to the obvious thing when a command is renamed to itself.

So if I want to remove the renaming of PING, I just rename it to PING
again.
 2018-06-25 17:50:46 +02:00
antirez 8ba670f55c Sentinel command renaming: document it into sentinel.conf. 2018-06-25 17:45:01 +02:00
antirez a8a76bda1b Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-06-25 17:34:12 +02:00
antirez 2358de6816 Sentinel command renaming: use case sensitive hashing for the dict. 2018-06-25 17:31:57 +02:00
antirez a9c5008895 Sentinel command renaming: fix CONFIG SET event logging. 2018-06-25 17:24:04 +02:00
antirez b72cecd7c8 Sentinel command renaming: fix CONFIG SET after refactoring. 2018-06-25 17:23:32 +02:00
antirez 91a384a5cd Sentinel command renaming: implement SENTINEL SET. 2018-06-25 17:13:20 +02:00
antirez 903582dd7b Sentinel: make SENTINEL SET able to handle different arities. 2018-06-25 17:12:39 +02:00
antirez c303e768bf Sentinel command renaming: config rewriting. 2018-06-25 16:55:01 +02:00
antirez 60df7dbea1 Sentinel command renaming: rename-command option parsing. 2018-06-25 16:47:50 +02:00
antirez 72e8a33b35 Sentinel command renaming: base machanism implemented. 2018-06-25 14:06:05 +02:00
Guy Benoish dfcc20f4fd Fix compiler warning in restoreCommand 2018-06-24 16:53:01 +07:00
Salvatore Sanfilippo cf7fcdbe4e Merge pull request #4634 from soloestoy/special-auth 
config: handle special configuration "" for auth
 2018-06-22 15:48:33 +02:00
Salvatore Sanfilippo 70b7fa2c52 Merge pull request #5049 from youjiali1995/fix-load-rdb 
Fix rdbLoadIntegerObject() to create shared objects when needed.
 2018-06-22 15:47:45 +02:00
Salvatore Sanfilippo 54d66d39a0 Merge pull request #5053 from michael-grunder/zpopminmax-keypos 
Update ZPOPMIN/ZPOPMAX command declaration
 2018-06-22 15:35:36 +02:00
Salvatore Sanfilippo 199e704a42 Merge pull request #5050 from shenlongxing/fix-typo 
fix typo
 2018-06-22 15:34:36 +02:00
michael-grunder db6b99f90c Update ZPOPMIN/ZPOPMAX command declaration 
Unlike the BZPOP variants, these functions take a single key.  This fixes
an erroneous CROSSSLOT error when passing a count to a cluster enabled
server.
 2018-06-21 12:30:42 -07:00
Salvatore Sanfilippo a16aa03a67 Merge pull request #5051 from oranagra/streams_mem_estimate 
fix streams memory estimation, missing raxSeek
 2018-06-21 17:36:28 +02:00
Oran Agra 20e10dc7b8 fix streams memory estimation, missing raxSeek 2018-06-21 17:58:29 +03:00
shenlongxing ec55df117f fix typo 2018-06-21 22:08:09 +08:00
youjiali1995 df6644fe0e Fix rdbLoadIntegerObject() to create shared objects when needed. 2018-06-21 19:23:16 +08:00
Salvatore Sanfilippo 1527bcada3 Merge pull request #5036 from bepahol/unstable 
Update README.md
 2018-06-20 17:42:37 +02:00
Salvatore Sanfilippo c1e8240565 Merge pull request #5039 from oranagra/rdb_dbsize_hint 
64 bit RDB_OPCODE_RESIZEDB in rdb saving
 2018-06-20 11:18:31 +02:00
Salvatore Sanfilippo 79f55eeddd Merge pull request #5040 from oranagra/memrev64ifbe_fix 
use safe macro (non empty) in memrev64ifbe to eliminate empty if warning
 2018-06-20 11:17:06 +02:00
Salvatore Sanfilippo c6f4118ce6 Merge pull request #5045 from guybe7/restore_fix 
Enhance RESTORE with RDBv9 new features
 2018-06-20 11:11:39 +02:00
Guy Benoish b5197f1fc9 Enhance RESTORE with RDBv9 new features 
RESTORE now supports:
1. Setting LRU/LFU
2. Absolute-time TTL

Other related changes:
1. RDB loading will not override LRU bits when RDB file
   does not contain the LRU opcode.
2. RDB loading will not set LRU/LFU bits if the server's
   maxmemory-policy does not match.
 2018-06-20 15:11:08 +07:00
Salvatore Sanfilippo c6fdebf533 Merge pull request #5042 from oranagra/malloc_usable_size_libc 
add malloc_usable_size for libc malloc
 2018-06-19 17:22:36 +02:00
Oran Agra 482785ac62 add malloc_usable_size for libc malloc 
this reduces the extra 8 bytes we save before each pointer.
but more importantly maybe, it makes the valgrind runs to be more similiar
to our normal runs.

note: the change in malloc_stats struct in server.h is to eliminate an name conflict.
structs that are not typedefed are resolved from a separate name space.
 2018-06-19 18:18:23 +03:00
Salvatore Sanfilippo 4da296307c Merge pull request #5023 from FX-HAO/unstable 
Fix update_zmalloc_stat_alloc in zrealloc
 2018-06-19 16:50:22 +02:00
antirez e72190252e Test RDB stream encoding saving/loading. 2018-06-19 16:29:15 +02:00
Salvatore Sanfilippo 5f5e1199ef Merge pull request #5041 from oranagra/redis-rdb-check_rdbLoadMillisecondTime 
fix redis-rdb-check to provide proper arguments to rdbLoadMillisecondTime
 2018-06-19 16:06:11 +02:00
antirez 4848fbec8b Modules: convert hash to hash table for big objects. 2018-06-19 16:03:00 +02:00
Oran Agra f31b0405f0 fix redis-rdb-check to provide proper arguments to rdbLoadMillisecondTime 
due to incorrect forward declaration, it didn't provide all arguments.
this lead to random value being read from the stack and return of incorrect time,
which in this case doesn't matter since no one uses it.
 2018-06-19 16:54:22 +03:00
antirez 333c98c43a AOF: remove no longer used variable "now". 2018-06-19 15:54:05 +02:00
antirez e94b2053c6 Modify clusterRedirectClient() to handle ZPOP and XREAD. 2018-06-19 15:53:32 +02:00
Oran Agra 26229aa607 use safe macro (non empty) in memrev64ifbe to eliminate empty if warning 2018-06-19 16:46:41 +03:00
Oran Agra 5cd3c9529d 64 bit RDB_OPCODE_RESIZEDB in rdb saving 
this complication in the code is from times were rdbSaveLen didn't support 64 bits.
 2018-06-19 16:43:12 +03:00
antirez ba92b517b8 Remove AOF optimization to skip expired keys. 
Basically we cannot be sure that if the key is expired while writing the
AOF, the main thread will surely find the key expired. There are
possible race conditions like the moment at which the "now" is sampled,
and the fact that time may jump backward.

Think about the following:

SET a 5
EXPIRE a 1

AOF rewrite starts after about 1 second. The child process finds the key
expired, while in the main thread instead an INCR command is called
against the key "a" immediately after a fork, and the scheduler was
faster to give execution time to the main thread, so "a" is yet not
expired.

The main thread will generate an INCR a command to the AOF log that will
be appended to the rewritten AOF file, but that INCR command will target
a non existin "a" key, so a new non volatile key "a" will be created.

Two observations:

A) In theory by computing "now" before the fork, we should be sure that
if a key is expired at that time, it will be expired later when the
main thread will try to access to such key. However this does not take
into account the fact that the computer time may jump backward.

B) Technically we may still make the process safe by using a monotonic
time source.

However there were other similar related bugs, and in general the new
"vision" is that Redis persistence files should represent the memory
state without trying to be too smart: this makes the design more
consistent, bugs less likely to arise from complex interactions, and in
the end what is to fix is the Redis expire process to have less expired
keys in RAM.

Thanks to Oran Agra and Guy Benoish for writing me an email outlining
this problem, after they conducted a Redis 5 code review.
 2018-06-19 15:43:06 +02:00
Benjamin Holst 36524060df Update README.md 2018-06-18 22:41:26 -07:00
antirez 44571088d8 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-06-18 17:09:16 +02:00
antirez 6967d0bd5a Revert fix #4976 just leaving the flush() part. 2018-06-18 17:09:00 +02:00
antirez 0ed0dc3c02 Fix incrDecrCommand() to create shared objects when needed. 
See #5011.
 2018-06-18 16:56:31 +02:00
antirez bd92389c2d Refactor createObjectFromLongLong() to be suitable for value objects. 2018-06-18 16:55:16 +02:00
Salvatore Sanfilippo 3518bb66d7 Merge pull request #5020 from shenlongxing/fix-config 
Fix config_set_numerical_field() integer overflow.
 2018-06-18 16:02:23 +02:00
antirez 2076660843 Streams: fix xreadGetKeys() for correctness. 
The old version could not handle the fact that "STREAMS" is a valid key
name for streams. Now we really try to parse the command like the
command implementation would do.

Related to #5028 and 4857.
 2018-06-18 14:06:06 +02:00
Salvatore Sanfilippo e670ccffea Merge pull request #4857 from youjiali1995/fix-command-getkeys 
Fix core dump when using some commands with wrong arguments.
 2018-06-18 13:54:38 +02:00
antirez a0b27dae85 Streams: fix xreadGetKeys() buffer overflow. 
The loop allocated a buffer for the right number of keys positions, then
overflowed it going past the limit.

Related to #4857 and cause of the memory violation seen in #5028.
 2018-06-18 13:51:19 +02:00
antirez 62f9ac6f43 Streams: Change XADD MAXLEN handling of values <= 0. 
Now a MAXLEN of 0 really does what it means: it will create a zero
entries stream. This is useful in order to make sure that the behavior
is identical to XTRIM, that must be able to reduce the stream to zero
elements when MAXLEN is given.

Also now MAXLEN with a count < 0 will return an error.
 2018-06-18 10:05:21 +02:00
Max Vetrov d4c4f20a45 Update sort.c 
Redundant second if, and we may use "not" operation for bool, I suppose it's faster.
 2018-06-17 21:25:51 +02:00
antirez 79a1c19ac2 XADD MAXLEN should return an error for values < 0. 2018-06-17 10:44:01 +02:00
Salvatore Sanfilippo 2e0ab4a807 Merge pull request #4976 from trevor211/fixDebugLoadaof 
Critical: Fix server crash and data inconsistency in some cases.
 2018-06-16 11:05:04 +02:00
Salvatore Sanfilippo 94658303e9 Merge pull request #4758 from soloestoy/rdb-save-incremental-fsync 
Rdb save incremental fsync
 2018-06-16 10:59:37 +02:00
antirez 6a66b93b18 Sentinel: add an option to deny online script reconfiguration. 
The ability of "SENTINEL SET" to change the reconfiguration script at
runtime is a problem even in the security model of Redis: any client
inside the network may set any executable to be ran once a failover is
triggered.

This option adds protection for this problem: by default the two
SENTINEL SET subcommands modifying scripts paths are denied. However the
user is still able to rever that using the Sentinel configuration file
in order to allow such a feature.
 2018-06-14 18:57:58 +02:00
antirez d353023a83 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-06-14 18:11:04 +02:00
antirez d6e8fe77af Fix infinite loop in dbRandomKey(). 
Thanks to @kevinmcgehee for signaling the issue and reasoning about the
consequences and potential fixes.

Issue #5015.
 2018-06-14 18:08:21 +02:00
Salvatore Sanfilippo 40d5df6547 Merge pull request #5008 from zwkno1/unstable 
fix geohashEncode bug when step > 30
 2018-06-14 14:01:40 +02:00
Salvatore Sanfilippo 8bc3ffcb29 Merge pull request #5021 from soloestoy/fix-exists 
fix exists command on slave
 2018-06-14 13:58:30 +02:00
Salvatore Sanfilippo 6c7847a183 Merge pull request #5018 from soloestoy/optimize-reply 
optimize reply list memory usage
 2018-06-14 13:32:00 +02:00
antirez 1e92fde3d3 Fix SCAN bug regression test, avoiding empty SREM call. 2018-06-14 12:21:58 +02:00
Fuxin Hao a4f658b2b5 Fix update_zmalloc_stat_alloc in zrealloc 2018-06-14 16:44:19 +08:00
antirez 48dfd42d72 Redis 5.0 RC3. 2018-06-14 09:52:48 +02:00
antirez b4b0251ca6 Rax library updated. 2018-06-14 09:50:46 +02:00
zhaozhao.zz 2ffa533f85 fix exists command on slave 2018-06-14 01:30:07 +08:00
shenlongxing e4e5a670e7 Fix config_set_numerical_field() integer overflow. 2018-06-13 23:27:53 +08:00
zhaozhao.zz 963002d71e optimize reply list memory usage 2018-06-13 20:35:40 +08:00
antirez f7209749a6 Redis 5.0 RC2. 2018-06-13 13:26:28 +02:00
antirez 5171a3ff8e RDB: Apply fix to rdbLoadMillisecondTime() only for new RDB versions. 
This way we let big endian systems to still load old RDB versions.
However newver versions will be saved and loaded in a way that make RDB
expires cross-endian again. Thanks to @oranagra for the reporting and
the discussion about this problem, leading to this fix.
 2018-06-13 13:26:13 +02:00
antirez af7069627e Streams: generate a few additional events. 
Currently it does not look it's sensible to generate events for streams
consumer groups modification, being them metadata, however at least for
key-level events, like the creation or removal of a consumer group, I
added a few events here and there. Later we can evaluate if it makes
sense to add more. From the POV instead of WAIT (in Redis transaciton)
and signaling the key as modified, it looks like that the transaction
should not fail when a stream is modified, so no calls are made in
consumer groups related functions to signalModifiedKey().
 2018-06-13 13:26:12 +02:00
antirez 3ad68a07ff Fix rdbSaveKeyValuePair() integer overflow. 
Again thanks to @oranagra. The object idle time does not fit into an int
sometimes: use the native type that the serialization function will get
as argument, which is uint64_t.
 2018-06-13 13:26:12 +02:00
antirez 68382091a0 In scanDatabaseForReadyLists() now we need to handle ZSETs as well. 
Since the introduction of ZPOP makes this needed. Thanks to @oranagra
for reporting.
 2018-06-13 13:26:12 +02:00
antirez 6b0cdbd903 RDB: store times consistently in little endian. 
I'm not sure how this escaped the attention of Redis users for years,
but finally @oranagra reported this issue... Thanks to Oran.
 2018-06-13 13:26:12 +02:00
antirez 574017b7d2 Streams: improve type correctness in t_stream.c. 2018-06-13 13:26:12 +02:00
antirez cd2b5a79ff Fix XGROUP help missing space. 2018-06-13 13:26:12 +02:00
Baoyi Chen 8246a38e83 fix typo 
fix [#5005](https://github.com/antirez/redis/issues/5005)
 2018-06-13 13:26:12 +02:00
antirez b80e4b6925 Security: fix redis-cli buffer overflow. 
Thanks to Fakhri Zulkifli for reporting it.

The fix switched to dynamic allocation, copying the final prompt in the
static buffer only at the end.
 2018-06-13 12:40:43 +02:00
antirez 5e0d9841d4 Security: fix Lua struct package offset handling. 
After the first fix to the struct package I found another similar
problem, which is fixed by this patch. It could be reproduced easily by
running the following script:

    return struct.unpack('f', "xxxxxxxxxxxxx",-3)

The above will access bytes before the 'data' pointer.
 2018-06-13 12:40:43 +02:00
antirez e27a040146 Security: more cmsgpack fixes by @soloestoy. 
@soloestoy sent me this additional fixes, after searching for similar
problems to the one reported in mp_pack(). I'm committing the changes
because it was not possible during to make a public PR to protect Redis
users and give Redis providers some time to patch their systems.
 2018-06-13 12:40:43 +02:00
antirez c5dfff465e Security: update Lua struct package for security. 
During an auditing Apple found that the "struct" Lua package
we ship with Redis (http://www.inf.puc-rio.br/~roberto/struct/) contains
a security problem. A bound-checking statement fails because of integer
overflow. The bug exists since we initially integrated this package with
Lua, when scripting was introduced, so every version of Redis with
EVAL/EVALSHA capabilities exposed is affected.

Instead of just fixing the bug, the library was updated to the latest
version shipped by the author.
 2018-06-13 12:40:43 +02:00
antirez 86aade7475 Security: fix Lua cmsgpack library stack overflow. 
During an auditing effort, the Apple Vulnerability Research team discovered
a critical Redis security issue affecting the Lua scripting part of Redis.

-- Description of the problem

Several years ago I merged a pull request including many small changes at
the Lua MsgPack library (that originally I authored myself). The Pull
Request entered Redis in commit 90b6337c1, in 2014.
Unfortunately one of the changes included a variadic Lua function that
lacked the check for the available Lua C stack. As a result, calling the
"pack" MsgPack library function with a large number of arguments, results
into pushing into the Lua C stack a number of new values proportional to
the number of arguments the function was called with. The pushed values,
moreover, are controlled by untrusted user input.

This in turn causes stack smashing which we believe to be exploitable,
while not very deterministic, but it is likely that an exploit could be
created targeting specific versions of Redis executables. However at its
minimum the issue results in a DoS, crashing the Redis server.

-- Versions affected

Versions greater or equal to Redis 2.8.18 are affected.

-- Reproducing

Reproduce with this (based on the original reproduction script by
Apple security team):

https://gist.github.com/antirez/82445fcbea6d9b19f97014cc6cc79f8a

-- Verification of the fix

The fix was tested in the following way:

1) I checked that the problem is no longer observable running the trigger.
2) The Lua code was analyzed to understand the stack semantics, and that
actually enough stack is allocated in all the cases of mp_pack() calls.
3) The mp_pack() function was modified in order to show exactly what items
in the stack were being set, to make sure that there is no silent overflow
even after the fix.

-- Credits

Thank you to the Apple team and to the other persons that helped me
checking the patch and coordinating this communication.
 2018-06-13 12:40:43 +02:00
antirez ce17f76bbd Security: fix redis-cli buffer overflow. 
Thanks to Fakhri Zulkifli for reporting it.

The fix switched to dynamic allocation, copying the final prompt in the
static buffer only at the end.
 2018-06-13 12:40:33 +02:00
antirez e89086e09a Security: fix Lua struct package offset handling. 
After the first fix to the struct package I found another similar
problem, which is fixed by this patch. It could be reproduced easily by
running the following script:

    return struct.unpack('f', "xxxxxxxxxxxxx",-3)

The above will access bytes before the 'data' pointer.
 2018-06-13 12:40:33 +02:00
antirez 5ccb6f7a79 Security: more cmsgpack fixes by @soloestoy. 
@soloestoy sent me this additional fixes, after searching for similar
problems to the one reported in mp_pack(). I'm committing the changes
because it was not possible during to make a public PR to protect Redis
users and give Redis providers some time to patch their systems.
 2018-06-13 12:40:33 +02:00
antirez 1eb08bcd46 Security: update Lua struct package for security. 
During an auditing Apple found that the "struct" Lua package
we ship with Redis (http://www.inf.puc-rio.br/~roberto/struct/) contains
a security problem. A bound-checking statement fails because of integer
overflow. The bug exists since we initially integrated this package with
Lua, when scripting was introduced, so every version of Redis with
EVAL/EVALSHA capabilities exposed is affected.

Instead of just fixing the bug, the library was updated to the latest
version shipped by the author.
 2018-06-13 12:40:33 +02:00
antirez 52a00201fc Security: fix Lua cmsgpack library stack overflow. 
During an auditing effort, the Apple Vulnerability Research team discovered
a critical Redis security issue affecting the Lua scripting part of Redis.

-- Description of the problem

Several years ago I merged a pull request including many small changes at
the Lua MsgPack library (that originally I authored myself). The Pull
Request entered Redis in commit 90b6337c1, in 2014.
Unfortunately one of the changes included a variadic Lua function that
lacked the check for the available Lua C stack. As a result, calling the
"pack" MsgPack library function with a large number of arguments, results
into pushing into the Lua C stack a number of new values proportional to
the number of arguments the function was called with. The pushed values,
moreover, are controlled by untrusted user input.

This in turn causes stack smashing which we believe to be exploitable,
while not very deterministic, but it is likely that an exploit could be
created targeting specific versions of Redis executables. However at its
minimum the issue results in a DoS, crashing the Redis server.

-- Versions affected

Versions greater or equal to Redis 2.8.18 are affected.

-- Reproducing

Reproduce with this (based on the original reproduction script by
Apple security team):

https://gist.github.com/antirez/82445fcbea6d9b19f97014cc6cc79f8a

-- Verification of the fix

The fix was tested in the following way:

1) I checked that the problem is no longer observable running the trigger.
2) The Lua code was analyzed to understand the stack semantics, and that
actually enough stack is allocated in all the cases of mp_pack() calls.
3) The mp_pack() function was modified in order to show exactly what items
in the stack were being set, to make sure that there is no silent overflow
even after the fix.

-- Credits

Thank you to the Apple team and to the other persons that helped me
checking the patch and coordinating this communication.
 2018-06-13 12:40:33 +02:00
antirez 032ea657d7 RDB: Apply fix to rdbLoadMillisecondTime() only for new RDB versions. 
This way we let big endian systems to still load old RDB versions.
However newver versions will be saved and loaded in a way that make RDB
expires cross-endian again. Thanks to @oranagra for the reporting and
the discussion about this problem, leading to this fix.
 2018-06-12 18:21:39 +02:00
antirez 6b8a24a665 Streams: generate a few additional events. 
Currently it does not look it's sensible to generate events for streams
consumer groups modification, being them metadata, however at least for
key-level events, like the creation or removal of a consumer group, I
added a few events here and there. Later we can evaluate if it makes
sense to add more. From the POV instead of WAIT (in Redis transaciton)
and signaling the key as modified, it looks like that the transaction
should not fail when a stream is modified, so no calls are made in
consumer groups related functions to signalModifiedKey().
 2018-06-12 18:11:15 +02:00
antirez b38682199b Fix rdbSaveKeyValuePair() integer overflow. 
Again thanks to @oranagra. The object idle time does not fit into an int
sometimes: use the native type that the serialization function will get
as argument, which is uint64_t.
 2018-06-12 17:31:04 +02:00
antirez e534e9aa83 In scanDatabaseForReadyLists() now we need to handle ZSETs as well. 
Since the introduction of ZPOP makes this needed. Thanks to @oranagra
for reporting.
 2018-06-12 17:28:40 +02:00
antirez f70e88c1f6 RDB: store times consistently in little endian. 
I'm not sure how this escaped the attention of Redis users for years,
but finally @oranagra reported this issue... Thanks to Oran.
 2018-06-12 17:22:03 +02:00
antirez 4774d61691 Streams: improve type correctness in t_stream.c. 2018-06-12 14:12:53 +02:00
Itamar Haber 6b675b9525 Applies addReplySubSyntaxError to stream commands 2018-06-12 14:52:50 +03:00
antirez bcc42028c1 Fix XGROUP help missing space. 2018-06-12 13:20:46 +02:00
antirez 071e235a52 Regression test for issue #5006. 2018-06-12 13:15:53 +02:00
Shen Longxing 6acd8a31a4 fix active-defrag-threshold value error 
The active-defrag-threshold-lower/active-defrag-threshold-upper min/max  value in redis.conf should be consistent with 'config set' command.
 2018-06-12 13:15:53 +02:00
antirez 20bc3786c0 Streams: fix backward iteration when entry is not flagged SAMEFIELD. 
See issue #5006. The comment in the code was also wrong and
was rectified as well.
 2018-06-12 13:15:53 +02:00
antirez 2cadef46f7 Streams: increment dirty counter for XGROUP SETID/DESTROY. 
See issue #5005 comments.
 2018-06-12 13:15:53 +02:00
antirez 093ec57d06 Use a less aggressive query buffer resize policy. 
A user with many connections (10 thousand) on a single Redis server
reports in issue #4983 that sometimes Redis is idle becuase at the same
time many clients need to resize their query buffer according to the old
policy.

It looks like this was created by the fact that we allow the query
buffer to grow without problems to a size up to PROTO_MBULK_BIG_ARG
normally, but when the client is idle we immediately are more strict,
and a query buffer greater than 1024 bytes is already enough to trigger
the resize. So for instance if most of the clients stop at the same time
this issue should be easily triggered.

This behavior actually looks odd, and there should be only a clear limit
after we say, let's look at this query buffer to check if it's time to
resize it. This commit puts the limit at PROTO_MBULK_BIG_ARG, and the
check is performed both if compared to the peak usage the current usage
is too big, or if the client is idle.

Then when the check is performed, to waste just a few kbytes is
considered enough to proceed with the resize. This should fix the issue.
 2018-06-12 13:15:52 +02:00
antirez d769ce0a20 Fix client unblocking for XREADGROUP, issue #4978. 
We unblocked the client too early, when the group name object was no
longer valid in client->bpop, so propagating XCLAIM later in
streamPropagateXCLAIM() deferenced a field already set to NULL.
 2018-06-12 13:15:52 +02:00
antirez 280b2dc1e4 Improved regression test for #4906. 
Removing the fix about 50% of the times the test will not be able to
pass cleanly. It's very hard to write a test that will always fail, or
actually, it is possible but then it's likely that it will consistently
pass if we change some random bit, so better to use randomization here.
 2018-06-12 13:15:52 +02:00
antirez 260b32edff Regression test for the dictScan() issue #4906. 2018-06-12 13:15:52 +02:00
zhaozhao.zz 28e0ca5a8c Streams: checkType for xread & xinfo 2018-06-12 13:15:52 +02:00
zhaozhao.zz 9d2b1f294d Streams: lookupKey[Read->Write]OrReply in xdel and xtrim 2018-06-12 13:15:52 +02:00
michael-grunder 3d4ad250cd Abort in XGROUP if the key is not a stream 2018-06-12 13:15:52 +02:00
shenlongxing 5a3d490c50 fix integer case error 2018-06-12 13:15:52 +02:00
antirez 24fb4e3894 Implement DEBUG htstats-key. 2018-06-12 13:15:52 +02:00
antirez a67ecb91f9 redis-cli inline help updated. 2018-06-12 13:15:52 +02:00
antirez 5cdd475066 Add the stream group to the script generating the help. 2018-06-12 13:15:52 +02:00
shenlongxing 2cd6b001da fix stream config typo 2018-06-12 13:15:52 +02:00
antirez b2a5a70e5a Streams: better document the max node limits. 2018-06-12 13:15:13 +02:00
antirez eb26a8174a Typo: entires -> entries in several places. 2018-06-12 13:15:13 +02:00
antirez f2d65c380d Streams: make macro node limits configurable. 2018-06-12 13:15:13 +02:00
antirez b16e5432b6 Streams: max node limits only checked if non zero. 2018-06-12 13:15:13 +02:00
antirez d01af7abe4 Streams: use non static macro node limits. 
Also add the concept of size/items limit, instead of just having as
limit the number of bytes.
 2018-06-12 13:15:13 +02:00
shenlongxing 77acc63eef Fix write() errno error 2018-06-12 13:15:13 +02:00
michael-grunder 3ddc537605 Return early in XPENDING if sent a nonexistent consumer group. 2018-06-12 13:15:13 +02:00
Krzysztof Filipek b85086c573 Typo in preprocessor condition 2018-06-12 13:15:13 +02:00
zhaozhao.zz 4a9670f517 RDB: expand dict if needed when rdb load object 2018-06-12 13:15:13 +02:00
zhaozhao.zz d5712d2151 adjust position of _dictNextPower in dictExpand 2018-06-12 13:15:13 +02:00
zhaozhao.zz 3ca02fb939 zset: change the span of zskiplistNode to unsigned long 2018-06-12 13:15:13 +02:00
zhaozhao.zz 59e51f92d2 zset: fix the int problem 2018-06-12 13:15:13 +02:00
antirez e916f4519c Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-06-12 13:14:01 +02:00
antirez 6bf65138de Regression test for issue #5006. 2018-06-12 13:13:35 +02:00
Salvatore Sanfilippo bcecbebb74 Merge pull request #5012 from shenlongxing/fix-config 
fix active-defrag-threshold value error
 2018-06-12 13:05:19 +02:00
Shen Longxing 13957c9d07 fix active-defrag-threshold value error 
The active-defrag-threshold-lower/active-defrag-threshold-upper min/max  value in redis.conf should be consistent with 'config set' command.
 2018-06-12 17:59:32 +08:00
antirez 7cc1312789 Streams: fix backward iteration when entry is not flagged SAMEFIELD. 
See issue #5006. The comment in the code was also wrong and
was rectified as well.
 2018-06-12 10:22:49 +02:00
球状闪电 7659619824 Update geohash.c 
fix geohasEncode bug when step > 31
 2018-06-12 15:28:28 +08:00
Salvatore Sanfilippo 82661ba329 Merge pull request #5007 from leonchen83/patch-2 
fix typo issue #5005
 2018-06-12 09:28:26 +02:00
antirez 923e63e5ec Streams: increment dirty counter for XGROUP SETID/DESTROY. 
See issue #5005 comments.
 2018-06-12 09:27:40 +02:00
Baoyi Chen fac3e8aab5 fix typo 
fix [#5005](https://github.com/antirez/redis/issues/5005)
 2018-06-12 08:52:18 +08:00
antirez cec404f099 Use a less aggressive query buffer resize policy. 
A user with many connections (10 thousand) on a single Redis server
reports in issue #4983 that sometimes Redis is idle becuase at the same
time many clients need to resize their query buffer according to the old
policy.

It looks like this was created by the fact that we allow the query
buffer to grow without problems to a size up to PROTO_MBULK_BIG_ARG
normally, but when the client is idle we immediately are more strict,
and a query buffer greater than 1024 bytes is already enough to trigger
the resize. So for instance if most of the clients stop at the same time
this issue should be easily triggered.

This behavior actually looks odd, and there should be only a clear limit
after we say, let's look at this query buffer to check if it's time to
resize it. This commit puts the limit at PROTO_MBULK_BIG_ARG, and the
check is performed both if compared to the peak usage the current usage
is too big, or if the client is idle.

Then when the check is performed, to waste just a few kbytes is
considered enough to proceed with the resize. This should fix the issue.
 2018-06-11 17:12:28 +02:00
antirez 5bbb00fbb4 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-06-11 16:52:45 +02:00
antirez 34bd44187a Fix client unblocking for XREADGROUP, issue #4978. 
We unblocked the client too early, when the group name object was no
longer valid in client->bpop, so propagating XCLAIM later in
streamPropagateXCLAIM() deferenced a field already set to NULL.
 2018-06-11 16:51:06 +02:00
Salvatore Sanfilippo ba2101738d Merge pull request #5000 from shenlongxing/fix-config 
fix integer case error
 2018-06-11 16:34:26 +02:00
Salvatore Sanfilippo b136502d0f Merge pull request #4995 from shenlongxing/unstable 
fix stream config typo
 2018-06-11 16:33:38 +02:00
Salvatore Sanfilippo 5db262b623 Merge pull request #5002 from soloestoy/streams-read-or-write 
Streams: lookupKey[Read->Write]OrReply in xdel and xtrim
 2018-06-11 16:33:10 +02:00
Salvatore Sanfilippo e2a9ea0405 Merge pull request #4901 from KFilipek/zmalloc_typo_fix 
HW_PHYSMEM typo in preprocessor condition
 2018-06-11 16:32:40 +02:00
Salvatore Sanfilippo 295db9ee0a Merge pull request #5003 from soloestoy/streams-checkType 
Streams: checkType for xread & xinfo
 2018-06-11 16:32:00 +02:00
Salvatore Sanfilippo 163f32c22a Merge pull request #5001 from michael-grunder/xgroup-check-type 
Abort in XGROUP if the key is not a stream
 2018-06-11 16:30:49 +02:00
antirez 00613bed06 Improved regression test for #4906. 
Removing the fix about 50% of the times the test will not be able to
pass cleanly. It's very hard to write a test that will always fail, or
actually, it is possible but then it's likely that it will consistently
pass if we change some random bit, so better to use randomization here.
 2018-06-11 13:10:06 +02:00
antirez 27beaf2f22 Regression test for the dictScan() issue #4906. 2018-06-11 12:51:26 +02:00
zhaozhao.zz b8913ce721 Streams: checkType for xread & xinfo 2018-06-11 01:17:52 +08:00
zhaozhao.zz cc81c6f1d3 Streams: lookupKey[Read->Write]OrReply in xdel and xtrim 2018-06-11 00:14:17 +08:00
michael-grunder c04082cf13 Abort in XGROUP if the key is not a stream 2018-06-10 08:55:58 -07:00
Itamar Haber e654b68d1f Merge branch 'unstable' into module_command_help 2018-06-09 21:10:53 +03:00
Itamar Haber fefde6e3e4 Capitalizes subcommands & orders lexicographically 2018-06-09 21:03:52 +03:00
Itamar Haber 21ef0376fe Capitalizes subscommands 2018-06-09 20:54:05 +03:00
shenlongxing 1210dd8a20 fix integer case error 2018-06-09 22:24:33 +08:00
Salvatore Sanfilippo be899b824e Merge pull request #4519 from soloestoy/zset-int-problem 
Zset int problem
 2018-06-08 12:45:11 +02:00
Salvatore Sanfilippo 5ebaadc96b Merge pull request #4800 from soloestoy/dict-expand 
adjust position of _dictNextPower in dictExpand
 2018-06-08 12:32:18 +02:00
Salvatore Sanfilippo 9fa2e7020e Merge pull request #4861 from soloestoy/rdb-dict-expand 
RDB: expand dict if needed when rdb load object
 2018-06-08 12:12:34 +02:00
antirez 269e80526f Implement DEBUG htstats-key. 2018-06-08 11:17:20 +02:00
antirez 97e1f6812c Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-06-08 11:02:30 +02:00
antirez 2268d7e5dd redis-cli inline help updated. 2018-06-07 18:53:00 +02:00
antirez b2fc2eaecb Add the stream group to the script generating the help. 2018-06-07 18:52:01 +02:00
Salvatore Sanfilippo 66c6218a3c Merge pull request #4977 from michael-grunder/xpending-early-return 
Return early in XPENDING if sent a nonexistent consumer group.
 2018-06-07 18:49:09 +02:00
Itamar Haber c199280edb Globally applies addReplySubSyntaxError 2018-06-07 18:39:36 +03:00
Itamar Haber 76ad23d012 Adds MODULE HELP and implements addReplySubSyntaxError 2018-06-07 18:34:58 +03:00
shenlongxing 018cf118d6 fix stream config typo 2018-06-07 23:13:36 +08:00
antirez f847dd3ad8 Streams: better document the max node limits. 2018-06-07 14:37:56 +02:00
antirez a7dbe37d53 Typo: entires -> entries in several places. 2018-06-07 14:36:56 +02:00
antirez dfb3bf1b1e Streams: make macro node limits configurable. 2018-06-07 14:36:13 +02:00
antirez 6d34ff3645 Streams: max node limits only checked if non zero. 2018-06-07 14:30:25 +02:00
antirez 19a438e2c0 Streams: use non static macro node limits. 
Also add the concept of size/items limit, instead of just having as
limit the number of bytes.
 2018-06-07 14:24:49 +02:00
shenlongxing c85ae56edc Fix write() errno error 2018-06-06 13:06:42 +02:00
antirez 3502d1f16b Fix streamIteratorRemoveEntry() to update elements count. 
Close #4989.
 2018-06-06 11:41:06 +02:00
antirez b61416bdf4 Fix streamIteratorRemoveEntry() to update elements count. 
Close #4989.
 2018-06-06 11:40:22 +02:00
antirez 1317bab0d1 ZPOP: invert score-ele to match ZRANGE WITHSCORES order. 2018-06-05 17:06:46 +02:00
antirez 63ed0f7593 Remove XINFO <key> special form. 
As observed by Michael Grunder this usage while practical is
inconsistent because for instance it does not work against a key called
HELP. Removed.
 2018-06-05 17:06:46 +02:00
antirez 0a698e499a ZPOP: invert score-ele to match ZRANGE WITHSCORES order. 2018-06-05 17:06:25 +02:00
antirez 9e25f3e1de Remove XINFO <key> special form. 
As observed by Michael Grunder this usage while practical is
inconsistent because for instance it does not work against a key called
HELP. Removed.
 2018-06-05 16:34:34 +02:00
antirez 8b9b02ade5 XGROUP SETID implemented + consumer groups core fixes. 
Now that we have SETID, the inetrnals of consumer groups should be able
to handle the case of the same message delivered multiple times just
as a side effect of calling XREADGROUP. Normally this should never
happen but if the admin manually "XGROUP SETID mykey mygroup 0",
messages will get re-delivered to clients waiting for the ">" special
ID. The consumer groups internals were not able to handle the case of a
message re-delivered in this circumstances that was already assigned to
another owner.
 2018-06-04 17:30:34 +02:00
Yossi Gottlieb d4ed462b39 Clean gcc 7.x warnings, redis-cli cluster fix. 2018-06-04 17:30:34 +02:00
antirez 2f123ea6a3 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-06-04 17:30:11 +02:00
antirez 36b392a0b2 XGROUP SETID implemented + consumer groups core fixes. 
Now that we have SETID, the inetrnals of consumer groups should be able
to handle the case of the same message delivered multiple times just
as a side effect of calling XREADGROUP. Normally this should never
happen but if the admin manually "XGROUP SETID mykey mygroup 0",
messages will get re-delivered to clients waiting for the ">" special
ID. The consumer groups internals were not able to handle the case of a
message re-delivered in this circumstances that was already assigned to
another owner.
 2018-06-04 17:28:03 +02:00
antirez 05a2996641 Rax library updated. 2018-06-04 17:26:16 +02:00
Salvatore Sanfilippo b1783d3ab9 Merge pull request #4981 from yossigo/gcc7 
Clean gcc 7.x warnings, redis-cli cluster fix.
 2018-06-04 17:18:39 +02:00
antirez 525a7b399b XGROUP DESTROY implemented. 2018-06-04 12:58:41 +02:00
antirez 7c6f1be5df XGROUP DESTROY implemented. 2018-06-04 12:58:26 +02:00
赵磊 10dedc2586 Fix core dump when using 'command getkeys' with wrong arguments. 2018-06-04 15:14:50 +08:00
Yossi Gottlieb e1222d8b10 Clean gcc 7.x warnings, redis-cli cluster fix. 2018-06-03 15:54:30 +03:00
michael-grunder 1749fe7a26 Return early in XPENDING if sent a nonexistent consumer group. 2018-06-02 18:22:20 -07:00
WuYunlong 7d269d5e63 Fix DEBUG LOADAOF so that redis-server will not crash unexpectedly 
and will not be inconsistent after we call debug loadaof.
Before this commit, there were 2 problems:

1, When appendonly is set to no and there is not a appendonly file,
   redis-server will crash if we call DEBUG LOADAOF.
2, When appendonly is set to no and there is a appendonly file,
   redis-server will hold different data after loading  appendonly
   file.
 2018-06-02 14:32:16 +08:00
赵磊 ae9687de6f Fix dictScan(): It can't scan all buckets when dict is shrinking. 2018-06-01 16:54:28 +02:00
Salvatore Sanfilippo 86de089a1e Merge pull request #4907 from youjiali1995/fix-dictScan 
Fix dictScan(): It can't scan all buckets when dict is shrinking.
 2018-06-01 16:54:05 +02:00
artix 9bb7c4b2e5 Cluster Manager: fixed master_id check in clusterManagerNodeLoadInfo 2018-06-01 16:20:08 +02:00
zhaozhao.zz 07142f034a ZPOP: unblock multiple clients in right way 2018-06-01 16:20:08 +02:00
zhaozhao.zz 733348bbb4 MEMORY: fix the missing of monitor clients buffers 2018-06-01 16:20:08 +02:00
Mota 4223c41587 Fix debug crash-and-recover help info. 2018-06-01 16:20:08 +02:00
artix d168b245b3 Cluster Manager: fixed master_id check in clusterManagerNodeLoadInfo 2018-06-01 16:19:47 +02:00
Salvatore Sanfilippo d49d023828 Merge pull request #4971 from soloestoy/bugfix-bzpop 
ZPOP: unblock multiple clients in right way
 2018-05-31 18:40:53 +02:00
zhaozhao.zz b9d19371e4 ZPOP: unblock multiple clients in right way 2018-05-31 23:35:47 +08:00
Salvatore Sanfilippo 2dee9a4118 Merge pull request #4945 from GitHubMota/debug_recover 
Fix debug crash-and-recover help info.
 2018-05-31 17:32:11 +02:00
Salvatore Sanfilippo 5bf08449f9 Merge pull request #4953 from soloestoy/fix-memory-overhead 
MEMORY: fix the missing of monitor clients buffers
 2018-05-31 17:31:24 +02:00
antirez ad4e5ca925 Capitalize OBJECT HELP subcommands. 2018-05-31 17:21:20 +02:00
Remi Collet d55c5e8caf include stdint.h for unit64_t definition 2018-05-31 17:21:20 +02:00
Salvatore Sanfilippo f3b48978f2 Merge pull request #4964 from remicollet/issue-uint64 
include stdint.h for unit64_t definition
 2018-05-31 17:20:33 +02:00
antirez 2f963080c1 Capitalize OBJECT HELP subcommands. 2018-05-31 17:11:46 +02:00
artix 4bce88b11f Cluster Manager: fixed infinite loop in rebalance (Issue #4941). 2018-05-31 15:59:16 +02:00
artix d2aa10d029 Cluster Manager: fixed infinite loop in rebalance (Issue #4941). 2018-05-31 15:58:55 +02:00
Remi Collet 9561fec496 include stdint.h for unit64_t definition 2018-05-30 15:33:06 +02:00
antirez 2ee4a1c980 Redis 5.0 RC1. 2018-05-29 14:19:01 +02:00
antirez 3d5e2c62e2 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-05-29 12:41:07 +02:00
antirez 49147f36e9 Don't expire keys while loading RDB from AOF preamble. 
The AOF tail of a combined RDB+AOF is based on the premise of applying
the AOF commands to the exact state that there was in the server while
the RDB was persisted. By expiring keys while loading the RDB file, we
change the state, so applying the AOF tail later may change the state.

Test case:

* Time1: SET a 10
* Time2: EXPIREAT a $time5
* Time3: INCR a
* Time4: PERSIT A. Start bgrewiteaof with RDB preamble. The value of a is 11 without expire time.
* Time5: Restart redis from the RDB+AOF: consistency violation.

Thanks to @soloestoy for providing the patch.
Thanks to @trevor211 for the original issue report and the initial fix.

Check issue #4950 for more info.
 2018-05-29 12:37:42 +02:00
WuYunlong 2a887bd53f Fix rdb save by allowing dumping of expire keys, so that when 
we add a new slave, and do a failover, eighter by manual or
not, other local slaves will delete the expired keys properly.
 2018-05-29 12:35:15 +02:00
zhaozhao.zz ce2e0c02fc MEMORY: fix the missing of monitor clients buffers 2018-05-27 16:34:58 +08:00
antirez 6536ce27a4 Streams: fix comments referring to old ID format. 2018-05-25 17:25:19 +02:00
Salvatore Sanfilippo f8fd6639e7 Merge pull request #4948 from soloestoy/avoid-warning-stream 
avoid warning in t_stream.c
 2018-05-25 17:00:46 +02:00
antirez 2bd36616ba Streams: fix test ID format. 2018-05-25 16:57:08 +02:00
zhaozhao.zz c2d7e6cea7 avoid warning in t_stream.c 2018-05-25 22:49:49 +08:00
antirez fb6ebaa598 Backport hiredis issue 525 fix to compile on FreeBSD. 
Close #4947.
 2018-05-25 16:33:23 +02:00
antirez fb9c1c3dbd Simplify XADD ID syntax error code path,. 2018-05-25 16:29:37 +02:00
Salvatore Sanfilippo a60f553b68 Merge pull request #4832 from 0xtonyxia/fix-xadd-ID-parse 
Bugfix: xadd command ID parse
 2018-05-25 16:27:21 +02:00
antirez 95fa564add Avoid warning by explicitly initializing var to NULL. 2018-05-25 16:19:14 +02:00
antirez b2b4e186a2 Explicitly cast void pointers to char pointers for printf style logging. 2018-05-25 16:15:59 +02:00
Salvatore Sanfilippo 42ec8e8cb2 Merge pull request #4946 from soloestoy/linux-make-lrt 
append -lrt to FINAL_LIBS for linux
 2018-05-25 16:12:19 +02:00
zhaozhao.zz 2711f053f2 append -lrt to FINAL_LIBS for linux 2018-05-25 20:16:57 +08:00
antirez e4f5c04930 Don't append libjemalloc.a otherwise it will be after -ldl. 2018-05-25 13:36:51 +02:00
Mota af3471a5dc Fix debug crash-and-recover help info. 2018-05-25 16:49:23 +08:00
antirez 694145bd74 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-05-24 18:21:23 +02:00
antirez 4e729fcdaf Generate configure for Jemalloc. 2018-05-24 18:21:13 +02:00
antirez 98d5d3f118 Make active defragmentation tests optional. 
They failed when active defrag could not be activated because the
Jemalloc version does not include the additional APIs.
 2018-05-24 18:04:21 +02:00
antirez fef42d0985 Add --with-version in Jemalloc config script. 2018-05-24 17:50:12 +02:00
antirez 8d76c6a878 Specify that Jemalloc configure requires version. 2018-05-24 17:49:36 +02:00
antirez 08e1c8e820 Jemalloc upgraded to version 5.0.1. 2018-05-24 17:17:37 +02:00
antirez 8f4e2075a7 Update documentation about how to upgrade Jemalloc. 2018-05-24 17:16:12 +02:00
Salvatore Sanfilippo b23a501135 Merge pull request #4099 from JunHe77/aarch64_stack_cflag 
Fixed stack trace generation on aarch64
 2018-05-24 16:16:12 +02:00
Salvatore Sanfilippo 18b7f71130 Merge pull request #4932 from 0xtonyxia/fix-redis-cli-set-prefer-memory-leak 
Fix redis-cli memory leak when sending set :preference command.
 2018-05-24 16:15:21 +02:00
Salvatore Sanfilippo ef7ccad180 Merge pull request #4922 from 0xtonyxia/enhance-cli-security 
Enhance security for redis-cli
 2018-05-24 16:14:42 +02:00
Salvatore Sanfilippo 8a200b04c3 Merge pull request #4831 from 0xtonyxia/fix-xadd-arity-check 
Bugfix: xadd comand <field,value> arity check
 2018-05-24 16:07:19 +02:00
antirez de4b6cb27b Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-05-23 17:14:59 +02:00
antirez 8631e64779 Sentinel: fix delay in detecting ODOWN. 
See issue #2819 for details. The gist is that when we want to send INFO
because we are over the time, we used to send only INFO commands, no
longer sending PING commands. However if a master fails exactly when we
are about to send an INFO command, the PING times will result zero
because the PONG reply was already received, and we'll fail to send more
PINGs, since we try only to send INFO commands: the failure detector
will delay until the connection is closed and re-opened for "long
timeout".

This commit changes the logic so that we can send the three kind of
messages regardless of the fact we sent another one already in the same
code path. It could happen that we go over the message limit for the
link by a few messages, but this is not significant. However now we'll
not introduce delays in sending commands just because there was
something else to send at the same time.
 2018-05-23 17:13:44 +02:00
Salvatore Sanfilippo 1ab3c82375 Merge pull request #4908 from soloestoy/aof-rdb-preamble-compatible-checksum-no 
AOF & RDB: be compatible with rdbchecksum no
 2018-05-23 17:11:00 +02:00
Salvatore Sanfilippo f1321349d3 Merge pull request #4935 from soloestoy/bugfix-zpop 
bugfix for ZPOP: fix the wrong keyc, should be 1
 2018-05-22 15:52:21 +02:00
zhaozhao.zz 4e7a160b9b ZPOP: fix the wrong keyc, should be 1 2018-05-22 21:31:22 +08:00
dejun.xdj 46e241ac8e Fix redis-cli memory leak when sending set preference command. 2018-05-21 12:19:37 +08:00
dejun.xdj 95b988b6c6 Check if the repeat value is positive in while loop of cliSendCommand(). 
In case that the incoming repeat parameter is negative and causes a
deadless loop.
 2018-05-21 12:06:48 +08:00
dejun.xdj cc7ffdfdf2 Change the type of repeat argument to long for function cliSendCommand. 
To be in consistent with the original definition.
 2018-05-21 12:04:53 +08:00
dejun.xdj b2762f1ff2 Fix negtive repeat command value issue. 
If command like "-1 set a b" is sent with redis-cli, it will cause a deadless loop. So some repeat value checking logic is added to avoid this.
 2018-05-19 22:50:40 +08:00
dejun.xdj c2e2314640 Detect and stop saving history for auth command with repeat option. 
Put the repeat option checking code a little forward to avoid repeat logic.
 2018-05-18 11:40:05 +08:00
dejun.xdj ef931ef93e Change the warning message a little bit to avoid trademark issuses. 2018-05-18 11:37:31 +08:00
antirez 3c968ff030 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-05-17 16:53:14 +02:00
antirez 3c43e984db Merge remote-tracking branch 'artix/trib2cli' into unstable 2018-05-17 16:52:31 +02:00
Salvatore Sanfilippo 13779c11f8 Merge pull request #4927 from oranagra/defrag_32bit_fix 
Active defrag fixes for 32bit builds
 2018-05-17 12:24:40 +02:00
Oran Agra ad133e1023 Active defrag fixes for 32bit builds 
problems fixed:
* failing to read fragmentation information from jemalloc
* overflow in jemalloc fragmentation hint to the defragger
* test suite not triggering eviction after population
 2018-05-17 09:52:00 +03:00
artix e47c751c74 Removed TODO in redis-cli 2018-05-16 18:04:13 +02:00
artix 1e4fb1b33a Cluster Manager: fixed unprinted reply error 2018-05-16 17:49:18 +02:00
dejun.xdj b263c7c465 Stop saving auth command in redis-cli history. 2018-05-16 16:18:00 +08:00
dejun.xdj c082221aef Add warning message when using password on command line 2018-05-16 16:15:12 +08:00
artix 2f9c032a13 Cluster Manager: print flags as strings. 2018-05-15 18:41:46 +02:00
antirez e344aa4a6d Test: fix blocking lists/zsets replication test. 
By verifying that it was able to find a regression, and fixing it
accordingly.
 2018-05-15 17:43:41 +02:00
antirez 8327ccef0e Test: replication test blocking lists/zsets ops. 2018-05-15 17:33:29 +02:00
antirez 25f017e563 ZPOP: fix replication of blocking ZPOP. 2018-05-15 16:03:56 +02:00
artix 3c039996b5 - Fixed mistyped redis command (clusterManagerGetNodeWithMostKeysInSlot) 
- Cluster node structure is now updated after ADDSLOTS
 2018-05-11 18:29:12 +02:00
antirez e87c1f1e6c ZPOP: fix the tests according to new non-blocking "count" argument. 2018-05-11 18:07:10 +02:00
antirez 56bbab238a ZPOP: change sync ZPOP to have a count argument instead of N keys. 
Usually blocking operations make a lot of sense with multiple keys so
that we can listen to multiple queues (or whatever the app models) with
a single connection. However in the synchronous case it is more useful
to be able to ask for N elements. This is a change that I also wanted to
perform soon or later in the blocking list variant, but here it is more
natural since there is no reply type difference.
 2018-05-11 18:00:32 +02:00
antirez 6efb6c1e06 ZPOP: renaming to have explicit MIN/MAX score idea. 
This commit also adds a top comment about a subtle behavior of mixing
blocking operations of different types in the same key.
 2018-05-11 17:31:53 +02:00
antirez 6b026b70a8 Merge branch 'zpop' of https://github.com/itamarhaber/redis into zpop 2018-05-11 10:40:13 +02:00
antirez b85aae78df Fix rdb.c dictionary iterator release in 2 more places. 2018-05-09 12:06:37 +02:00
antirez cd87b3c71f Fix rdb.c dictionary iterator release. 
Some times it was not released on error, sometimes it was released two
times because the error path expected the "di" var to be NULL if the
iterator was already released. Thanks to @oranagra for pinging me about
potential problems of this kind inside rdb.c.
 2018-05-09 11:03:27 +02:00
Salvatore Sanfilippo aaef5f00cc Merge pull request #4594 from smallstool/bugfix-freeMemoryIfneeded-int-overflow 
fix int overflow problem in freeMemoryIfNeeded
 2018-05-08 17:27:18 +02:00
zhaozhao.zz edb92db533 AOF & RDB: be compatible with rdbchecksum no 2018-05-08 19:22:13 +08:00
赵磊 8d93f924ea Fix dictScan(): It can't scan all buckets when dict is shrinking. 2018-05-08 15:30:11 +08:00
artix 3312de067c Cluster Manager: --cluster options can now be placed everywhere 2018-05-07 17:31:34 +02:00
artix 35b3a8e1ee - Updated create-cluster with redis-cli 
- Updated README
 2018-05-07 15:56:15 +02:00
Krzysztof Filipek fd9177dd33 Typo in preprocessor condition 2018-05-06 20:18:48 +02:00
Itamar Haber 438125b47c Implements [B]Z[REV]POP and the respective unit tests 
An implementation of the
[Ze POP Redis Module](https://github.com/itamarhaber/zpop) as core
Redis commands.

Fixes #1861.
 2018-04-30 02:10:42 +03:00
artix 26b3a0eb21 Merge branch 'trib2cli' of github.com:artix75/redis into trib2cli 2018-04-23 16:29:29 +02:00
artix af4584d608 Cluster tests now using redis-cli instead of redis-trib 2018-04-23 16:28:48 +02:00
artix be94e89031 Cluster Manager: fixed expected slots calculation (rebalance) 
Cluster Manager: fixed argument parsing after --cluster-weight
 2018-04-23 16:28:48 +02:00
artix 2f31545beb Cluster Manager: fixed bug when parsing CLUSTER NODES reply (clusterManagerNodeLoadInfo) 2018-04-23 16:28:48 +02:00
artix 5bc2c98789 Cluster Manager: code improvements and more comments added. 2018-04-23 16:28:48 +02:00
artix 5f358dae33 Cluster Manager: set-timeout command 2018-04-23 16:28:48 +02:00
artix 52f17f6f8e - Cluster Manager: del-node command. 
- Cluster Manager: fixed bug in clusterManagerNodeWithLeastReplicas
 2018-04-23 16:28:48 +02:00
artix aeaf6ee1c3 Cluster Manager: add-node command. 2018-04-23 16:28:48 +02:00
artix efa51f1617 Cluster Manager: added clusterManagerCheckCluster to import command 2018-04-23 16:28:48 +02:00
artix 1f548359cb Cluster Manager: import command 2018-04-23 16:28:48 +02:00
artix 3f8a4adb49 Cluster Manager: fix command. 2018-04-23 16:28:48 +02:00
artix 6d1a7cec23 Cluster Manager: rebalance command 2018-04-23 16:28:48 +02:00
artix adebee8adc clusterManagerAddSlots: changed the way ADDSLOTS command is built 2018-04-23 16:28:48 +02:00
artix a4a1c7bb52 ClusterManager: fixed --cluster-from 'all' parsing 2018-04-23 16:28:48 +02:00
Artix 5b3d73b3d7 Cluster Manager: fixed some memory error 2018-04-23 16:28:48 +02:00
artix 220375b424 Fixed memory write error in clusterManagerGetConfigSignature 2018-04-23 16:28:48 +02:00
artix 99da9c9508 Cluster Manager: reshard command, fixed slots 
parsing bug and other minor bugs.
 2018-04-23 16:28:48 +02:00
artix 7d609ff952 Cluster Manager: 
- Almost all Cluster Manager related code moved to
  the same section.
- Many macroes converted to functions
- Added various comments
- Little code restyling
 2018-04-23 16:28:48 +02:00
artix 4e0c2f9c3c - Fixed bug in clusterManagerGetAntiAffinityScore 
- Code improvements
 2018-04-23 16:28:48 +02:00
artix 18910013cd Cluster Manager: colorized output 2018-04-23 16:28:48 +02:00
artix 307d995f75 Cluster Manager: improved cleanup/error handling in various functions 2018-04-23 16:28:48 +02:00
artix 2d677e2bf3 Cluster Manager: 'call' command. 2018-04-23 16:28:48 +02:00
artix 1ad1f00163 Cluster Manager: CLUSTER_MANAGER_NODE_CONNECT macro 2018-04-23 16:28:48 +02:00
artix 2f48d62423 ClusterManager: added replicas count to clusterManagerNode 2018-04-23 16:28:48 +02:00
artix c002b95d89 Cluster Manager: cluster is considered consistent if only one node has been found 2018-04-23 16:28:48 +02:00
artix 4cc8de1a37 Cluster Manager: reply error catch for MEET command 2018-04-23 16:28:48 +02:00
artix 65d37960e7 Cluster Manager: slots coverage check. 2018-04-23 16:28:48 +02:00
artix b3e0ca3412 - Cluster Manager: fixed various memory leaks 
- Cluster Manager: fixed flags assignment in
  clusterManagerNodeLoadInfo
 2018-04-23 16:28:48 +02:00
artix 8c7ad80f9f Added check for open slots (clusterManagerCheckCluster) 2018-04-23 16:28:47 +02:00
artix 486c7af7b8 Cluster Manager: 'create', 'info' and 'check' commands 2018-04-23 16:28:47 +02:00
artix ad911a338a Cluster Manager mode 2018-04-23 16:28:47 +02:00
zhaozhao.zz 24036b4d32 RDB: expand dict if needed when rdb load object 2018-04-22 22:30:44 +08:00
artix 04eac76bae Cluster tests now using redis-cli instead of redis-trib 2018-04-20 19:29:42 +02:00
artix 57d895104c Cluster Manager: fixed expected slots calculation (rebalance) 
Cluster Manager: fixed argument parsing after --cluster-weight
 2018-04-20 19:25:10 +02:00
artix cc29fd3e28 Cluster Manager: fixed bug when parsing CLUSTER NODES reply (clusterManagerNodeLoadInfo) 2018-04-20 18:08:30 +02:00
artix 3fce4301ec Cluster Manager: code improvements and more comments added. 2018-04-19 18:52:01 +02:00
antirez e6b0e8d9ec Streams: XTRIM command added. 2018-04-19 16:25:29 +02:00
antirez 19ae809458 Streams: XADD, handle signaling of write in different ways. 2018-04-19 16:11:43 +02:00
antirez aba76320d5 Streams: XDEL command. 2018-04-18 13:12:09 +02:00
antirez 9c149bf1f1 Streams: higher level "delete by ID" API for streams. 2018-04-17 17:55:56 +02:00
antirez 24ac2b4c74 Streams: iterator entry deletion abilities. 2018-04-17 17:18:00 +02:00
Salvatore Sanfilippo 7980d87c3c Merge pull request #4842 from soloestoy/fix-timer-api-wrong-insert 
Modules Timer API: fix wrong raxInsert() usage
 2018-04-13 19:51:46 +02:00
zhaozhao.zz 775adf544c Modules Timer API: fix wrong raxInsert() usage 2018-04-14 01:20:02 +08:00
artix 07bd371087 Cluster Manager: set-timeout command 2018-04-13 16:09:22 +02:00
antirez e07af6a2b7 Modules API: Add call to get the blocked client handle from the context. 
This is useful in the reply and timeout callback, if the module wants to
do some cleanup of the blocked client handle that may be stored around
in the module-private data structures.
 2018-04-13 13:48:11 +02:00
antirez da0e192277 Modules API: moduleGetReplyClient() refactoring. 2018-04-13 13:42:49 +02:00
antirez 404160a271 Modules API: blocked client disconnection callback. 2018-04-12 13:21:48 +02:00
antirez 005c932f22 Modules API: fix timer example. 2018-04-12 13:00:18 +02:00
artix 551f8f0591 - Cluster Manager: del-node command. 
- Cluster Manager: fixed bug in clusterManagerNodeWithLeastReplicas
 2018-04-11 18:23:28 +02:00
artix 615aefe6ba Cluster Manager: add-node command. 2018-04-11 17:08:53 +02:00
antirez 4e53624052 Modules API: fix OOM_WARNING flag implementation. 2018-04-11 16:25:54 +02:00
antirez 0afac6939a Modules API: OOM_WARNING flags. 
In some modules it may be useful to have an idea about being near to
OOM. Anyway additionally an explicit call to get the fill ratio will be
added in the future.
 2018-04-11 16:22:52 +02:00
antirez de7de53e64 getMaxmemoryState() fixed and improved. 2018-04-11 12:48:26 +02:00
artix 81ab5a3b28 Cluster Manager: added clusterManagerCheckCluster to import command 2018-04-10 16:53:24 +02:00
artix eaaa3202e6 Cluster Manager: import command 2018-04-10 16:25:25 +02:00
dejun.xdj b6b00f35ca Return more accurate message when specified ID is invalid 2018-04-10 17:11:10 +08:00
dejun.xdj 3c20b3fc24 Bugfix: xadd command ID parse 
strictly check the string to be converted,
strtoull() in libc may not set errno to EINVAL when the string contains
invalid digits.
 2018-04-10 17:05:48 +08:00
dejun.xdj 07125b8d22 Bugfix: xadd comand <field,value> arity check 
missing parenthesis causes wrong arithmetic priority.
 2018-04-10 16:36:05 +08:00
antirez f97efe0cac Modules: context flags now include OOM flag. 
Plus freeMemoryIfNeeded() refactoring to improve legibility.
Please review this commit for sanity.
 2018-04-09 17:44:30 +02:00
antirez 6888c1a10d freeMemoryIfNeeded() lacked a top comment. 2018-04-09 17:20:36 +02:00
antirez 9a0dbbb594 Modules: remove trailing empty spaces. 2018-04-09 17:16:55 +02:00
antirez eaafea4828 Modules API: experimental APIs version. 
This way it is possible to use conditional compilation to be compatible
with a larger amount of Redis versions, however note that this breaks
binary compatibiltiy, so the module must be compiled with the
corresponding redismodule.h file depending on the version of Redis
targeted.
 2018-04-09 12:17:19 +02:00
antirez 49e098234a Modules API: blocked client free callback modified to get a context. 
Note that this was an experimental API that can only be enabled with
REIDSMODULE_EXPERIMENTAL_API, so it is subject to change until its
promoted to stable API. Sorry for the breakage, it is trivial to
resolve btw. This change will not be back ported to Redis 4.0.
 2018-04-09 11:54:44 +02:00
artix 8969254e66 Cluster Manager: fix command. 2018-04-06 18:02:40 +02:00
charpty 0fd2b25c8d Remove unnecessary return statements 
Signed-off-by: charpty <charpty@gmail.com>
 2018-04-06 18:46:24 +08:00
antirez b2868c7b9c Modules API: RM_GetRandomBytes() / GetRandomHexChars(). 2018-04-05 13:24:22 +02:00
antirez c75582889a Modules Cluster API: GetClusterSize() added. 2018-04-01 16:36:32 +02:00
antirez 27f9c8108c Modules Cluster API: GetMyClusterID() added. 2018-04-01 16:20:57 +02:00
antirez 19c42c901b Modules Timer API: add example of API. 2018-03-31 10:12:34 +02:00
antirez ee982f4031 Modules Timer API: Wait at least 1 ms per iteration. Convert to ms. 2018-03-31 10:11:30 +02:00
antirez 4c11bc6cf0 Modules Timer API: fix wrong raxRemove() key argument. 2018-03-31 09:58:43 +02:00
antirez 2f7da0fd1a Modules Timer API: fix infinite loop and export API. 2018-03-31 00:44:46 +02:00
antirez b85a465c25 Modules Timer API: timer handling implemented. 2018-03-30 22:50:21 +02:00
antirez 561039c125 Modules Timer API: initial implementation. 2018-03-30 20:40:35 +02:00
antirez 192361b562 Modules Cluster API: node API exported, example improved. 2018-03-30 17:00:45 +02:00
antirez 16178b692e Modules Cluster API: nodes list and info API. 2018-03-30 16:16:47 +02:00
antirez 83ec35770e Modules Cluster API: node information struct and flags. 2018-03-30 13:16:55 +02:00
antirez a97df1a6e1 Modules Cluster API: make node IDs pointers constant. 2018-03-30 13:16:07 +02:00
antirez 061f03d730 Modules Cluster API: add a simple example module. 2018-03-30 12:49:45 +02:00
antirez 82004f9dbe Modules Cluster API: fix new API calls exporting. 2018-03-30 12:49:16 +02:00
antirez b4dc782e4e Modules Cluster API: sending / receiving API first implementation. 2018-03-30 11:06:08 +02:00
antirez 0701cad3de Modules Cluster API: message bus implementation. 2018-03-29 15:13:31 +02:00
zhaozhao.zz 83cf0e3668 adjust position of _dictNextPower in dictExpand 2018-03-29 17:36:15 +08:00
antirez 8ac7af1c5d Fix ae.c when a timer finalizerProc adds an event. 
While this feature is not used by Redis, ae.c implements the ability for
a timer to call a finalizer callback when an timer event is deleted.
This feature was bugged since the start, and because it was never used
we never noticed a problem. However Anthony LaTorre was using the same
library in order to implement a different system: he found a bug that he
describes as follows, and which he fixed with the patch in this commit,
sent me by private email:

    --- Anthony email ---

've found one bug in the current implementation of the timed events.
It's possible to lose track of a timed event if an event is added in
the finalizerProc of another event.

For example, suppose you start off with three timed events 1, 2, and
3. Then the linked list looks like:

3 -> 2 -> 1

Then, you run processTimeEvents and events 2 and 3 finish, so now the
list looks like:

-1 -> -1 -> 2

Now, on the next iteration of processTimeEvents it starts by deleting
the first event, and suppose this finalizerProc creates a new event,
so that the list looks like this:

4 -> -1 -> 2

On the next iteration of the while loop, when it gets to the second
event, the variable prev is still set to NULL, so that the head of the
event loop after the next event will be set to 2, i.e. after deleting
the next event the event loop will look like:

2

and the event with id 4 will be lost.

I've attached an example program to illustrate the issue. If you run
it you will see that it prints:

```
foo id = 0
spam!
```

But if you uncomment line 29 and run it again it won't print "spam!".

    --- End of email ---

Test.c source code is as follows:

    #include "ae.h"
    #include <stdio.h>

    aeEventLoop *el;

    int foo(struct aeEventLoop *el, long long id, void *data)
    {
	printf("foo id = %lld\n", id);

	return AE_NOMORE;
    }

    int spam(struct aeEventLoop *el, long long id, void *data)
    {
	printf("spam!\n");

	return AE_NOMORE;
    }

    void bar(struct aeEventLoop *el, void *data)
    {
	aeCreateTimeEvent(el, 0, spam, NULL, NULL);
    }

    int main(int argc, char **argv)
    {
	el = aeCreateEventLoop(100);

	//aeCreateTimeEvent(el, 0, foo, NULL, NULL);
	aeCreateTimeEvent(el, 0, foo, NULL, bar);

	aeMain(el);

	return 0;
    }

Anthony fixed the problem by using a linked list for the list of timers, and
sent me back this patch after he tested the code in production for some time.
The code looks sane to me, so committing it to Redis.
 2018-03-28 14:11:04 +02:00
antirez 674909f442 Add INIT INFO to the provided init script. 2018-03-26 11:29:16 +02:00
antirez 2b2652d7c4 AOF: run tests with preamble off when it makes sense. 2018-03-25 13:03:38 +02:00
antirez 28d28ef3cf AOF: enable RDB-preamble rewriting by default. 
There are too many advantages in doing this, RDB is faster to persist,
more compact, much faster to load back. The main issues here are that
the code is less tested because this was not the old default (so we are
enabling it for the new 5.0 release), and that the AOF is no longer a
trivially parsable format from now on. However the non-preamble mode
will be supported in the future as well, if new data types will be
added.
 2018-03-25 11:43:30 +02:00
antirez 8b20112860 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-03-23 17:21:46 +01:00
antirez 1392c83fb8 CG: AOF rewriting implemented. 2018-03-23 17:21:31 +01:00
artix d8fc307cc6 Cluster Manager: rebalance command 2018-03-23 16:46:43 +01:00
Salvatore Sanfilippo e08d6bb37e Merge pull request #4703 from gechunlin/unstable 
Update object.c
 2018-03-22 16:32:45 +01:00
Salvatore Sanfilippo bad679137e Merge pull request #4679 from youjiali1995/fix-lru 
Some commands will udpate lfu-counter many times in one call.
 2018-03-22 16:26:01 +01:00
Salvatore Sanfilippo 44f2cfa631 Merge pull request #4722 from charsyam/feature/refactoring-call-aeDeleteFileEvent-twice-in-freeClusterLink 
Refactoring to call aeDeleteFileEvent twice as once
 2018-03-22 16:23:40 +01:00
Salvatore Sanfilippo b38f52169b Merge pull request #4777 from soloestoy/avoid-double-close 
anet: avoid double close
 2018-03-22 16:22:32 +01:00
Salvatore Sanfilippo 3163c9cb63 Merge pull request #4781 from guybe7/block_list_notify 
Make blocking list commands send keyspace notifications
 2018-03-22 16:21:19 +01:00
antirez 6c4cb1670a Add top comments in two addReply*() functions. 2018-03-22 11:45:04 +01:00
antirez b86c26b2fd Massivily simplify addReply*() functions in networking.c 2018-03-22 11:42:50 +01:00
Guy Benoish fa00e20b16 Make blocking list commands send keyspace notifications 2018-03-22 17:22:26 +07:00
Salvatore Sanfilippo 5d8c89163c Merge pull request #4757 from charsyam/feature/fix-invalid-listpack-type-in-makefile 
fix listpack.c to listpack.o in Makefile
 2018-03-22 09:20:17 +01:00
Salvatore Sanfilippo 7acf0e0a18 Merge pull request #3826 from charsyam/feature/fix_dlopen_leak 
fix missed dlclose leak in module.c
 2018-03-22 09:19:26 +01:00
Salvatore Sanfilippo da621783f0 Merge pull request #4691 from oranagra/active_defrag_v2 
Active defrag v2
 2018-03-22 09:16:32 +01:00
Salvatore Sanfilippo 3851d30ada Merge pull request #4779 from charsyam/feature/fix-streamPropagateXCLAIM 
fix memory leak for streamPropagateXCLAIM
 2018-03-22 09:02:37 +01:00
charsyam 71c7477fd4 fix memory leak for streamPropagateXCLAIM 2018-03-21 17:41:59 +09:00
zhaozhao.zz 46f767aa19 fix missed call on freeaddrinfo 2018-03-21 15:34:13 +08:00
zhaozhao.zz 11baa8487c anet: avoid double close 2018-03-21 15:13:51 +08:00
antirez 233da29f4d CG: propagate XACK by incrementing server.dirty. 
This should be more than enough, even if in case of partial IDs that are
not found, we send all the IDs to the slave/AOF, but this is definitely
a corner case without bad effects if not some wasted space.
 2018-03-20 17:53:45 +01:00
antirez 0f2c6b6a18 Streams: improve MEMORY USAGE computation, include CGs. 2018-03-20 17:50:40 +01:00
antirez 5577130451 CG: Make XINFO Great Again (and more Redis-ish). 
With XINFO out of the blue I invented a new syntax for commands never
used in Redis in the past... Let's fix it and make it

        Great Again!!11one (TM)
 2018-03-20 11:52:42 +01:00
antirez 2ba9a57c9a CG: Add JUSTID to XCLAIM propagation to reduce CPU usage. 2018-03-20 11:25:49 +01:00
antirez ad716e29c0 CG: Fix propagate() arg count in streamPropagateXCLAIM(). 2018-03-20 11:13:41 +01:00
antirez 3cae5f0321 CG: XGROUP CREATE/DELCONUSMER replication. 2018-03-20 11:07:01 +01:00
antirez 0b58ad301e CG: Replication WIP 1: XREADGROUP and XCLAIM propagated as XCLAIM. 2018-03-19 18:02:19 +01:00
antirez 3c2a952912 Merge branch 'oertl-hyperloglog-improvement' into unstable 2018-03-16 16:58:16 +01:00
antirez 36b78e8dfe Aesthetic changes to PR #4749. 2018-03-16 16:57:53 +01:00
antirez 87cc94864c Fix HyperLogLog test script for new redis-rb API. 2018-03-16 16:34:04 +01:00
antirez 7e3c583532 Merge branch 'hyperloglog-improvement' of https://github.com/oertl/redis into oertl-hyperloglog-improvement 2018-03-16 16:27:38 +01:00
antirez 38bc8ea531 RDB version 9. 2018-03-16 13:48:44 +01:00
antirez 8b0cfb1e66 RDB: Implement future-proof module AUX data loading. 2018-03-16 13:47:10 +01:00
zhaozhao.zz fd0c4c0277 add rdb-save-incremental-fsync option in redis.conf 2018-03-16 00:48:59 +08:00
zhaozhao.zz 54cae05ea7 rdb: incremental fsync when redis saves rdb 2018-03-16 00:44:50 +08:00
antirez b1aae86fc6 RDB: make RDB check aware of LFU/LRU opcodes. 2018-03-15 16:39:33 +01:00
antirez 8176a2ee76 RDB: LRU/LFU branches missed continue. 2018-03-15 16:33:18 +01:00
antirez 1ce50a7adf RDB: Ability to load LFU/LRU info. 2018-03-15 16:24:53 +01:00
antirez d7a5c0eb71 RDB: Ability to save LFU/LRU info. 
This is a big win for caching use cases, since on reloading Redis will
still have some idea about what is worth to evict and what not.
However this only solves part of the problem because the information is
only partially propagated to slaves (on write operations). Reads will
not affect slaves LFU and LRU counters, so after a failover the eviction
decisions are kinda random until keys start to collect some aging/freq info.

However since new slaves are initially populated via RDB file transfer,
this means that if we spin up a new slave from a master, and perform an
immediate manual failover (for instance in order to upgrade the master),
the slave will have eviction informations to use for some time.

The LFU/LRU info is persisted only if the maxmemory policy is set to one
of the relevant type, even if no actual "maxmemory"  memory limit is
set.
 2018-03-15 13:15:55 +01:00
antirez 6614361615 CG: XINFO STREAM. 2018-03-15 12:54:10 +01:00
antirez d7d8cd0b2f CG: XINFO GROUPS + output format changes. 
XINFO is mainly an observability command that will be used more by
humans than computers, and even when used by computers it will be a very
low traffic command. For this reason the format was changed in order to
have field names. They'll consume some bandwidth and CPU cycles, but in
this context this is much better than having to understand what the
numbers in the output array are.
 2018-03-15 12:54:10 +01:00
antirez 0cf6b1e3ae CG: XINFO CONSUMERS implemented. 2018-03-15 12:54:10 +01:00
antirez 67eeeb0b10 Streams: fix XREAD missing check for NULL object. 2018-03-15 12:54:10 +01:00
antirez 8d8755c7b5 CG: throw an error if XREADGROUP is used without GROUP. 2018-03-15 12:54:10 +01:00
antirez d4f81ebdba CG: XGROUP DELCONSUMER implemented. 2018-03-15 12:54:10 +01:00
antirez 9b423ae237 CG: remove stale comment. XREADGROUP is now implemented. 2018-03-15 12:54:10 +01:00
antirez e6d434c167 CG: implement RETRYCOUNT in XCLAIM. 2018-03-15 12:54:10 +01:00
antirez f3d9520ccb CG: fix CG RDB loading not found conditional. 2018-03-15 12:54:10 +01:00
antirez 34614a1508 CG: NACK created by XCLAIM FORCE must be set in global PEL. 2018-03-15 12:54:10 +01:00
antirez 596264aee9 CG: implement XCLAIM FORCE option. 2018-03-15 12:54:10 +01:00
antirez b26f03bd69 CG: XCLAIM now updates the idle time of the message. 2018-03-15 12:54:10 +01:00
antirez f7d4c3acdf Streams: trap more errors in stream loading + RDB check type name. 2018-03-15 12:54:10 +01:00
antirez 8727b4845b CG: XCLAIM, use minidle and fix array len. 2018-03-15 12:54:10 +01:00
antirez 09e3b3b975 CG: remove unused argument from streamReplyWithRangeFromConsumerPEL(). 2018-03-15 12:54:10 +01:00
antirez 13ff7bc3ef CG: fix RDB saving when there are no consumer groups. 2018-03-15 12:54:10 +01:00
antirez 267f7f2c97 Streams: fix error description for XADD when specified ID is small. 2018-03-15 12:54:10 +01:00
antirez 0a6780e560 CG: XCLAIM initial draft. 2018-03-15 12:54:10 +01:00
antirez 00a29b1a81 Make addReplyError...() family functions able to get error codes. 
Now you can use:

    addReplyError("-MYERRORCODE some message");

If the error code is omitted, the behavior is like in the past,
the generic -ERR will be used.
 2018-03-15 12:54:10 +01:00
antirez 54b9a0e612 CG: test XACK remaining features. 2018-03-15 12:54:10 +01:00
antirez 5f8c57f28a CG: test XACK ability to remove items from the PELs. 2018-03-15 12:54:10 +01:00
antirez f5799e728a CG: test XPENDING ability to return pending items. 2018-03-15 12:54:10 +01:00
antirez c2ecac4746 CG: test XGROUPREAD abilities. 2018-03-15 12:54:10 +01:00
antirez 18ab0e31f3 CG: test group creation. 2018-03-15 12:54:10 +01:00
antirez c9d86c2b16 CG: More specific duplicated group error. 2018-03-15 12:54:10 +01:00
antirez 9f60a6bcee CG: RDB loading, fix inverted conditional. 2018-03-15 12:54:10 +01:00
antirez f4e1a4de25 CG: RDB loading first implementation. 2018-03-15 12:54:10 +01:00
antirez db7a5f23b4 CG: RDB saving part 2, consumers. 2018-03-15 12:54:10 +01:00
antirez 8fb6048ed0 CG: RDB saving part 1, metadata and PEL. 2018-03-15 12:54:10 +01:00
antirez e76fb4ab25 CG: XPENDING should not create consumers and obey to count. 2018-03-15 12:54:10 +01:00
antirez f3708af7f9 CG: XPENDING with start/stop/count variant implemented. 2018-03-15 12:54:10 +01:00
antirez 1bc31666da CG: XPENDING without start/stop variant implemented. 2018-03-15 12:54:10 +01:00
antirez b65fe09bb8 CG: Now XREADGROUP + blocking operations work. 2018-03-15 12:54:10 +01:00
antirez 5ad29325fe CG: XACK should return zero when nothing is processed. 2018-03-15 12:54:10 +01:00
antirez 388c69fe4e CG: XACK implementation. 2018-03-15 12:54:10 +01:00
antirez 5bbd117c29 CG: XREADGROUP can fetch data from the consumer PEL. 2018-03-15 12:54:10 +01:00
antirez aa808394f6 CG: first draft of streamReplyWithRangeFromConsumerPEL(). 2018-03-15 12:54:10 +01:00
antirez bbec4569a5 CG: Fix order of calls in streamReplyWithRange(). 
We need to check if we are going to serve the request via the PEL before
inserting a deferred array len in the client output buffer.
 2018-03-15 12:54:10 +01:00
antirez 41809fd969 CG: creation of NACK entries in PELs. 2018-03-15 12:54:10 +01:00
antirez 1ffb6723f5 CG: fix XREADGROUP ">" special ID parsing due to missing "continue". 2018-03-15 12:54:10 +01:00
antirez 6c0af37b6e CG: streamCompareID() + group last_id updating. 2018-03-15 12:54:10 +01:00
antirez 86fe8fde20 CG: consumer lookup + initial streamReplyWithRange() work to supprot CG. 2018-03-15 12:54:10 +01:00
antirez ccdae09046 CG: add & populate group+consumer in the blocking state. 2018-03-15 12:54:10 +01:00
antirez b8e5232161 CG: fix parsing in XREADGROUP and streamLookupCG() NULL check. 2018-03-15 12:54:10 +01:00
antirez bd1c11dc35 CG: add XREADGROUP in the command table. 2018-03-15 12:54:10 +01:00
antirez 2bbb2bf427 CG: XGROUPREAD group option parsing and groups lookup. 2018-03-15 12:54:10 +01:00
antirez 1fafe7def1 CG: fix raxFind() retval check in streamCreateCG(). 2018-03-15 12:54:10 +01:00
antirez 58f0c000a5 CG: data structures design + XGROUP CREATE implementation. 2018-03-15 12:54:10 +01:00
charsyam c76f890209 fix listpack.c to listpack.o in Makefile 2018-03-15 20:32:08 +09:00
Otmar Ertl 15d7e61701 fixed compilation error when using clang as reported by michael-grunder 2018-03-14 21:00:06 +01:00
antirez d8207d09ee Cluster: add test for the nofailover flag. 2018-03-14 16:30:32 +01:00
antirez 432bf4770e Cluster: ability to prevent slaves from failing over their masters. 
This commit, in some parts derived from PR #3041 which is no longer
possible to merge (because the user deleted the original branch),
implements the ability of slaves to have a special configuration
preventing that they try to start a failover when the master is failing.

There are multiple reasons for wanting this, and the feautre was
requested in issue #3021 time ago.

The differences between this patch and the original PR are the
following:

1. The flag is saved/loaded on the nodes configuration.
2. The 'myself' node is now flag-aware, the flag is updated as needed
   when the configuration is changed via CONFIG SET.
3. The flag name uses NOFAILOVER instead of NO_FAILOVER to be consistent
   with existing NOADDR.
4. The redis.conf documentation was rewritten.

Thanks to @deep011 for the original patch.
 2018-03-14 14:01:38 +01:00
Oran Agra 806736cdf9 Adding real allocator fragmentation to INFO and MEMORY command + active defrag test 
other fixes / improvements:
- LUA script memory isn't taken from zmalloc (taken from libc malloc)
  so it can cause high fragmentation ratio to be displayed (which is false)
- there was a problem with "fragmentation" info being calculated from
  RSS and used_memory sampled at different times (now sampling them together)

other details:
- adding a few more allocator info fields to INFO and MEMORY commands
- improve defrag test to measure defrag latency of big keys
- increasing the accuracy of the defrag test (by looking at real grag info)
  this way we can use an even lower threshold and still avoid false positives
- keep the old (total) "fragmentation" field unchanged, but add new ones for spcific things
- add these the MEMORY DOCTOR command
- deduct LUA memory from the rss in case of non jemalloc allocator (one for which we don't "allocator active/used")
- reduce sampling rate of the rss and allocator info
 2018-03-12 15:08:52 +02:00
Oran Agra be1b4aa9aa active defrag v2 
- big keys are not defragged in one go from within the dict scan
  instead they are scanned in parts after the main dict hash bucket is done.
- add latency monitor sample for defrag
- change default active-defrag-cycle-min to induce lower latency
- make active defrag start a new scan right away if needed, so it's easier
  (for the test suite) to detect when it's done
- make active defrag quick the current cycle after each db / big key
- defrag  some non key long term global allocations
- some refactoring for smaller functions and more reusable code
- during dict rehashing, one scan iteration of the dict, can end up scanning
  one bucket in the smaller dict and many many buckets in the larger dict.
  so waiting for 16 scan iterations before checking the time, may be much too long.
 2018-03-12 15:07:43 +02:00
Otmar Ertl 97bde9f623 use all 64 bits of the hash value instead of 63 2018-03-11 09:18:00 +01:00
Otmar Ertl 44698f45e7 made constant static 2018-03-10 20:44:20 +01:00
Otmar Ertl 633983d479 improved definition of HLL_Q 2018-03-10 20:22:42 +01:00
Otmar Ertl 1e9a774871 improved HyperLogLog cardinality estimation 
based on method described in https://arxiv.org/abs/1702.01284
that does not rely on any magic constants
 2018-03-10 20:13:21 +01:00
Otmar Ertl 6470b21f59 replaced tab by spaces 2018-03-10 20:09:41 +01:00
artix a4cfd503ea clusterManagerAddSlots: changed the way ADDSLOTS command is built 2018-03-06 13:06:04 +02:00
artix d518733073 ClusterManager: fixed --cluster-from 'all' parsing 2018-03-02 17:06:50 +01:00
pan.liangp f4eb64cd35 move get clients max buffer calculate into info clients command 2018-03-02 17:16:00 +08:00
antirez 84b281209a Stream: update the listpack pointer in streamTrimByLength(). 2018-03-01 17:26:02 +01:00
antirez efcbc01fbd Remove warning from lpGet snprintf(). 2018-03-01 15:26:27 +01:00
antirez d63caaa820 redis-cli: fix missed unit in array. Change define name. 2018-03-01 15:06:41 +01:00
charsyam da7f5700cf refactoring-call-aeDeleteFileEvent-twice-in-freeClusterLink 2018-03-01 22:30:39 +09:00
charsyam 51a03f6356 fix dlopen leak 2018-03-01 21:22:42 +09:00
Salvatore Sanfilippo 83b5b5a476 Merge pull request #4714 from charsyam/feature/fix-out-of-index-range 
[BugFix] Fix out of array index range for findBigKeys in redis-cli
 2018-03-01 03:39:15 -08:00
antirez 3a5bf75ede Actually use ae_flags to add AE_BARRIER if needed. 
Many thanks to @Plasma that spotted this problem reviewing the code.
 2018-02-28 18:03:51 +01:00
Artix ce14d23740 Cluster Manager: fixed some memory error 2018-02-28 15:21:08 +01:00
artix fb41b8bb9c Fixed memory write error in clusterManagerGetConfigSignature 2018-02-28 11:49:10 +01:00
artix 2f056b8331 Cluster Manager: reshard command, fixed slots 
parsing bug and other minor bugs.
 2018-02-28 10:44:14 +01:00
Salvatore Sanfilippo 7a73db7512 Merge pull request #4715 from charsyam/feature/refactoring-make-condition-clear-for-rdb 
[BugFix] fix calculation length in rdbSaveAuxField
 2018-02-27 10:15:27 -08:00
antirez 92696e49d2 expireIfNeeded() needed a top comment documenting the behavior. 2018-02-27 16:44:43 +01:00
antirez b00c4ffab5 expireIfNeeded() comment: claim -> pretend. 2018-02-27 16:37:37 +01:00
charsyam 76386c48b8 refactoring-make-condition-clear-for-rdb 2018-02-27 21:55:20 +09:00
charsyam 6168d5a1a6 fix-out-of-index-range-for-redis-cli-findbigkey 2018-02-27 21:46:19 +09:00
antirez 956350ef89 ae.c: insetad of not firing, on AE_BARRIER invert the sequence. 
AE_BARRIER was implemented like:

    - Fire the readable event.
    - Do not fire the writabel event if the readable fired.

However this may lead to the writable event to never be called if the
readable event is always fired. There is an alterantive, we can just
invert the sequence of the calls in case AE_BARRIER is set. This commit
does that.
 2018-02-27 13:06:42 +01:00
antirez 75987431f0 AOF: fix a bug that may prevent proper fsyncing when fsync=always. 
In case the write handler is already installed, it could happen that we
serve the reply of a query in the same event loop cycle we received it,
preventing beforeSleep() from guaranteeing that we do the AOF fsync
before sending the reply to the client.

The AE_BARRIER mechanism, introduced in a previous commit, prevents this
problem. This commit makes actual use of this new feature to fix the
bug.
 2018-02-27 13:06:42 +01:00
antirez 533d0e0375 Cluster: improve crash-recovery safety after failover auth vote. 
Add AE_BARRIER to the writable event loop so that slaves requesting
votes can't be served before we re-enter the event loop in the next
iteration, so clusterBeforeSleep() will fsync to disk in time.
Also add the call to explicitly fsync, given that we modified the last
vote epoch variable.
 2018-02-27 13:06:42 +01:00
antirez 548e478e40 ae.c: introduce the concept of read->write barrier. 
AOF fsync=always, and certain Redis Cluster bus operations, require to
fsync data on disk before replying with an acknowledge.
In such case, in order to implement Group Commits, we want to be sure
that queries that are read in a given cycle of the event loop, are never
served to clients in the same event loop iteration. This way, by using
the event loop "before sleep" callback, we can fsync the information
just one time before returning into the event loop for the next cycle.
This is much more efficient compared to calling fsync() multiple times.

Unfortunately because of a bug, this was not always guaranteed: the
actual way the events are installed was the sole thing that could
control. Normally this problem is hard to trigger when AOF is enabled
with fsync=always, because we try to flush the output buffers to the
socekt directly in the beforeSleep() function of Redis. However if the
output buffers are full, we actually install a write event, and in such
a case, this bug could happen.

This change to ae.c modifies the event loop implementation to make this
concept explicit. Write events that are registered with:

    AE_WRITABLE|AE_BARRIER

Are guaranteed to never fire after the readable event was fired for the
same file descriptor. In this way we are sure that data is persisted to
disk before the client performing the operation receives an
acknowledged.

However note that this semantics does not provide all the guarantees
that one may believe are automatically provided. Take the example of the
blocking list operations in Redis.

With AOF and fsync=always we could have:

    Client A doing: BLPOP myqueue 0
    Client B doing: RPUSH myqueue a b c

In this scenario, Client A will get the "a" elements immediately after
the Client B RPUSH will be executed, even before the operation is persisted.
However when Client B will get the acknowledge, it can be sure that
"b,c" are already safe on disk inside the list.

What to note here is that it cannot be assumed that Client A receiving
the element is a guaranteed that the operation succeeded from the point
of view of Client B.

This is due to the fact that the barrier exists within the same socket,
and not between different sockets. However in the case above, the
element "a" was not going to be persisted regardless, so it is a pretty
synthetic argument.
 2018-02-27 13:06:42 +01:00
Salvatore Sanfilippo d8830200b4 Merge pull request #3828 from oranagra/sdsnewlen_pr 
add SDS_NOINIT option to sdsnewlen to avoid unnecessary memsets.
 2018-02-27 04:04:32 -08:00
antirez 813960dbdd Fix ziplist prevlen encoding description. See #4705. 2018-02-23 12:19:35 +01:00
gechunlin d4e6d1086f Update object.c 2018-02-22 20:57:54 -06:00
artix 8f4f001dc3 Cluster Manager: 
- Almost all Cluster Manager related code moved to
  the same section.
- Many macroes converted to functions
- Added various comments
- Little code restyling
 2018-02-22 18:35:40 +01:00
artix 87f5a7c0b4 - Fixed bug in clusterManagerGetAntiAffinityScore 
- Code improvements
 2018-02-22 18:35:40 +01:00
artix 605d7262e6 Cluster Manager: colorized output 2018-02-22 18:35:40 +01:00
artix 4ca8dbdc2b Cluster Manager: improved cleanup/error handling in various functions 2018-02-22 18:35:40 +01:00
artix 8128f1bf03 Cluster Manager: 'call' command. 2018-02-22 18:35:40 +01:00
artix 7b9f945b37 Cluster Manager: CLUSTER_MANAGER_NODE_CONNECT macro 2018-02-22 18:35:40 +01:00
artix dad69ac320 ClusterManager: added replicas count to clusterManagerNode 2018-02-22 18:35:40 +01:00
artix 956bec4ca8 Cluster Manager: cluster is considered consistent if only one node has been found 2018-02-22 18:35:40 +01:00
artix 1b1f80e60f Cluster Manager: reply error catch for MEET command 2018-02-22 18:35:40 +01:00
artix be7e2b84bd Cluster Manager: slots coverage check. 2018-02-22 18:35:40 +01:00
artix d38045805d - Cluster Manager: fixed various memory leaks 
- Cluster Manager: fixed flags assignment in
  clusterManagerNodeLoadInfo
 2018-02-22 18:35:40 +01:00
artix 74dcd14d13 Added check for open slots (clusterManagerCheckCluster) 2018-02-22 18:35:40 +01:00
artix bafdc1a56c Cluster Manager: 'create', 'info' and 'check' commands 2018-02-22 18:35:40 +01:00
artix 1dd67ebceb Cluster Manager mode 2018-02-22 18:35:39 +01:00
antirez ffde73c57d Track number of logically expired keys still in memory. 
This commit adds two new fields in the INFO output, stats section:

expired_stale_perc:0.34
expired_time_cap_reached_count:58

The first field is an estimate of the number of keys that are yet in
memory but are already logically expired. They reason why those keys are
yet not reclaimed is because the active expire cycle can't spend more
time on the process of reclaiming the keys, and at the same time nobody
is accessing such keys. However as the active expire cycle runs, while
it will eventually have to return to the caller, because of time limit
or because there are less than 25% of keys logically expired in each
given database, it collects the stats in order to populate this INFO
field.

Note that expired_stale_perc is a running average, where the current
sample accounts for 5% and the history for 95%, so you'll see it
changing smoothly over time.

The other field, expired_time_cap_reached_count, counts the number
of times the expire cycle had to stop, even if still it was finding a
sizeable number of keys yet to expire, because of the time limit.
This allows people handling operations to understand if the Redis
server, during mass-expiration events, is able to collect keys fast
enough usually. It is normal for this field to increment during mass
expires, but normally it should very rarely increment. When instead it
constantly increments, it means that the current workloads is using
a very important percentage of CPU time to expire keys.

This feature was created thanks to the hints of Rashmi Ramesh and
Bart Robinson from Twitter. In private email exchanges, they noted how
it was important to improve the observability of this parameter in the
Redis server. Actually in big deployments, the amount of keys that are
yet to expire in each server, even if they are logically expired, may
account for a very big amount of wasted memory.
 2018-02-19 11:12:49 +01:00
antirez aa57481d8c Remove non semantical spaces from module.c. 2018-02-15 21:41:03 +01:00
Salvatore Sanfilippo 7830f8492f Merge pull request #4479 from dvirsky/notify 
Keyspace notifications API for modules
 2018-02-15 21:36:32 +01:00
antirez f4dc736cca Fix typo in notifyKeyspaceEvent() comment. 2018-02-15 21:33:06 +01:00
Dvir Volk 0a36196ce4 Add doc comment about notification flags 2018-02-14 21:54:00 +02:00
Dvir Volk 10efdf307b Add REDISMODULE_NOTIFY_STREAM flag to support stream notifications 2018-02-14 21:50:42 +02:00
Dvir Volk 613831f820 Fix indentation and comment style in testmodule 2018-02-14 21:43:06 +02:00
Dvir Volk f27a64232e Use one static client for all keyspace notification callbacks 2018-02-14 21:40:10 +02:00
Dvir Volk 3aab12414f Remove the NOTIFY_MODULE flag and simplify the module notification flow if there aren't subscribers 2018-02-14 21:40:10 +02:00
Dvir Volk a8e2e99a88 Document flags for notifications 2018-02-14 21:38:58 +02:00
Dvir Volk d4d753dae4 removed some trailing whitespaces 2018-02-14 21:38:58 +02:00
Dvir Volk 5b7b12e38f removed hellonotify.c 2018-02-14 21:38:58 +02:00
Dvir Volk 896db12b41 fixed test 2018-02-14 21:38:58 +02:00
Dvir Volk 2136035e47 finished implementation of notifications. Tests unfinished 2018-02-14 21:38:58 +02:00
Salvatore Sanfilippo 4f2d279dd7 Merge pull request #4685 from charsyam/refactoring/set_max_latency 
Removing duplicated code to set max latency
 2018-02-13 16:20:32 +01:00
charsyam 9d41436115 getting rid of duplicated code 2018-02-14 00:12:13 +09:00
antirez ae29bcd8e2 More verbose logging when slave sends errors to master. 
See #3832.
 2018-02-13 16:01:31 +01:00
Salvatore Sanfilippo 756df19134 Merge pull request #3832 from oranagra/slave_reply_to_master_pr 
when a slave responds with an error on commands that come from master, log it
 2018-02-13 15:55:26 +01:00
Salvatore Sanfilippo f9e6c2046f Merge pull request #3745 from guybe7/unstable 
enlarged buffer given to ld2string
 2018-02-13 15:50:21 +01:00
antirez c14ba46e3a Make it explicit with a comment why we kill the old AOF rewrite. 
See #3858.
 2018-02-13 15:43:34 +01:00
Guy Benoish f782006782 rewriteAppendOnlyFileBackground() failure fix 
It is possible to do BGREWRITEAOF even if appendonly=no. This is by design.
stopAppendonly() didn't turn off aof_rewrite_scheduled (it can be turned on
again by BGREWRITEAOF even while appendonly is off anyway).
After configuring `appendonly yes` it will see that the state is AOF_OFF,
there's no RDB fork, so it will do rewriteAppendOnlyFileBackground() which
will fail since the aof_child_pid is set (was scheduled and started by cron).

Solution:
stopAppendonly() will turn off the schedule flag (regardless of who asked for it).
startAppendonly() will terminate any existing fork and start a new one (so it is the most recent).
 2018-02-13 15:41:06 +01:00
Salvatore Sanfilippo 480d4af1e7 Merge pull request #4684 from oranagra/latency_monitor_max 
fix to latency monitor reporting wrong max latency
 2018-02-13 15:31:11 +01:00
Oran Agra 8e8d957ff8 fix to latency monitor reporting wrong max latency 
in some cases LATENCY HISTORY reported latency that was
higher than the max latency reported by LATENCY LATEST / DOCTOR
 2018-02-13 15:58:40 +02:00
赵磊 aacecbc997 Remove updateLFU() in dbOverwrite(). 2018-02-11 21:02:07 +08:00
antirez 32ac4c64ba Rax updated to latest antirez/rax commit. 2018-02-02 11:10:18 +01:00
zhaozhao.zz 968cb26693 config: handle special configuration "" for auth 2018-01-26 22:49:39 +08:00
Salvatore Sanfilippo 4aa2ecd98b Merge pull request #4269 from jianqingdu/unstable 
fix not call va_end() when syncWrite() failed
 2018-01-24 10:55:25 +01:00
Salvatore Sanfilippo 4bffeda8b0 Merge pull request #4628 from mnunberg/patch-1 
redismodule.h: Check ModuleNameBusy before calling it
 2018-01-24 10:48:04 +01:00
antirez 5f5be1ee33 Fix integration test NOREPLICAS error time dependent false positive. 2018-01-24 10:10:48 +01:00
Mark Nunberg 062bd733da redismodule.h: Check ModuleNameBusy before calling it 
Older versions might not have this function.
 2018-01-23 10:49:18 -05:00
antirez 727dd43614 Fix migrateCommand() access of not initialized byte. 2018-01-18 12:41:05 +01:00
Guy Benoish fd8efb7c36 Replication buffer fills up on high rate traffic. 
When feeding the master with a high rate traffic the the slave's feed is much slower.
This causes the replication buffer to grow (indefinitely) which leads to slave disconnection.
The problem is that writeToClient() decides to stop writing after NET_MAX_WRITES_PER_EVENT
writes (In order to be fair to clients).
We should ignore this when the client is a slave.
It's better if clients wait longer, the alternative is that the slave has no chance to stay in
sync in this situation.
 2018-01-18 12:10:48 +01:00
antirez 1673a3f32c Cluster: improve anti-affinity algo in redis-trib.rb. 
See #3462 and related PRs.

We use a simple algorithm to calculate the level of affinity violation,
and then an optimizer that performs random swaps until things improve.
 2018-01-18 11:44:19 +01:00
antirez e1e0bbe04d Remove useless comment from serverCron(). 
The behavior is well specified by the code itself.
 2018-01-17 11:23:41 +01:00
Salvatore Sanfilippo a18e4c964e Merge pull request #4546 from hqin6/unstable 
fixbug for #4545 dead loop aof rewrite
 2018-01-17 11:21:55 +01:00
heqin 3d3faa0a19 fixbug for #4545 dead loop aof rewrite 2018-01-17 18:08:30 +08:00
Salvatore Sanfilippo 81401878de Merge pull request #4609 from Qinch/unstable 
fix assert problem in ZIP_DECODE_PREVLENSIZE macro
 2018-01-17 10:45:11 +01:00
antirez b23927b240 Hopefully more clear comment to explain the change in #4607. 2018-01-16 15:52:13 +01:00
qinchao 1e0e168570 fix assert problem in ZIP_DECODE_PREVLENSIZE 
, see issue: https://github.com/antirez/redis/issues/4587
 2018-01-16 22:43:06 +08:00
Salvatore Sanfilippo 0cc43760d7 Merge pull request #4607 from oranagra/psync2_backlog 
PSYNC2 fix - promoted slave should hold on to it's backlog
 2018-01-16 15:32:58 +01:00
Oran Agra 689b64c3ad PSYNC2 fix - promoted slave should hold on to it's backlog 
after a slave is promoted (assuming it has no slaves
and it booted over an hour ago), it will lose it's replication
backlog at the next replication cron, rather than waiting for slaves
to connect to it.
so on a simple master/slave faiover, if the new slave doesn't connect
immediately, it may be too later and PSYNC2 will fail.
 2018-01-16 10:10:42 +02:00
zhaozhao.zz 1b8eec3e53 aof: format code and comment 2018-01-15 13:01:03 +01:00
antirez c45366be0a Put more details in the comment introduced by #4601. 2018-01-15 12:50:08 +01:00
Salvatore Sanfilippo 1ed5ac7ce5 Merge pull request #4601 from soloestoy/fix-memoryleak-for-lazy-server-del 
lazyfree: fix memory leak for lazyfree-lazy-server-del
 2018-01-15 12:43:55 +01:00
zhaozhao.zz 0517ab8397 lazyfree: fix memory leak for lazyfree-lazy-server-del 2018-01-15 00:45:37 +08:00
Salvatore Sanfilippo aeeb747796 Merge pull request #4575 from soloestoy/bugfix-benchmark 
redis-benchmark: bugfix - handle zero liveclients in right way
 2018-01-12 17:43:01 +01:00
Salvatore Sanfilippo 72187fa8a5 Merge pull request #4581 from dvirsky/module_unlink 
Added RM_UnlinkKey - a low level analog to UNLINK command
 2018-01-12 17:41:09 +01:00
antirez a5b6bc2bd7 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2018-01-12 17:16:38 +01:00
Salvatore Sanfilippo 71914387ba Merge pull request #4586 from gnuhpc/fix-crashlog-typo 
Fix a typo(maybe instruction?) in crash log
 2018-01-12 17:16:12 +01:00
antirez 2f8476df91 Fix getKeysUsingCommandTable() in the case of nagative arity. 
This fixes a crash with Redis Cluster when OBJECT is mis-used, because
getKeysUsingCommandTable() will call serverPanic() detecting we are
accessing an invalid argument in the case "OBJECT foo" is called.

This bug was introduced when OBJECT HELP was introduced, because the key
argument is set fixed at index 2 in the command table, however now
OBJECT may be called with an insufficient number of arguments to extract
the key.

The "Right Thing" would be to have a specific function to extract keys
from the OBJECT command, however this is kinda of an overkill, so I
preferred to make getKeysUsingCommandTable() more robust and just return
no keys when it's not possible to honor the command table, because new
commands are often added and also there are a number with an HELP
subcommand violating the normal form, and crashing for this trivial
reason or having many command-specific key extraction functions is not
great.
 2018-01-12 11:26:29 +01:00
antirez db7bd6fda9 Document new protocol options in #4568 into redis.conf. 2018-01-11 11:40:57 +01:00
antirez 13650446ac proto-max-querybuf-len -> client-query-buffer-limit. 
We already had client buffer limits exported as configuration options.
Stick with the naming scheme already used.

See #4568.
 2018-01-11 11:36:26 +01:00
antirez 8075572207 New config options about protocol prefixed with "proto". 
Related to #4568.
 2018-01-11 11:27:41 +01:00
Salvatore Sanfilippo c2fa4b8a60 Merge pull request #4568 from oranagra/restore_size_limit 
fix RESTORE command size limits
 2018-01-11 11:20:02 +01:00
huijing.whj f0e09bf803 fix int overflow problem in freeMemoryIfNeeded 2018-01-10 23:47:02 +08:00
Salvatore Sanfilippo e954742fbd Merge pull request #3356 from yusaku/fix-module-firstkey 
Fix the firstkey, lastkey, and keystep of moduleCommand
 2018-01-09 18:58:31 +01:00
antirez c17a0e134f Test: MIGRATE AUTH test added. 
Related to #2507.
 2018-01-09 18:49:19 +01:00
antirez 3ce1c28d47 Rewrite MIGRATE AUTH option. 
See PR #2507. This is a reimplementation of the fix that contained
different problems.
 2018-01-09 18:48:26 +01:00
gnuhpc 36e71b88b5 Fix a typo(maybe instruction?) in crash log 2018-01-09 16:13:02 +08:00
Dvir Volk e76dfc90a7 Added RM_UnlinkKey - a low level analog to UNLINK command 2018-01-07 16:41:43 +02:00
Salvatore Sanfilippo e509fbb8d9 Merge pull request #4579 from gnuhpc/fix-memoryusage-list 
Fix memory usage list bug
 2018-01-05 11:53:11 +01:00
gnuhpc 5113a53df6 Fix memory usage list bug 2018-01-05 12:16:24 +08:00
zhaozhao.zz c0f611f12a redis-benchmark: bugfix - handle zero liveclients in right way 2018-01-03 12:47:25 +08:00
Oran Agra b509a14c3e Add config options for max-bulk-len and max-querybuf-len mainly to support RESTORE of large keys 2017-12-29 12:43:48 +02:00
Oran Agra 60a4f12f8b fix processing of large bulks (above 2GB) 
- protocol parsing (processMultibulkBuffer) was limitted to 32big positions in the buffer
  readQueryFromClient potential overflow
- rioWriteBulkCount used int, although rioWriteBulkString gave it size_t
- several places in sds.c that used int for string length or index.
- bugfix in RM_SaveAuxField (return was 1 or -1 and not length)
- RM_SaveStringBuffer was limitted to 32bit length
 2017-12-29 12:24:19 +02:00
antirez 0b561883b4 Hyperloglog: refresh hdr variable correctly. 
This is a fix for the #3819 improvements. The o->ptr may change because
of hllSparseSet() calls, so 'hdr' must be correctly re-fetched.
 2017-12-22 11:26:31 +01:00
antirez 27b81f3fbc Hyperloglog: Support for PFMERGE sparse encoding as target. 
This is a fix for #3819.
 2017-12-22 11:01:27 +01:00
antirez cbe9d725a7 Hyperloglog: refactoring of sparse/dense add function. 
The commit splits the add functions into a set() and add() set of
functions, so that it's possible to set registers in an independent way
just having the index and count.

Related to #3819, otherwise a fix is not possible.
 2017-12-22 11:00:22 +01:00
heqin 365e6c45ea fixbug for #4545 dead loop aof rewrite 2017-12-18 17:59:03 +08:00
Salvatore Sanfilippo 238c9bd086 Merge pull request #4540 from hqin6/unstable 
fixbug for #4538 Error opening /setting AOF rewrite IPC pipes: No suc…
 2017-12-15 12:41:02 +01:00
heqin ee724cac49 fixbug for #4538 Error opening /setting AOF rewrite IPC pipes: No such file or directory 2017-12-15 11:06:58 +08:00
Salvatore Sanfilippo d745e8dfb3 Merge pull request #4536 from tporadowski/antirez_unstable-cli-history 
Always enable command history in redis-cli run on a TTY
 2017-12-14 22:14:45 +01:00
Salvatore Sanfilippo 398b2084af Merge pull request #4528 from nashe/patch-oob-read-4527 
Prevent off-by-one read in stringmatchlen()
 2017-12-14 12:21:32 +01:00
antirez 12e65a424c safe_write -> aofWrite. Function commented. 
Related to #4498.
 2017-12-14 12:19:36 +01:00
Salvatore Sanfilippo 5182dcd645 Merge pull request #4498 from soloestoy/aof-safe-write 
Aof safe write -- fix the short write
 2017-12-14 12:15:46 +01:00
Tomasz Poradowski 77653ebe8a always enable command history in redis-cli 
- when redis-cli is running in a TTY - always enable command history
buffering, regardless if history file path can be successfully
determined
 2017-12-13 23:49:41 +01:00
antirez de276b6a43 Cluster: allow read-only EVAL/EVALSHA in slaves. 
Fix #3665.
 2017-12-13 13:36:01 +01:00
nashe f43eb5adcf Prevent off-by-one read in stringmatchlen() (fixes #4527) 2017-12-12 01:25:03 +01:00
zhaozhao.zz 109ee497be zset: change the span of zskiplistNode to unsigned long 2017-12-08 16:09:27 +08:00
zhaozhao.zz e8901b2fe4 zset: fix the int problem 2017-12-08 15:37:08 +08:00
antirez 522760fac7 Change indentation and other minor details of PR #4489. 
The main change introduced by this commit is pretending that help
arrays are more text than code, thus indenting them at level 0. This
improves readability, and is an old practice when defining arrays of
C strings describing text.

Additionally a few useless return statements are removed, and the HELP
subcommand capitalized when printed to the user.
 2017-12-06 12:05:14 +01:00
Itamar Haber 482d678e95 C style 2017-12-05 19:09:19 +02:00
Itamar Haber b23c8babed Uses an offset in addReplyHelp 2017-12-05 18:17:14 +02:00
Itamar Haber 8b51121998 Merge remote-tracking branch 'upstream/unstable' into help_subcommands 2017-12-05 18:14:59 +02:00
antirez 62a4b817c6 add linkClient(): adds the client and caches the list node. 
We have this operation in two places: when caching the master and
when linking a new client after the client creation. By having an API
for this we avoid incurring in errors when modifying one of the two
places forgetting the other. The function is also a good place where to
document why we cache the linked list node.

Related to #4497 and #4210.
 2017-12-05 16:02:03 +01:00
Salvatore Sanfilippo 03cfc8bf3a Merge pull request #4497 from soloestoy/optimize-unlink-client 
networking: optimize unlinkClient() in freeClient()
 2017-12-05 15:51:15 +01:00
Salvatore Sanfilippo c56fbb246c Merge pull request #4508 from trevor211/fixNotes 
fix some notes
 2017-12-05 15:47:05 +01:00
antirez 479ea57797 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2017-12-05 15:38:21 +01:00
zhaozhao.zz 7c6ddbc37d dict: fix the int problem for defrag 2017-12-05 15:38:03 +01:00
zhaozhao.zz d1176b582c dict: fix the int problem 2017-12-05 15:37:59 +01:00
Salvatore Sanfilippo 6830fa25e3 Merge pull request #4488 from itamarhaber/debug_arity 
Standardizes arity handling of DEBUG
 2017-12-05 15:29:42 +01:00
Salvatore Sanfilippo e4903ce586 Merge pull request #4509 from soloestoy/set-int-problem 
Set int problem
 2017-12-05 15:28:08 +01:00
zhaozhao.zz 42387d6c1a set: fix the int problem for qsort 2017-12-05 17:42:19 +08:00
zhaozhao.zz de809666f8 set: fix the int problem for SPOP & SRANDMEMBER 2017-12-05 17:19:19 +08:00
WuYunlong 3f232ebfb8 fix some notes 2017-12-05 14:41:16 +08:00
Salvatore Sanfilippo e6c3bcf9e0 Merge pull request #4506 from soloestoy/quicklist-int-problem 
Quicklist int problem
 2017-12-04 17:25:34 +01:00
zhaozhao.zz b9491b65d9 quicklist: change the len of quicklist to unsigned long 2017-12-04 19:49:40 +08:00
zhaozhao.zz 664bbfe760 quicklist: fix the return value of quicklistCount 2017-12-04 19:49:23 +08:00
antirez 2869284e44 Streams: fix a few type mismatches in t_stream.c. 2017-12-04 11:59:15 +01:00
antirez 60d26acfc8 Refactoring: improve luaCreateFunction() API. 
The function in its initial form, and after the fixes for the PSYNC2
bugs, required code duplication in multiple spots. This commit modifies
it in order to always compute the script name independently, and to
return the SDS of the SHA of the body: this way it can be used in all
the places, including for SCRIPT LOAD, without duplicating the code to
create the Lua function name. Note that this requires to re-compute the
body SHA1 in the case of EVAL seeing a script for the first time, but
this should not change scripting performance in any way because new
scripts definition is a rare event happening the first time a script is
seen, and the SHA1 computation is anyway not a very slow process against
the typical Redis script and compared to the actua Lua byte compiling of
the body.

Note that the function used to assert() if a duplicated script was
loaded, however actually now two times over three, we want the function
to handle duplicated scripts just fine: this happens in SCRIPT LOAD and
in RDB AUX "lua" loading. Moreover the assert was not defending against
some obvious failure mode, so now the function always tests against
already defined functions at start.
 2017-12-04 11:25:20 +01:00
antirez c6eca690ee Remove useless variable check from luaCreateFunction(). 
The block is already inside if (allow_dup).
 2017-12-04 10:55:54 +01:00
antirez 68681f2bcf Fix issue #4505, Lua RDB AUX field loading of existing scripts. 
Unfortunately, as outlined by @soloestoy in #4505, "lua" AUX RDB field
loading in case of duplicated script was still broken. This commit fixes
this problem and also a memory leak introduced by the past commit.

Note that now we have a regression test able to duplicate the issue, so
this commit was actually tested against the regression. The original PR
also had a valid fix, but I prefer to hide the details of scripting.c
outside scripting.c, and later "SCRIPT LOAD" should also be able to use
the function luaCreateFunction() instead of redoing the work.
 2017-12-04 10:33:04 +01:00
antirez 6f0b19bc5b Regression test for #4505 (Lua AUX field loading). 2017-12-04 10:26:02 +01:00
antirez 6a1bf07a46 DEBUG change-repl-id implemented. 
With PSYNC2 to force a full SYNC in tests is hard. With this new DEBUG
subcommand we just need to call it and then CLIENT KILL TYPE master in
the slave.
 2017-12-04 10:24:52 +01:00
Itamar Haber 51eb6cb395 Adds help to CONFIG 2017-12-03 19:34:31 +02:00
Itamar Haber bd5af03dbd Adds help to CLUSTER command 2017-12-03 19:05:10 +02:00
Itamar Haber ee3884e63c Improve slowlog help 2017-12-03 17:39:52 +02:00
Itamar Haber d884ba4bc9 Helps CLIENT 2017-12-03 16:49:29 +02:00
antirez 65a9740fa8 Fix loading of RDB files lua AUX fields when the script is defined. 
In the case of slaves loading the RDB from master, or in other similar
cases, the script is already defined, and the function registering the
script should not fail in the assert() call.
 2017-12-01 16:01:10 +01:00
antirez 8ac76be5f2 Streams: DEBUG DIGEST support. 2017-12-01 15:04:05 +01:00
antirez f42df6f43a Streams: add code to compute the stream memory usage. 
It's a bit of black magic without actually tracking it inside rax.c,
however Redis usage of the radix tree for the stream data structure is
quite consistent, so a few magic constants apparently are producing
results that make sense.
 2017-12-01 12:50:27 +01:00
antirez 45fe1f5e00 Streams: add some initial test for XREVRANGE. 2017-12-01 10:24:25 +01:00
antirez 115d076d65 Streams: fix lp-count field for non-same-fields entries. 2017-12-01 10:24:25 +01:00
antirez 9bb18e5438 Streams: XRANGE REV option -> XREVRANGE command. 2017-12-01 10:24:25 +01:00
antirez 9dc79c039a Streams: fix reverse iterator discarding of items out of range. 2017-12-01 10:24:25 +01:00
antirez 6919280cc5 Streams: fix reverse iteration next node jumping. 2017-12-01 10:24:25 +01:00
antirez ee3490ec48 Streams: state machine for reverse iteration WIP 1. 2017-12-01 10:24:25 +01:00
antirez 3c5d773f82 Streams: augment stream entries to allow backward scanning. 2017-12-01 10:24:25 +01:00
antirez 0381931b4c Streams: Update listpack to fix 32bit strings encoding error. 
Note that streams produced by XADD in previous broken versions having
elements with 4096 bytes or more will be permanently broken and must be
created again from scratch.

Fix #4428
Fix #4349
 2017-12-01 10:24:24 +01:00
antirez 020fe26bd6 Streams: fix COUNT parsing, issue #4433. 2017-12-01 10:24:24 +01:00
antirez abab0b7817 Streams: fix redis-cli to understand the stream type. 2017-12-01 10:24:24 +01:00
antirez 671b1f6a9d Streams: fix TYPE for stream type. 2017-12-01 10:24:24 +01:00
antirez 1898c50573 Streams: fix XREAD test broken after previous tests improvements. 
10% of times the data is not just "item 0" but there is also the
"otherfield" part. Use [lrange] to avoid the issue.
This commit fixes #4416.
 2017-12-01 10:24:24 +01:00
antirez 5082ec6419 Streams: move ID ms/seq separator from '.' to '-' 
After checking with the community via Twitter (here:
https://twitter.com/antirez/status/915130876861788161) the verdict was to
use ":". However I later realized, after users lamented the fact that
it's hard to copy IDs just with double click, that this was the reason
why I moved to "." in the first instance. Fortunately "-", that was the
other option with most votes, also gets selected with double click on
most terminal applications on Linux and MacOS.

So my reasoning was:

1) We can't retain "." because it's actually confusing to newcomers, it
looks like a floating number, people may be tricked into thinking they
can order IDs numerically as floats.

2) Moving to a double-click-to-select format is much better. People will
work with such IDs for long time when coding / debugging. Why making now
a choice that will impact this for the next years?

The only other viable option was "-", and that's what I did. Thanks.
 2017-12-01 10:24:24 +01:00
antirez 50595a5889 Streams: fix XADD + MAXLEN propagation due to var shadowing. 
Clang should be more prone to return warnings by default when there is
same-var-name shadowing. GCC does this and can avoid bugs like that.
 2017-12-01 10:24:24 +01:00
antirez a4e6aae6b8 Streams: fix memory leak in streamTrimByLength(). 2017-12-01 10:24:24 +01:00
antirez e53c90308b Streams: add XADD + MAXLEN test. 2017-12-01 10:24:24 +01:00
antirez 0248a6b125 Streams: fix streamTrimByLength() standalone items skipping. 2017-12-01 10:24:24 +01:00
antirez 0540803288 Streams: XADD MAXLEN implementation. 
The core of this change is the implementation of stream trimming, and
the resulting MAXLEN option of XADD as a trivial result of having
trimming functionalities. MAXLEN already works but in order to be more
efficient listpack GC should be implemented, currently marked as a TODO
item inside the comments.
 2017-12-01 10:24:24 +01:00
antirez 0c00fd7834 Streams: reduce listpack max size to 2k to speedup range queries. 
Listpack max size is a tradeoff between space and time. A 2k max entry
puts the memory usage approximately at a similar order of magnitude (5
million entries went from 96 to 120 MB), but the range queries speed
doubled (because there are half entries to scan in the average case).

Lower values could be considered, or maybe this parameter should be
made tunable.
 2017-12-01 10:24:24 +01:00
antirez f24d3a7de0 Streams: delta encode IDs based on key. Add count + deleted fields. 
We used to have the master ID stored at the start of the listpack,
however using the key directly makes more sense in order to create a
space efficient representation: anyway the key at the radix tree is very
unlikely to change because of how the stream is implemented. Moreover on
nodes merging, to rewrite the merged listpacks is anyway the most
sensible operation, and we can use the iterator and the append-to-stream
function in order to avoid re-implementing the code needed for merging.

This commit also adds two items at the start of the listpack: the
number of valid items inside the listpack, and the number of items
marked as deleted. This means that there is no need to scan a listpack
in order to understand if it's a good candidate for garbage collection,
if the ration between valid/deleted items triggers the GC.
 2017-12-01 10:24:24 +01:00
antirez cea421a021 Streams: specify better how the master enty works. 2017-12-01 10:24:24 +01:00
antirez 7d0d9693c1 Streams: modify tests to stress compression. 2017-12-01 10:24:24 +01:00
antirez 3f2d7e277e Streams: items compression implemented. 
The approach used is to set a fixed header at the start of every
listpack blob (that contains many entries). The header contains a
"master" ID and fields, that are initially just obtained from the first
entry inserted in the listpack, so that the first enty is always well
compressed. Later every new entry is checked against these fields, and
if it matches, the SAMEFIELD flag is set in the entry so that we know to
just use the master entry flags. The IDs are always delta-encoded
against the first entry. This approach avoids cascading effects in which
entries are encoded depending on the previous entries, in order to avoid
complexity and rewritings of the data when data is removed in the middle
(which is a planned feature).
 2017-12-01 10:24:24 +01:00
antirez 8f00cf85a7 Streams: fixed memory leaks when blocking again for same stream. 
blockForKeys() was not freeing the allocation holding the ID when the
key was already found busy. Fortunately the unit test checked explicitly
for blocking multiple times for the same key (copying a regression in
the blocking lists tests), so the bug was detected by the Redis test leak
checker.
 2017-12-01 10:24:24 +01:00
antirez ae9065d808 Streams: tests for blocking and non-blocking XREAD. 2017-12-01 10:24:24 +01:00
antirez eb1230c999 Streams: XRANGE fuzz testing. 2017-12-01 10:24:24 +01:00
antirez fa707ca154 Streams: more advanced XADD and XRANGE tests. 2017-12-01 10:24:24 +01:00
antirez 7a41b402c1 Streams: basic XADD tests. 2017-12-01 10:24:24 +01:00
antirez 26d4f8e3ec Streams: AOF rewriting + minor iterator improvements. 2017-12-01 10:24:24 +01:00
antirez 01ea018c40 Streams: export iteration API. 2017-12-01 10:24:24 +01:00
antirez 9ed40f0fc3 Streams: implement streamReplyWithRange() in terms of the iterator. 2017-12-01 10:24:24 +01:00
antirez a58733cacf Streams: stream iteration refactoring, WIP 2. 2017-12-01 10:24:24 +01:00
antirez b1ec333633 Streams: stream iteration refactoring, WIP 1. 2017-12-01 10:24:24 +01:00
antirez 1a603e1a87 Streams: fix bug in XREAD last received ID processing. 2017-12-01 10:24:24 +01:00
antirez 94af55c5ea Streams: fix memory leak in freeStream(). 2017-12-01 10:24:24 +01:00
antirez 3a0b78bc52 Streams: rewrite XADD ID argument for AOF/slaves. 2017-12-01 10:24:24 +01:00
antirez 19b06935d5 Streams: fix XADD API and keyspace notifications. 
XADD was suboptimal in the first incarnation of the command, not being
able to accept an ID (very useufl for replication), nor options for
having capped streams.

The keyspace notification for streams was not implemented.
 2017-12-01 10:24:24 +01:00
antirez db89f7474d Streams: When XREAD blocks without COUNT, set a default one. 
A client may lose a lot of time between invocations of blocking XREAD,
for example because it is processing the messages or for any other
cause. When it returns back, it may provide a low enough message ID that
the server will block to send an unreasonable number of messages in a
single call. For this reason we set a COUNT when the client is blocked
with XREAD calls, even if no COUNT is given. This is arbitrarily set to
1000 because it's enough to avoid slowing down the reception of many
messages, but low enough to avoid to block.
 2017-12-01 10:24:24 +01:00
antirez c128190026 Streams: fix handleClientsBlockedOnKeys() access to invalid ID. 2017-12-01 10:24:24 +01:00
antirez 6468cb2e82 Streams: fix XREAD ready-key signaling. 
With lists we need to signal only on key creation, but streams can
provide data to clients listening at every new item added.
To make this slightly more efficient we now track different classes of
blocked clients to avoid signaling keys when there is nobody listening.
A typical case is when the stream is used as a time series DB and
accessed only by range with XRANGE.
 2017-12-01 10:24:24 +01:00
antirez b5be5093fe Streams: fix XREAD timeout handling, zero is valid. 2017-12-01 10:24:24 +01:00
antirez 2cacdcd6f8 Streams: XREAD related code to serve blocked clients. 2017-12-01 10:24:24 +01:00
antirez 0adb43b68f Streams: XREAD ability to block fixed. 2017-12-01 10:24:24 +01:00
antirez 6a1c92d52d Streams: synchronous xread fixes and improvements. 2017-12-01 10:24:24 +01:00
antirez a7d898334a Streams: XREAD get-key method fixed. 2017-12-01 10:24:24 +01:00
antirez 110041825c Streams: XREAD get-keys method. 2017-12-01 10:24:24 +01:00
antirez fa61720d30 Streams: XREAD, first draft. Handling of blocked clients still missing. 2017-12-01 10:24:24 +01:00
antirez e65b4825f0 Streams: XREAD arguments parsing. 2017-12-01 10:24:24 +01:00
antirez 4086dff477 Streams: augment client.bpop with XREAD specific fields. 2017-12-01 10:24:24 +01:00
antirez f80dfbf464 Streams: more internal preparation for blocking XREAD. 2017-12-01 10:24:24 +01:00
antirez 4a377cecd8 Streams: initial work to use blocking lists logic for streams XREAD. 2017-12-01 10:24:24 +01:00
antirez 439120c620 Streams: implement stream object release. 2017-12-01 10:24:24 +01:00
antirez ec9bbe96bf Streams: XLEN command. 2017-12-01 10:24:24 +01:00
antirez 98d184db12 Streams: Save stream->length in RDB. 2017-12-01 10:24:24 +01:00
antirez cd18f06e9c Streams: change listpack allocator to zmalloc. 2017-12-01 10:24:24 +01:00
antirez edd70c1993 Streams: RDB loading. RDB saving modified. 
After a few attempts it looked quite saner to just add the last item ID
at the end of the serialized listpacks, instead of scanning the last
listpack loaded from head to tail just to fetch it. It's a disk space VS
CPU-and-simplicity tradeoff basically.
 2017-12-01 10:24:24 +01:00
antirez 485014cc74 Streams: RDB saving. 2017-12-01 10:24:24 +01:00
antirez 100d43c1ac Streams: assign value of 6 to OBJ_STREAM + some refactoring. 2017-12-01 10:24:24 +01:00
antirez 79866a6361 Streams: 12 commits squashed into the initial Streams implementation. 2017-12-01 10:24:24 +01:00
antirez 045d65c3af PSYNC2: Fix off by one buffer size in luaCreateFunction(). 2017-11-30 18:38:29 +01:00
antirez 452ad2e928 PSYNC2: just store script bodies into RDB. 
Related to #4483. As suggested by @soloestoy, we can retrieve the SHA1
from the body. Given that in the new implementation using AUX fields we
ended copying around a lot to create new objects and strings, extremize
such concept and trade CPU for space inside the RDB file.
 2017-11-30 18:38:26 +01:00
antirez 28dfdca733 PSYNC2: luaCreateFunction() should handle NULL client parameter. 
See #4483. This is needed because luaCreateFunction() is now called
from RDB loading code outside a client context.
 2017-11-30 18:37:52 +01:00
antirez f11a7585a8 PSYNC2: Save Lua scripts state into RDB file. 
This is currently needed in order to fix #4483, but this can be
useful in other contexts, so maybe later we may want to remove the
conditionals and always save/load scripts.

Note that we are using the "lua" AUX field here, in order to guarantee
backward compatibility of the RDB file. The unknown AUX fields must be
discarded by past versions of Redis.
 2017-11-30 18:37:52 +01:00
antirez 6fb04d4637 Regression test: Slave restart with EVALSHA in backlog issue #4483. 2017-11-30 18:37:10 +01:00
antirez 3b9be93fda Prevent corruption of server.executable after DEBUG RESTART. 
Doing the following ended with a broken server.executable:

1. Start Redis with src/redis-server
2. Send CONFIG SET DIR /tmp/
3. Send DEBUG RESTART

At this point we called execve with an argv[0] that is no longer related
to the new path. So after the restart the absolute path of the
executable is recomputed in the wrong way. With this fix we pass the
absolute path already computed as argv[0].
 2017-11-30 18:30:06 +01:00
antirez d8f8701032 Be more verbose when DEBUG RESTART fails. 2017-11-30 18:08:21 +01:00
zhaozhao.zz 43be967690 networking: optimize unlinkClient() in freeClient() 2017-11-30 18:11:05 +08:00
zhaozhao.zz 1b5f56d042 aof: cast sdslen to ssize_t 2017-11-30 10:27:12 +08:00
zhaozhao.zz 2d73cf2367 aof: fix the short write 2017-11-30 10:22:12 +08:00
Itamar Haber 0752a834f9 Check arity in SLOWLOG before accessing arg 2017-11-30 00:30:30 +02:00
antirez 2785d6caa0 Merge branch 'lfu-fixes' into unstable 2017-11-29 17:16:13 +01:00
Itamar Haber 51d42c88a6 Merge branch 'unstable' into help_subcommands 2017-11-28 21:28:40 +02:00
Itamar Haber 59d52f7fab Standardizes the 'help' subcommand 
This adds a new `addReplyHelp` helper that's used by commands
when returning a help text. The following commands have been
touched: DEBUG, OBJECT, COMMAND, PUBSUB, SCRIPT and SLOWLOG.

WIP

Fix entry command table entry for OBJECT for HELP option.

After #4472 the command may have just 2 arguments.

Improve OBJECT HELP descriptions.

See #4472.

WIP 2

WIP 3
 2017-11-28 21:15:45 +02:00
Salvatore Sanfilippo 565e139a56 Merge pull request #4200 from jeesyn/fix_typo 
fix a typo
 2017-11-28 18:44:11 +01:00
Salvatore Sanfilippo b998e2035c Merge pull request #4165 from zyegfryed/patch-1 
Fix some typos
 2017-11-28 18:43:45 +01:00
Salvatore Sanfilippo 923502a70b Merge pull request #4166 from charpty/wip-redisclic-typo 
redis-cli.c typo: helpe -> helper.
 2017-11-28 18:41:51 +01:00
Salvatore Sanfilippo 26826329f5 Merge pull request #4167 from charpty/wip-redisclic-typo2 
redis-cli.c typo: Requets -> Requests.
 2017-11-28 18:41:28 +01:00
Salvatore Sanfilippo 3508b9c440 Merge pull request #4170 from TehWebby/patch-2 
Fix typo
 2017-11-28 18:40:43 +01:00
Salvatore Sanfilippo 4e55df4b5c Merge pull request #4172 from TehWebby/patch-3 
Fix typo
 2017-11-28 18:40:02 +01:00
Salvatore Sanfilippo ad171f5ce0 Merge pull request #4176 from KrauseFx/patch-1 
Update link to https and use inline link
 2017-11-28 18:39:43 +01:00
antirez 851e9fc48b t_hash.c: clarify calling two times the same function. 2017-11-28 18:39:00 +01:00
antirez aeee039a87 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2017-11-28 18:27:35 +01:00
antirez c44732ac58 adlist: fix listJoin() in the case the second list is empty. 
See #4192, the original PR removed lines of code that are actually
needed, so thanks to @chunqiulfq for reporting the problem, but merging
solution from @jeesyn after checking, together with @artix75, that the
logic covers all the cases.
 2017-11-28 18:25:14 +01:00
Salvatore Sanfilippo c3806f5b72 Merge pull request #4215 from lamby/correct-faield-spelling 
Correct spelling of "faield".
 2017-11-28 18:08:32 +01:00
Salvatore Sanfilippo a13106e001 Merge pull request #4374 from rouzier/patch-1 
Fix file descriptor leak and error handling
 2017-11-28 17:33:23 +01:00
Salvatore Sanfilippo bf71b120f1 Merge pull request #4451 from devnexen/minor_build_fixes 
Fix undefined behavior constant defined.
 2017-11-28 17:23:48 +01:00
Itamar Haber 8c7f90e91e Standardizes arity handling of DEBUG 2017-11-28 18:18:45 +02:00
antirez 06ca9d6839 LFU: Fix LFUDecrAndReturn() to just decrement. 
Splitting the popularity in half actually just needs decrementing the
counter because the counter is logarithmic.
 2017-11-28 12:18:30 +01:00
zhaozhao.zz 9f131c9a89 LFU: add hotkeys option to redis-cli 2017-11-27 18:39:29 +01:00
zhaozhao.zz 583c314725 LFU: do some changes about LFU to find hotkeys 
Firstly, use access time to replace the decreas time of LFU.
For function LFUDecrAndReturn,
it should only try to get decremented counter,
not update LFU fields, we will update it in an explicit way.
And we will times halve the counter according to the times of
elapsed time than server.lfu_decay_time.
Everytime a key is accessed, we should update the LFU
including update access time, and increment the counter after
call function LFUDecrAndReturn.
If a key is overwritten, the LFU should be also updated.
Then we can use `OBJECT freq` command to get a key's frequence,
and LFUDecrAndReturn should be called in `OBJECT freq` command
in case of the key has not been accessed for a long time,
because we update the access time only when the key is read or
overwritten.
 2017-11-27 18:39:22 +01:00
zhaozhao.zz 53cea97204 LFU: change lfu* parameters to int 2017-11-27 18:38:55 +01:00
zhaozhao.zz dfc42ec447 LFU: fix the missing of config get and rewrite 2017-11-27 18:38:33 +01:00
antirez 75fa7879e6 Improve OBJECT HELP descriptions. 
See #4472.
 2017-11-27 18:09:08 +01:00
antirez b412c544fd Fix entry command table entry for OBJECT for HELP option. 
After #4472 the command may have just 2 arguments.
 2017-11-27 13:16:07 +01:00
Salvatore Sanfilippo 29252391c4 Merge pull request #4472 from itamarhaber/object_patch 
A minor fix and `help` subcommand for `OBJECT`
 2017-11-27 12:41:02 +01:00
Itamar Haber 1c08220022 Adds -u <uri> option to redis-cli. 2017-11-27 11:34:11 +01:00
Itamar Haber 02d38f6b51 Adds OBJECT help 2017-11-24 19:59:05 +02:00
Itamar Haber b28fb3d753 Prevents OBJECT freq with noeviction 
When maxmemory is set to noeviction, idletime is implicitly kept. This renders access frequency nonsensical.
 2017-11-24 19:58:37 +02:00
antirez dc2df135b3 Test: regression test for latency expire events logging bug. 
Regression for #4452.
 2017-11-24 18:33:31 +01:00
Salvatore Sanfilippo c508cb6793 Merge pull request #4452 from soloestoy/expire-latency 
expire & latency: fix the missing latency records generated by expire
 2017-11-24 18:21:35 +01:00
antirez 7229fa8d6d Modules: fix memory leak in RM_IsModuleNameBusy(). 2017-11-24 13:29:54 +01:00
antirez 4d063bb6ba PSYNC2: reorganize comments related to recent fixes. 
Related to PR #4412 and issue #4407.
 2017-11-24 11:08:29 +01:00
Salvatore Sanfilippo 9d86ae4597 Merge pull request #4412 from soloestoy/bugfix-psync2 
PSYNC2: safe free backlog when reach the time limit and others
 2017-11-24 10:56:18 +01:00
Salvatore Sanfilippo f739c27229 Merge pull request #4344 from soloestoy/fix-module-name-conflict 
Fix module name conflict
 2017-11-24 09:37:06 +01:00
Salvatore Sanfilippo 58ac57f665 Merge pull request #4470 from oranagra/fix_string_to_double 
fix string to double conversion, stopped parsing on \0 …
 2017-11-24 08:59:23 +01:00
Oran Agra adf2701cc9 fix string to double conversion, stopped parsing on \0 even if the string has more data. 
getLongLongFromObject calls string2ll which has this line:
/* Return if not all bytes were used. */
so if you pass an sds with 3 characters "1\01" it will fail.

but getLongDoubleFromObject calls strtold, and considers it ok if eptr[0]==`\0`
i.e. if the end of the string found by strtold ends with null terminator

127.0.0.1:6379> set a 1
OK
127.0.0.1:6379> setrange a 2 2
(integer) 3
127.0.0.1:6379> get a
"1\x002"
127.0.0.1:6379> incrbyfloat a 2
"3"
127.0.0.1:6379> get a
"3"
 2017-11-23 17:15:27 +02:00
antirez de914ede93 Modules: fix for scripting replication of modules commands. 
See issue #4466 / #4467.
 2017-11-23 15:14:17 +01:00
Salvatore Sanfilippo ada206559d Merge pull request #4467 from yossigo/fix-nested-multi 
Nested MULTI/EXEC may replicate in different cases.
 2017-11-23 13:38:43 +01:00
Yossi Gottlieb 2c70d28295 Nested MULTI/EXEC may replicate in different cases. 
For example:
1. A module command called within a MULTI section.
2. A Lua script with replicate_commands() called within a MULTI section.
3. A module command called from a Lua script in the above context.
 2017-11-22 22:02:51 +02:00
zhaozhao.zz ea2e51c630 PSYNC2: persist cached_master's dbid inside the RDB 2017-11-22 12:11:26 +08:00
zhaozhao.zz 93037f7642 PSYNC2: make repl_stream_db never be -1 
it means that after this change all the replication
info in RDB is valid, and it can distinguish us from
the older version.
 2017-11-22 12:05:34 +08:00
zhaozhao.zz 7a808fd8a7 expire & latency: fix the missing latency records generated by expire 2017-11-21 23:35:30 +08:00
zhaozhao.zz 57bd8feb8d rehash: handle one db until finished 2017-11-21 09:49:42 +01:00
David Carlier 62689ef0cf Fix undefined behavior constant defined. 2017-11-19 16:23:42 +00:00
Salvatore Sanfilippo cf9a3f7048 Merge pull request #2741 from kmiku7/unstable 
fix boundary case for _dictNextPower
 2017-11-08 17:06:09 +01:00
Salvatore Sanfilippo 30db805548 Merge pull request #4431 from itamarhaber/patch-10 
Fixes an off-by-one in argument handling of `MEMORY USAGE`
 2017-11-08 15:42:45 +01:00
Itamar Haber 2564963dc8 Fixes an off-by-one in argument handling of MEMORY USAGE 
Fixes #4430
 2017-11-08 16:08:29 +02:00
antirez a1944c3e4d Fix saving of zero-length lists. 
Normally in modern Redis you can't create zero-len lists, however it's
possible to load them from old RDB files generated, for instance, using
Redis 2.8 (see issue #4409). The "Right Thing" would be not loading such
lists at all, but this requires to hook in rdb.c random places in a not
great way, for a problem that is at this point, at best, minor.

Here in this commit instead I just fix the fact that zero length lists,
materialized as quicklists with the first node set to NULL, were
iterated in the wrong way while they are saved, leading to a crash.

The other parts of the list implementation are apparently able to deal
with empty lists correctly, even if they are no longer a thing.
 2017-11-06 12:37:03 +01:00
antirez 34d5804d4c SDS: improve sdsRemoveFreeSpace() to avoid useless data copy. 
Since SDS v2, we no longer have a single header, so the function to
rewrite the SDS in terms of the minimum space required, instead of just
using realloc() and let the underlying allocator decide what to do,
was doing an allocation + copy every time the minimum possible header
needed to represent the string was different than the current one.
This could be often a bit wasteful, because if we go, for instance, from
the 32 bit fields header to the 16 bit fields header, the overhead of
the header is normally very small. With this commit we call realloc
instead, unless the change in header size is very significant in relation
to the string length.
 2017-11-03 10:19:27 +01:00
zhaozhao.zz b8579c225c PSYNC2: clarify the scenario when repl_stream_db can be -1 2017-11-02 10:45:33 +08:00
zhaozhao.zz 885c4f856e PSYNC2 & RDB: fix the missing rdbSaveInfo for BGSAVE 2017-11-01 17:52:43 +08:00
zhaozhao.zz 6ddf0ea293 PSYNC2: safe free backlog when reach the time limit 
When we free the backlog, we should use a new
replication ID and clear the ID2. Since without
backlog we can not increment master_repl_offset
even do write commands, that may lead to inconsistency
when we try to connect a "slave-before" master
(if this master is our slave before, our replid
equals the master's replid2). As the master have our
history, so we can match the master's replid2 and
second_replid_offset, that make partial sync work,
but the data is inconsistent.
 2017-11-01 17:32:27 +08:00
antirez ffcf7d5ab1 Fix buffer overflows occurring reading redis.conf. 
There was not enough sanity checking in the code loading the slots of
Redis Cluster from the nodes.conf file, this resulted into the
attacker's ability to write data at random addresses in the process
memory, by manipulating the index of the array. The bug seems
exploitable using the following techique: the config file may be altered so
that one of the nodes gets, as node ID (which is the first field inside the
structure) some data that is actually executable: then by writing this
address in selected places, this node ID part can be executed after a
jump. So it is mostly just a matter of effort in order to exploit the
bug. In practice however the issue is not very critical because the
bug requires an unprivileged user to be able to modify the Redis cluster
nodes configuration, and at the same time this should result in some
gain. However Redis normally is unprivileged as well. Yet much better to
have this fixed indeed.

Fix #4278.
 2017-10-31 09:41:22 +01:00
antirez b2e295971f Regression test for issue #4391. 2017-10-30 13:45:46 +01:00
antirez de474186bd More robust object -> double conversion. 
Certain checks were useless, at the same time certain malformed inputs
were accepted without problems (emtpy strings parsed as zero).
Cases where strtod() returns ERANGE but we still want to parse the input
where ok in getDoubleFromObject() but not in the long variant.

As a side effect of these fixes, this commit fixes #4391.
 2017-10-30 13:39:58 +01:00
rouzier 6eb996540c Fix file descriptor leak and error handling 2017-10-13 13:20:45 -04:00
antirez 2bf8c2c130 Limit statement in RM_BlockClient() to 80 cols. 2017-09-28 23:15:34 +02:00
Salvatore Sanfilippo 54ad626cce Merge pull request #4337 from dvirsky/module_ctx_flags 
Module Context flags
 2017-09-28 14:40:55 +02:00
zhaozhao.zz 6dffc1b7a3 Modules: handle the busy module name 2017-09-28 17:38:40 +08:00
zhaozhao.zz cb9dde3280 Modules: handle the conflict of registering commands 2017-09-28 16:21:21 +08:00
Dvir Volk 7393fd814e Added safety net preventing redis from crashing if a module decide to block in MULTI 2017-09-27 15:17:53 +03:00
Dvir Volk b246635d6d Renamed GetCtxFlags to GetContextFlags 2017-09-27 11:58:16 +03:00
Dvir Volk 616c546b01 Added support for module context flags with RM_GetCtxFlags 2017-09-27 11:58:07 +03:00
antirez 474adba9fa Clarify comment in change fixing #4323. 2017-09-21 12:35:04 +02:00
Salvatore Sanfilippo fa2c0582cf Merge pull request #4324 from soloestoy/lazyfree-fix-memory-leak 
Lazyfree: avoid memory leak when free slowlog entry
 2017-09-21 12:32:02 +02:00
zhaozhao.zz 269760edbb Lazyfree: avoid memory leak when free slowlog entry 2017-09-21 14:19:21 +08:00
antirez bb3b5ddd19 PSYNC2: More refinements related to #4316. 2017-09-20 11:28:13 +02:00
zhaozhao.zz b541ccef25 PSYNC2: make persisiting replication info more solid 
This commit is a reinforcement of commit c1c99e9.

1. Replication information can be stored when the RDB file is
generated by a mater using server.slaveseldb when server.repl_backlog
is not NULL, or set repl_stream_db be -1. That's safe, because
NULL server.repl_backlog will trigger full synchronization,
then master will send SELECT command to replicaiton stream.
2. Only do rdbSave* when rsiptr is not NULL,
if we do rdbSave* without rdbSaveInfo, slave will miss repl-stream-db.
3. Save the replication informations also in the case of
SAVE command, FLUSHALL command and DEBUG reload.
 2017-09-20 11:18:10 +02:00
antirez c1c99e9f4e PSYNC2: Fix the way replication info is saved/loaded from RDB. 
This commit attempts to fix a number of bugs reported in #4316.
They are related to the way replication info like replication ID,
offsets, and currently selected DB in the master client, are stored
and loaded by Redis. In order to avoid inconsistencies the changes in
this commit try to enforce that:

1. Replication information are only stored when the RDB file is
generated by a slave that has a valid 'master' client, so that we can
always extract the currently selected DB.
2. When replication informations are persisted in the RDB file, all the
info for a successful PSYNC or nothing is persisted.
3. The RDB replication informations are only loaded if the instance is
configured as a slave, otherwise a master can start with IDs that relate
to a different history of the data set, and stil retain such IDs in the
future while receiving unrelated writes.
 2017-09-19 23:03:39 +02:00
antirez a4152119c6 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2017-09-19 10:35:49 +02:00
antirez b75ae0bbea PSYNC2: Create backlog on slave partial sync as well. 
A slave may be started with an RDB file able to provide enough slave to
perform a successful partial SYNC with its master. However in such a
case, how outlined in issue #4268, the slave backlog will not be
started, since it was only initialized on full syncs attempts. This
creates different problems with successive PSYNC attempts that will
always result in full synchronizations.

Thanks to @fdingiit for discovering the issue.
 2017-09-19 10:33:14 +02:00
Salvatore Sanfilippo 2d13bf4c59 Merge pull request #3785 from GitHubMota/unstable 
redis-benchmark: default value size usage update.
 2017-09-18 12:18:57 +02:00
Salvatore Sanfilippo 9b4cb4addc Merge pull request #3554 from jybaek/Delete_duplicate 
Remove Duplicate Processing
 2017-09-18 12:18:15 +02:00
Salvatore Sanfilippo 8885921dfb Merge pull request #4311 from oranagra/aof_shutdown_flush 
Flush append only buffers before existing.
 2017-09-18 12:03:41 +02:00
Oran Agra b122cadc66 Flush append only buffers before existing. 
when SHUTDOWN command is recived it is possible that some of the recent
command were not yet flushed from the AOF buffer, and the server
experiences data loss at shutdown.
 2017-09-17 07:22:16 +03:00
jianqingdu 498f65ffb7 fix not call va_end when syncWrite() failed 
fix not call va_end when syncWrite() failed in sendSynchronousCommand()
 2017-08-30 21:20:14 -05:00
Richard Li 2ef8c2f6c2 Update the comment 2017-08-18 11:27:45 +08:00
Chris Lamb 6b9f02ac12 Correct spelling of "faield". 2017-08-12 22:21:03 -07:00
jeesyn.liu 447b373fc9 fix a typo 2017-08-08 17:45:51 +08:00
Salvatore Sanfilippo 202c2ebec4 Merge pull request #4191 from jybaek/unstable 
Add missing fclose()
 2017-08-03 12:18:58 +02:00
jybaek a8c08b9b76 Add missing fclose() 2017-08-03 17:28:04 +09:00
Salvatore Sanfilippo 34a79c353f Merge pull request #3935 from itamarhaber/module-cmdstats 
Changes command stats iteration to being dict-based
 2017-08-02 12:51:26 +02:00
Felix Krause 0c6ea46f2b Update link to https and use inline link 2017-07-28 13:04:52 -04:00
antirez bc64df9a66 Add MEMORY DOCTOR to MEMORY HELP. 2017-07-28 17:47:54 +02:00
Shaun Webb f06f10e66b Typo fix 2017-07-27 15:27:46 +09:00
Shaun Webb 2e6f285009 Fix typo 2017-07-27 09:37:37 +09:00
Bo Cai 00954f4d48 redis-cli.c typo: Requets -> Requests. 
Signed-off-by: Bo Cai <charpty@gmail.com>
 2017-07-26 21:33:29 +08:00
Bo Cai 005d9fa861 redis-cli.c typo: helpe -> helper. 
Signed-off-by: Bo Cai <charpty@gmail.com>
 2017-07-26 21:24:28 +08:00
Sébastien Fievet 22bcee5178 Fix some typos 2017-07-26 14:11:05 +02:00
Mota 81fe7a4733 redis-benchmark: default value size usage update. 
default size of SET/GET value in usage should be 3 bytes as in main code.
 2017-07-25 23:43:46 +08:00
Salvatore Sanfilippo dc997755bf Merge pull request #1934 from badboy/install-script-1922 
Don't use extended Regexp Syntax
 2017-07-24 15:20:31 +02:00
Salvatore Sanfilippo 6b64cc47a0 Merge pull request #2259 from badboy/fix-2258 
Check that the whole first argument is a number
 2017-07-24 15:19:53 +02:00
Salvatore Sanfilippo 964224b77f Merge pull request #4124 from lamby/proceding-proceeding-typo 
Correct proceding -> proceeding typo.
 2017-07-24 15:19:21 +02:00
Salvatore Sanfilippo ae40e5f362 Merge pull request #4125 from trevor211/fixAutoAofRewirteMinSize 
fix rewrite config: auto-aof-rewrite-min-size
 2017-07-24 15:18:56 +02:00
Salvatore Sanfilippo 25c231c4c1 Merge pull request #1998 from grobe0ba/unstable 
Fix missing '-' in redis-benchmark help output (Issue #1996)
 2017-07-24 15:18:08 +02:00
Salvatore Sanfilippo d9565379da Merge pull request #4128 from leonchen83/unstable 
fix mismatch argument and return wrong value of clusterDelNodeSlots
 2017-07-24 14:18:28 +02:00
Salvatore Sanfilippo 7c3c472b5e Merge pull request #4154 from liangsijian/unstable 
Fix lua ldb command log
 2017-07-24 14:11:10 +02:00
liangsijian ffbbe5a720 Fix lua ldb command log 2017-07-24 19:24:06 +08:00
antirez 314043552b Modules: don't crash when Lua calls a module blocking command. 
Lua scripting does not support calling blocking commands, however all
the native Redis commands are flagged as "s" (no scripting flag), so
this is not possible at all. With modules there is no such mechanism in
order to flag a command as non callable by the Lua scripting engine,
moreover we cannot trust the modules users from complying all the times:
it is likely that modules will be released to have blocking commands
without such commands being flagged correctly, even if we provide a way to
signal this fact.

This commit attempts to address the problem in a short term way, by
detecting that a module is trying to block in the context of the Lua
scripting engine client, and preventing to do this. The module will
actually believe to block as usually, but what happens is that the Lua
script receives an error immediately, and the background call is ignored
by the Redis engine (if not for the cleanup callbacks, once it
unblocks).

Long term, the more likely solution, is to introduce a new call called
RedisModule_GetClientFlags(), so that a command can detect if the caller
is a Lua script, and return an error, or avoid blocking at all.

Being the blocking API experimental right now, more work is needed in
this regard in order to reach a level well blocking module commands and
all the other Redis subsystems interact peacefully.

Now the effect is like the following:

    127.0.0.1:6379> eval "redis.call('hello.block',1,5000)" 0
    (error) ERR Error running script (call to
    f_b5ba35ff97bc1ef23debc4d6e9fd802da187ed53): @user_script:1: ERR
    Blocking module command called from Lua script

This commit fixes issue #4127 in the short term.
 2017-07-23 12:55:37 +02:00
antirez 5bfdfbe174 Fix typo in unblockClientFromModule() top comment. 2017-07-23 12:41:26 +02:00
antirez a3778f3b0f Make representClusterNodeFlags() more robust. 
This function failed when an internal-only flag was set as an only flag
in a node: the string was trimmed expecting a final comma before
exiting the function, causing a crash. See issue #4142.
Moreover generation of flags representation only needed at DEBUG log
level was always performed: a waste of CPU time. This is fixed as well
by this commit.
 2017-07-20 15:17:35 +02:00
antirez b1c2e1a19c Fix two bugs in moduleTypeLookupModuleByID(). 
The function cache was not working at all, and the function returned
wrong values if there where two or more modules exporting native data
types.

See issue #4131 for more details.
 2017-07-20 14:59:42 +02:00
Leon Chen 9e7a8c0207 fix return wrong value of clusterDelNodeSlots 2017-07-20 17:24:38 +08:00
Leon Chen 2cdf4cc656 fix mismatch argument 2017-07-18 02:28:24 -05:00
WuYunlong c32c690de6 fix rewrite config: auto-aof-rewrite-min-size 2017-07-15 10:20:56 +08:00
Chris Lamb 7560d347da Correct proceding -> proceeding typo. 2017-07-14 22:53:14 +01:00
antirez b80e467023 changelog.tcl: output 100 lines. 2017-07-14 13:04:37 +02:00
antirez bd1782fa0a Modules: fix thread safe context DB selection. 
Before this fix the DB currenty selected by the client blocked was not
respected and operations were always performed on DB 0.
 2017-07-14 13:02:15 +02:00
antirez 8eefc9323d Allow certain modules APIs only defining REDISMODULE_EXPERIMENTAL_API. 
Those calls may be subject to changes in the future, so the user should
acknowledge it is using non stable API.
 2017-07-14 12:07:52 +02:00
antirez f03947a676 Modules documentation removed from source. 
Moving to redis-doc repository to publish via Redis.io.
 2017-07-14 11:33:59 +02:00
antirez 43aaf96163 Markdown generation of Redis Modules API reference improved. 2017-07-14 11:29:31 +02:00
antirez e74f0aa6d1 Fix replication of SLAVEOF inside transaction. 
In Redis 4.0 replication, with the introduction of PSYNC2, masters and
slaves replicate commands to cascading slaves and to the replication
backlog itself in a different way compared to the past.

Masters actually replicate the effects of client commands.
Slaves just propagate what they receive from masters.

This mechanism can cause problems when the configuration of an instance
is changed from master to slave inside a transaction. For instance
we could send to a master instance the following sequence:

    MULTI
    SLAVEOF 127.0.0.1 0
    EXEC
    SLAVEOF NO ONE

Before the fixes in this commit, the MULTI command used to be propagated
into the replication backlog, however after the SLAVEOF command the
instance is a slave, so the EXEC implementation failed to also propagate
the EXEC command. When the slaves of the above instance reconnected,
they were incrementally synchronized just sending a "MULTI". This put
the master client (in the slaves) into MULTI state, breaking the
replication.

Notably even Redis Sentinel uses the above approach in order to guarantee
that configuration changes are always performed together with rewrites
of the configuration and with clients disconnection. Sentiel does:

    MULTI
    SLAVEOF ...
    CONFIG REWRITE
    CLIENT KILL TYPE normal
    EXEC

So this was a really problematic issue. However even with the fix in
this commit, that will add the final EXEC to the replication stream in
case the instance was switched from master to slave during the
transaction, the result would be to increment the slave replication
offset, so a successive reconnection with the new master, will not
permit a successful partial resynchronization: no way the new master can
provide us with the backlog needed, we incremented our offset to a value
that the new master cannot have.

However the EXEC implementation waits to emit the MULTI, so that if the
commands inside the transaction actually do not need to be replicated,
no commands propagation happens at all. From multi.c:

    if (!must_propagate && !(c->cmd->flags & (CMD_READONLY|CMD_ADMIN))) {
	execCommandPropagateMulti(c);
	must_propagate = 1;
    }

The above code is already modified by this commit you are reading.
Now also ADMIN commands do not trigger the emission of MULTI. It is actually
not clear why we do not just check for CMD_WRITE... Probably I wrote it this
way in order to make the code more reliable: better to over-emit MULTI
than not emitting it in time.

So this commit should indeed fix issue #3836 (verified), however it looks
like some reconsideration of this code path is needed in the long term.

BONUS POINT: The reverse bug.

Even in a read only slave "B", in a replication setup like:

	A -> B -> C

There are commands without the READONLY nor the ADMIN flag, that are also
not flagged as WRITE commands. An example is just the PING command.

So if we send B the following sequence:

    MULTI
    PING
    SLAVEOF NO ONE
    EXEC

The result will be the reverse bug, where only EXEC is emitted, but not the
previous MULTI. However this apparently does not create problems in practice
but it is yet another acknowledge of the fact some work is needed here
in order to make this code path less surprising.

Note that there are many different approaches we could follow. For instance
MULTI/EXEC blocks containing administrative commands may be allowed ONLY
if all the commands are administrative ones, otherwise they could be
denined. When allowed, the commands could simply never be replicated at all.
 2017-07-12 11:07:28 +02:00
antirez e1b8b4b6da CLUSTER GETKEYSINSLOT: avoid overallocating. 
Close #3911.
 2017-07-11 15:49:09 +02:00
antirez 5bd46d33db Fix isHLLObjectOrReply() to handle integer encoded strings. 
Close #3766.
 2017-07-11 12:44:59 +02:00
antirez e203a46cf3 Clients blocked in modules: free argv/argc later. 
See issue #3844 for more information.
 2017-07-11 12:33:01 +02:00
antirez 14c32c3569 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2017-07-11 09:46:58 +02:00
antirez 54e4bbeabd Event loop: call after sleep() only from top level. 
In general we do not want before/after sleep() callbacks to be called
when we re-enter the event loop, since those calls are only designed in
order to perform operations every main iteration of the event loop, and
re-entering is often just a way to incrementally serve clietns with
error messages or other auxiliary operations. However, if we call the
callbacks, we are then forced to think at before/after sleep callbacks
as re-entrant, which is much harder without any good need.

However here there was also a clear bug: beforeSleep() was actually
never called when re-entering the event loop. But the new afterSleep()
callback was. This is broken and in this instance re-entering
afterSleep() caused a modules GIL dead lock.
 2017-07-11 00:13:52 +02:00
Salvatore Sanfilippo 58104d8327 Merge pull request #4113 from guybe7/module_io_bytes 
Modules: Fix io->bytes calculation in RDB save
 2017-07-10 19:14:34 +02:00
antirez 11182a1a58 redis-check-aof: tell users there is a --fix option. 2017-07-10 16:41:25 +02:00
Guy Benoish dfb68cd235 Modules: Fix io->bytes calculation in RDB save 2017-07-10 14:41:57 +03:00
antirez fc7ecd8d35 AOF check utility: ability to check files with RDB preamble. 2017-07-10 13:38:23 +02:00
Salvatore Sanfilippo 6b0670daad Merge pull request #3853 from itamarhaber/issue-3851 
Sets up fake client to select current db in RM_Call()
 2017-07-06 15:02:11 +02:00
Salvatore Sanfilippo 38dd30af42 Merge pull request #4105 from spinlock/unstable-networking 
Optimize addReplyBulkSds for better performance
 2017-07-06 14:31:08 +02:00
Salvatore Sanfilippo 2d5aa00959 Merge pull request #4106 from petersunbag/unstable 
minor fix in listJoin().
 2017-07-06 14:29:37 +02:00
sunweinan 87f771bff1 minor fix in listJoin(). 2017-07-06 19:47:21 +08:00
antirez 2b36950e9b Free IO context if any in RDB loading code. 
Thanks to @oranagra for spotting this bug.
 2017-07-06 11:20:49 +02:00
antirez 51ffd062d3 Modules: DEBUG DIGEST interface. 2017-07-06 11:04:46 +02:00
spinlock 10db81af71 update Makefile for test-sds 2017-07-05 14:32:09 +00:00
spinlock ea31a4eae3 Optimize addReplyBulkSds for better performance 2017-07-05 14:25:05 +00:00
antirez f9fac7f777 Avoid closing invalid FDs to make Valgrind happier. 2017-07-05 15:40:25 +02:00
antirez 413c2bc180 Modules: no MULTI/EXEC for commands replicated from async contexts. 
They are technically like commands executed from external clients one
after the other, and do not constitute a single atomic entity.
 2017-07-05 10:10:20 +02:00
Salvatore Sanfilippo 09dd7b5ff0 Merge pull request #4101 from dvirsky/fix_modules_reply_len 
Proposed fix to #4100
 2017-07-04 12:01:51 +02:00
antirez eddd8d34c4 Add symmetrical assertion to track c->reply_buffer infinite growth. 
Redis clients need to have an instantaneous idea of the amount of memory
they are consuming (if the number is not exact should at least be
proportional to the actual memory usage). We do that adding and
subtracting the SDS length when pushing / popping from the client->reply
list. However it is quite simple to add bugs in such a setup, by not
taking the objects in the list and the count in sync. For such reason,
Redis has an assertion to track counts near 2^64: those are always the
result of the counter wrapping around because we subtract more than we
add. This commit adds the symmetrical assertion: when the list is empty
since we sent everything, the reply_bytes count should be zero. Thanks
to the new assertion it should be simple to also detect the other
problem, where the count slowly increases because of over-counting.
The assertion adds a conditional in the code that sends the buffer to
the socket but should not create any measurable performance slowdown,
listLength() just accesses a structure field, and this code path is
totally dominated by write(2).

Related to #4100.
 2017-07-04 11:55:05 +02:00
Dvir Volk 86e564e9ff fixed #4100 2017-07-04 00:02:19 +03:00
antirez b2cd9fcab6 Fix GEORADIUS edge case with huge radius. 
This commit closes issue #3698, at least for now, since the root cause
was not fixed: the bounding box function, for huge radiuses, does not
return a correct bounding box, there are points still within the radius
that are left outside.

So when using GEORADIUS queries with radiuses in the order of 5000 km or
more, it was possible to see, at the edge of the area, certain points
not correctly reported.

Because the bounding box for now was used just as an optimization, and
such huge radiuses are not common, for now the optimization is just
switched off when the radius is near such magnitude.

Three test cases found by the Continuous Integration test were added, so
that we can easily trigger the bug again, both for regression testing
and in order to properly fix it as some point in the future.
 2017-07-03 19:38:31 +02:00
Jun He c8ca71d40b Fixed stack trace generation on aarch64 
Change-Id: I9801239c98cb7362ed07e8b9ec2ba7e45749dba7
Signed-off-by: Jun He <jun.he@arm.com>
 2017-07-03 08:47:55 +00:00
antirez 26e638a8e9 redis-cli --latency: ability to run non interactively. 
This feature was proposed by @rosmo in PR #2643 and later redesigned
in order to fit better with the other options for non-interactive modes
of redis-cli. The idea is basically to allow to collect latency
information in scripts, cron jobs or whateever, just running for a
limited time and then producing a single output.
 2017-06-30 15:41:58 +02:00
antirez 7bad78bd2f Fix abort typo in Lua debugger help screen. 2017-06-30 12:12:00 +02:00
antirez f8547e53f0 Added GEORADIUS(BYMEMBER)_RO variants for read-only operations. 
Issue #4084 shows how for a design error, GEORADIUS is a write command
because of the STORE option. Because of this it does not work
on readonly slaves, gets redirected to masters in Redis Cluster even
when the connection is in READONLY mode and so forth.

To break backward compatibility at this stage, with Redis 4.0 to be in
advanced RC state, is problematic for the user base. The API can be
fixed into the unstable branch soon if we'll decide to do so in order to
be more consistent, and reease Redis 5.0 with this incompatibility in
the future. This is still unclear.

However, the ability to scale GEO queries in slaves easily is too
important so this commit adds two read-only variants to the GEORADIUS
and GEORADIUSBYMEMBER command: GEORADIUS_RO and GEORADIUSBYMEMBER_RO.
The commands are exactly as the original commands, but they do not
accept the STORE and STOREDIST options.
 2017-06-30 10:03:37 +02:00
antirez 01a4b9892d HMSET and MSET implementations unified. HSET now variadic. 
This is the first step towards getting rid of HMSET which is a command
that does not make much sense once HSET is variadic, and has a saner
return value.
 2017-06-29 17:38:46 +02:00
Salvatore Sanfilippo 634c64dd18 Merge pull request #4075 from sgn1/brpop_keys 
Fix Issues in blocking commands in cluster mode.
 2017-06-27 17:51:19 +02:00
antirez 365dd037dc RDB modules values serialization format version 2. 
The original RDB serialization format was not parsable without the
module loaded, becuase the structure was managed only by the module
itself. Moreover RDB is a streaming protocol in the sense that it is
both produce di an append-only fashion, and is also sometimes directly
sent to the socket (in the case of diskless replication).

The fact that modules values cannot be parsed without the relevant
module loaded is a problem in many ways: RDB checking tools must have
loaded modules even for doing things not involving the value at all,
like splitting an RDB into N RDBs by key or alike, or just checking the
RDB for sanity.

In theory module values could be just a blob of data with a prefixed
length in order for us to be able to skip it. However prefixing the values
with a length would mean one of the following:

1. To be able to write some data at a previous offset. This breaks
stremaing.
2. To bufferize values before outputting them. This breaks performances.
3. To have some chunked RDB output format. This breaks simplicity.

Moreover, the above solution, still makes module values a totally opaque
matter, with the fowllowing problems:

1. The RDB check tool can just skip the value without being able to at
least check the general structure. For datasets composed mostly of
modules values this means to just check the outer level of the RDB not
actually doing any checko on most of the data itself.
2. It is not possible to do any recovering or processing of data for which a
module no longer exists in the future, or is unknown.

So this commit implements a different solution. The modules RDB
serialization API is composed if well defined calls to store integers,
floats, doubles or strings. After this commit, the parts generated by
the module API have a one-byte prefix for each of the above emitted
parts, and there is a final EOF byte as well. So even if we don't know
exactly how to interpret a module value, we can always parse it at an
high level, check the overall structure, understand the types used to
store the information, and easily skip the whole value.

The change is backward compatible: older RDB files can be still loaded
since the new encoding has a new RDB type: MODULE_2 (of value 7).
The commit also implements the ability to check RDB files for sanity
taking advantage of the new feature.
 2017-06-27 13:19:16 +02:00
antirez c3998728a2 ARM: Fix stack trace generation on crash. 2017-06-26 10:36:16 +02:00
antirez c9097393bf Issue #4027: unify comment and modify return value in freeMemoryIfNeeded(). 
It looks safer to return C_OK from freeMemoryIfNeeded() when clients are
paused because returning C_ERR may prevent success of writes. It is
possible that there is no difference in practice since clients cannot
execute writes while clients are paused, but it looks more correct this
way, at least conceptually.

Related to PR #4028.
 2017-06-23 11:42:25 +02:00
Salvatore Sanfilippo 936ade80b2 Merge pull request #4028 from zintrepid/prevent_expirations_while_paused 
Prevent expirations and evictions while paused
 2017-06-23 11:39:02 +02:00
Suraj Narkhede f85f36f50d Fix following issues in blocking commands: 
1. brpop last key index, thus checking all keys for slots.
2. Memory leak in clusterRedirectBlockedClientIfNeeded.
3. Remove while loop in clusterRedirectBlockedClientIfNeeded.
 2017-06-23 00:30:21 -07:00
Suraj Narkhede d303bca587 Fix brpop command table entry and redirect blocked clients. 2017-06-22 23:52:00 -07:00
antirez 8b768e8ea4 Aesthetic changes to #4068 PR to conform to Redis coding standard. 
1. Inline if ... statement if short.
2. No lines over 80 columns.
 2017-06-22 11:00:34 +02:00
Salvatore Sanfilippo 6476f1a979 Merge pull request #4068 from FreedomU007/unstable 
Fix set with ex/px option when propagated to aof
 2017-06-22 10:46:58 +02:00
xuzhou 86e9f48a0c Optimize set command with ex/px when updating aof. 2017-06-22 11:06:40 +08:00
Salvatore Sanfilippo ef446bf16d Merge pull request #3802 from flowly/bugfix-calc-stat-net-output-bytes 
Bugfix calc stat net output bytes
 2017-06-20 17:01:16 +02:00
Salvatore Sanfilippo 1d857a99d5 Merge pull request #4056 from season89/unstable 
Fixed comments of slowlog duration
 2017-06-20 16:55:29 +02:00
Salvatore Sanfilippo 0a03187ac4 Merge pull request #3659 from cbgbt/cli-elapsed 
cli: Only print elapsed time on OUTPUT_STANDARD.
 2017-06-20 16:53:56 +02:00
Salvatore Sanfilippo a2146c7592 Merge pull request #4062 from concreted/patch-1 
(fix) Update create-cluster README
 2017-06-20 16:41:10 +02:00
antirez 2a84927f35 redis-benchmark: add -t hset target. 2017-06-19 09:41:11 +02:00
Aric Huang ee5044eef0 (fix) Update create-cluster README 
Fix a few typos/adjust wording in `create-cluster` README
 2017-06-16 16:10:00 -07:00
xuzhou 530fcf8687 Fix set with ex/px option when propagated to aof 2017-06-16 17:51:38 +08:00
antirez 53cb27b1d7 SLOWLOG: log offending client address and name. 2017-06-15 12:57:54 +02:00
antirez ab9d398835 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2017-06-14 18:29:53 +02:00
Qu Chen 4740424049 Implement getKeys procedure for georadius and georadiusbymember 
commands.
 2017-06-14 18:15:48 +02:00
xuchengxuan 3fc4bf07cc Fixed comments of slowlog duration 2017-06-14 16:42:21 +08:00
Salvatore Sanfilippo d3b32ca48d Merge pull request #4034 from amallia/patch-1 
Fixed comment in clusterMsg version field
 2017-06-13 06:28:23 -07:00
Salvatore Sanfilippo 33035cad04 Merge pull request #4035 from amallia/patch-2 
Removed duplicate 'sys/socket.h'  include
 2017-06-13 06:27:31 -07:00
antirez 5877c02c51 Fix PERSIST expired key resuscitation issue #4048. 2017-06-13 10:35:51 +02:00
Antonio Mallia 2d1d57eb47 Removed duplicate 'sys/socket.h' include 2017-06-04 15:26:53 +01:00
Antonio Mallia 591dba8055 Fixed comment in clusterMsg version field 2017-06-04 15:09:05 +01:00
Zachary Marquez a3e53cf9bc Prevent expirations and evictions while paused 
Proposed fix to https://github.com/antirez/redis/issues/4027
 2017-06-01 16:28:40 -05:00
antirez e91b81c612 More informative -MISCONF error message. 2017-05-19 12:03:30 +02:00
antirez e498d9ee3e Collect fork() timing info only if fork succeeded. 2017-05-19 11:10:36 +02:00
antirez 78211aaaaf redis-cli --bigkeys: show error when TYPE fails. 
Close #3993.
 2017-05-15 11:22:28 +02:00
antirez 58f79e2ff4 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2017-05-10 13:15:30 +02:00
antirez 1f598fc2bb Modules TSC: use atomic var for server.unixtime. 
This avoids Helgrind complaining, but we are actually not using
atomicGet() to get the unixtime value for now: too many places where it
is used and given tha time_t is word-sized it should be safe in all the
archs we support as it is.

On the other hand, Helgrind, when Redis is compiled with "make helgrind"
in order to force the __sync macros, will detect the write in
updateCachedTime() as a read (because atomic functions are used) and
will not complain about races.

This commit also includes minor refactoring of mutex initializations and
a "helgrind" target in the Makefile.
 2017-05-10 10:04:16 +02:00
antirez de786186a5 atomicvar.h: show used API in INFO. Add macro to force __sync builtin. 
The __sync builtin can be correctly detected by Helgrind so to force it
is useful for testing. The API in the INFO output can be useful for
debugging after problems are reported.
 2017-05-10 09:33:49 +02:00
Guy Benoish 89a9e5a9a2 Merge branch 'unstable' of https://github.com/antirez/redis into unstable 2017-05-09 18:42:32 +03:00
antirez 6eb51bf1ec zmalloc.c: remove thread safe mode, it's the default way. 2017-05-09 16:59:51 +02:00
antirez 9390c384b8 Modules TSC: Add mutex for server.lruclock. 
Only useful for when no atomic builtins are available.
 2017-05-09 16:32:49 +02:00
antirez ece658713b Modules TSC: Improve inter-thread synchronization. 
More work to do with server.unixtime and similar. Need to write Helgrind
suppression file in order to suppress the valse positives.
 2017-05-09 11:57:09 +02:00
antirez 2a51bac44e Simplify atomicvar.h usage by having the mutex name implicit. 2017-05-04 17:01:00 +02:00
antirez 52bc74f221 Lazyfree: fix lazyfreeGetPendingObjectsCount() race reading counter. 2017-05-04 10:35:40 +02:00
antirez 7d9326b1f3 Modules TSC: HELLO.KEYS reply format fixed. 2017-05-03 23:43:49 +02:00
antirez 9b01b64430 Modules TSC: put the client in the pending write list. 2017-05-03 14:54:48 +02:00
antirez e67fb915eb adlist: fix final list count in listJoin(). 2017-05-03 14:54:14 +02:00
antirez 79226cb9fa adlist: fix listJoin() to handle empty lists. 2017-05-03 14:15:25 +02:00
antirez 6798736909 Modules: remove unused var in example module. 2017-05-03 14:10:21 +02:00
antirez 1ed2ff5570 Modules TSC: HELLO.KEYS example draft finished. 2017-05-03 14:08:12 +02:00
antirez 7127f15ebe Module: fix RedisModule_Call() "l" specifier to create a raw string. 2017-05-03 14:07:10 +02:00
antirez 3fcf959e60 Modules TSC: Release the GIL for all the time we are blocked. 
Instead of giving the module background operations just a small time to
run in the beforeSleep() function, we can have the lock released for all
the time we are blocked in the multiplexing syscall.
 2017-05-03 11:26:21 +02:00
antirez ba4a5a3255 Modules TSC: Export symbols of the new API. 2017-05-02 15:19:28 +02:00
antirez 275905b328 Modules TSC: Handling of RM_Reply* functions. 2017-05-02 15:05:39 +02:00
antirez 9c500b89fb Modules TSC: Basic TS context creeation and handling. 2017-05-02 12:53:10 +02:00
antirez 59b06b14c9 Modules TSC: GIL and cooperative multi tasking setup. 2017-04-28 18:41:10 +02:00
antirez a4c7f34d3a Regression test for #3899 fixed. 2017-04-28 11:16:39 +02:00
antirez c180bc7d98 Regression test for PSYNC2 issue #3899 added. 
Experimentally verified that it can trigger the issue reverting the fix.
At least on my system... Being the bug time/backlog dependant, it is
very hard to tell if this test will be able to trigger the problem
consistently, however even if it triggers the problem once in a while,
we'll see it in the CI environment at http://ci.redis.io.
 2017-04-28 10:37:07 +02:00
antirez 469d6e2b37 PSYNC2: fix master cleanup when caching it. 
The master client cleanup was incomplete: resetClient() was missing and
the output buffer of the client was not reset, so pending commands
related to the previous connection could be still sent.

The first problem caused the client argument vector to be, at times,
half populated, so that when the correct replication stream arrived the
protcol got mixed to the arugments creating invalid commands that nobody
called.

Thanks to @yangsiran for also investigating this problem, after
already providing important design / implementation hints for the
original PSYNC2 issues (see referenced Github issue).

Note that this commit adds a new function to the list library of Redis
in order to be able to reset a list without destroying it.

Related to issue #3899.
 2017-04-27 17:08:37 +02:00
antirez c861e1e1ee Defrag: test currently disabled, too many false positives. 
Related to #3786.
 2017-04-22 15:59:57 +02:00
antirez a17390853d Defrag: fix test false positive. 
Apparently 1.4 is too low compared to what you get in certain setups
(including mine). I raised it to 1.55 that hopefully is still enough to
test that the fragmentation went down from 1.7 but without incurring in
issues, however the test setup may be still fragile so certain times this
may lead to false positives again, it's hard to test for these things
in a determinsitic way.

Related to #3786.
 2017-04-22 13:21:41 +02:00
oranagra 0fb5c4ebd8 add test for active defrag 2017-04-22 13:17:09 +02:00
antirez e3b8492e83 Revert "Jemalloc updated to 4.4.0." 
This reverts commit 36c1acc222.
 2017-04-22 13:17:07 +02:00
antirez 238cebdd5e Check event loop creation return value. Fix #3951. 
Normally we never check for OOM conditions inside Redis since the
allocator will always return a pointer or abort the program on OOM
conditons. However we cannot have control on epool_create(), that may
fail for kernel OOM (according to the manual page) even if all the
parameters are correct, so the function aeCreateEventLoop() may indeed
return NULL and this condition must be checked.
 2017-04-21 16:27:38 +02:00
Salvatore Sanfilippo 3773c06d28 Merge pull request #3950 from kensou97/unstable 
update block->free after some diff data are written to the child process
 2017-04-20 07:55:51 +02:00
antirez 7d9dd80db3 Fix getKeysUsingCommandTable() in cluster mode. 
Close #3940.
 2017-04-19 16:17:08 +02:00
antirez 189a12afb4 PSYNC2: discard pending transactions from cached master. 
During the review of the fix for #3899, @yangsiran identified an
implementation bug: given that the offset is now relative to the applied
part of the replication log, when we cache a master, the successive
PSYNC2 request will be made in order to *include* the transaction that
was not completely processed. This means that we need to discard any
pending transaction from our replication buffer: it will be re-executed.
 2017-04-19 14:02:52 +02:00
antirez 22be435efe Fix PSYNC2 incomplete command bug as described in #3899. 
This bug was discovered by @kevinmcgehee and constituted a major hidden
bug in the PSYNC2 implementation, caused by the propagation from the
master of incomplete commands to slaves.

The bug had several results:

1. Borrowing from Kevin text in the issue: "Given that slaves blindly
copy over their master's input into their own replication backlog over
successive read syscalls, it's possible that with large commands or
small TCP buffers, partial commands are present in this buffer. If the
master were to fail before successfully propagating the entire command
to a slave, the slaves will never execute the partial command (since the
client is invalidated) but will copy it to replication backlog which may
relay those invalid bytes to its slaves on PSYNC2, corrupting the
backlog and possibly other valid commands that follow the failover.
Simple command boundaries aren't sufficient to capture this, either,
because in the case of a MULTI/EXEC block, if the master successfully
propagates a subset of the commands but not the EXEC, then the
transaction in the backlog becomes corrupt and could corrupt other
slaves that consume this data."

2. As identified by @yangsiran later, there is another effect of the
bug. For the same mechanism of the first problem, a slave having another
slave, could receive a full resynchronization request with an already
half-applied command in the backlog. Once the RDB is ready, it will be
sent to the slave, and the replication will continue sending to the
sub-slave the other half of the command, which is not valid.

The fix, designed by @yangsiran and @antirez, and implemented by
@antirez, uses a secondary buffer in order to feed the sub-masters and
update the replication backlog and offsets, only when a given part of
the query buffer is actually *applied* to the state of the instance,
that is, when the command gets processed and the command is not pending
in the Redis transaction buffer because of CLIENT_MULTI state.

Given that now the backlog and offsets representation are in agreement
with the actual processed commands, both issue 1 and 2 should no longer
be possible.

Thanks to @kevinmcgehee, @yangsiran and @oranagra for their work in
identifying and designing a fix for this problem.
 2017-04-19 10:25:45 +02:00
Salvatore Sanfilippo 27fe8e9fb2 Merge pull request #3945 from badboy/dicthash-bench-compile 
Reorder to make dict-benchmark compile on Linux
 2017-04-18 16:31:18 +02:00
antirez 02d02a3754 Fix #3848 by closing the descriptor on error. 2017-04-18 16:24:06 +02:00
antirez 8b7b4d6734 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2017-04-18 16:15:24 +02:00
antirez da2f9cd186 Fix descriptor leak. Close #3848. 2017-04-18 16:15:16 +02:00
Salvatore Sanfilippo 332a05dc33 Merge pull request #3856 from viennadd/issue-3847 
fix #3847: add close socket before return ANET_ERR.
 2017-04-18 16:13:23 +02:00
张文康 5f88bd320e update block->free after some diff data are written to the child process 2017-04-18 20:10:08 +08:00
antirez c33493277a Clarify why we save ziplist elements in revserse order. 
Also get rid of variables that are now kinda redundant, since the
dictionary iterator was removed.

This is related to PR #3949.
 2017-04-18 11:01:47 +02:00
Salvatore Sanfilippo 0a942f1751 Merge pull request #3949 from spinlock/unstable-rdb-encoding 
rdb: saving skiplist in reversed order to accelerate the deserialisation process
 2017-04-18 10:56:57 +02:00
Jan-Erik Rediger c4ad4765b0 Reorder to make dict-benchmark compile on Linux 
Fixes #3944
 2017-04-17 13:37:59 +02:00
spinlock 23ec36909e rdb: saving skiplist in reversed order to accelerate the deserialisation process 2017-04-17 13:22:34 +08:00
antirez 271733f4f8 Cluster: discard pong times in the future. 
However we allow for 500 milliseconds of tolerance, in order to
avoid often discarding semantically valid info (the node is up)
because of natural few milliseconds desync among servers even when
NTP is used.

Note that anyway we should ping the node from time to time regardless and
discover if it's actually down from our point of view, since no update
is accepted while we have an active ping on the node.

Related to #3929.
 2017-04-15 10:12:08 +02:00
antirez 3f068b92b9 Test: fix, hopefully, false PSYNC failure like in issue #2715. 
And many other related Github issues... all reporting the same problem.
There was probably just not enough backlog in certain unlucky runs.
I'll ask people that can reporduce if they see now this as fixed as
well.
 2017-04-14 17:53:11 +02:00
antirez 02777bb252 Cluster: always add PFAIL nodes at end of gossip section. 
To rely on the fact that nodes in PFAIL state will be shared around by
randomly adding them in the gossip section is a weak assumption,
especially after changes related to sending less ping/pong packets.

We want to always include gossip entries for all the nodes that are in
PFAIL state, so that the PFAIL -> FAIL state promotion can happen much
faster and reliably.

Related to #3929.
 2017-04-14 13:39:49 +02:00
antirez 8c829d9e43 Cluster: fix gossip section ping/pong times encoding. 
The gossip section times are 32 bit, so cannot store the milliseconds
time but just the seconds approximation, which is good enough for our
uses. At the same time however, when comparing the gossip section times
of other nodes with our node's view, we need to convert back to
milliseconds.

Related to #3929. Without this change the patch to reduce the traffic in
the bus message does not work.
 2017-04-14 11:01:22 +02:00
antirez 6878a3fedd Cluster: add clean-logs command to create-cluster script. 2017-04-14 10:52:00 +02:00
antirez 8f7bf2841a Cluster: decrease ping/pong traffic by trusting other nodes reports. 
Cluster of bigger sizes tend to have a lot of traffic in the cluster bus
just for failure detection: a node will try to get a ping reply from
another node no longer than when the half the node timeout would elapsed,
in order to avoid a false positive.

However this means that if we have N nodes and the node timeout is set
to, for instance M seconds, we'll have to ping N nodes every M/2
seconds. This N*M/2 pings will receive the same number of pongs, so
a total of N*M packets per node. However given that we have a total of N
nodes doing this, the total number of messages will be N*N*M.

In a 100 nodes cluster with a timeout of 60 seconds, this translates
to a total of 100*100*30 packets per second, summing all the packets
exchanged by all the nodes.

This is, as you can guess, a lot... So this patch changes the
implementation in a very simple way in order to trust the reports of
other nodes: if a node A reports a node B as alive at least up to
a given time, we update our view accordingly.

The problem with this approach is that it could result into a subset of
nodes being able to reach a given node X, and preventing others from
detecting that is actually not reachable from the majority of nodes.
So the above algorithm is refined by trusting other nodes only if we do
not have currently a ping pending for the node X, and if there are no
failure reports for that node.

Since each node, anyway, pings 10 other nodes every second (one node
every 100 milliseconds), anyway eventually even trusting the other nodes
reports, we will detect if a given node is down from our POV.

Now to understand the number of packets that the cluster would exchange
for failure detection with the patch, we can start considering the
random PINGs that the cluster sent anyway as base line:
Each node sends 10 packets per second, so the total traffic if no
additioal packets would be sent, including PONG packets, would be:

    Total messages per second = N*10*2

However by trusting other nodes gossip sections will not AWALYS prevent
pinging nodes for the "half timeout reached" rule all the times. The
math involved in computing the actual rate as N and M change is quite
complex and depends also on another parameter, which is the number of
entries in the gossip section of PING and PONG packets. However it is
possible to compare what happens in cluster of different sizes
experimentally. After applying this patch a very important reduction in
the number of packets exchanged is trivial to observe, without apparent
impacts on the failure detection performances.

Actual numbers with different cluster sizes should be published in the
Reids Cluster documentation in the future.

Related to #3929.
 2017-04-14 10:43:53 +02:00
antirez c5d6f577f0 Cluster: collect more specific bus messages stats. 
First step in order to change Cluster in order to use less messages.
Related to issue #3929.
 2017-04-13 19:22:35 +02:00
Itamar Haber b8286d1fc9 Changes command stats iteration to being dict-based 
With the addition of modules, looping over the redisCommandTable
misses any added commands. By moving to dictionary iteration this
is resolved.
 2017-04-13 17:03:46 +03:00
antirez 104584b95e Fix typo in feedReplicationBacklog() top comment. 2017-04-12 12:28:05 +02:00
antirez 1210af3804 Add a top comment in crucial functions inside networking.c. 2017-04-12 10:12:27 +02:00
antirez 4a850be4dc Set lua-time-limit default value at safe place. 
Otherwise, as it was, it will overwrite whatever the user set.

Close #3703.
 2017-04-11 16:56:00 +02:00
antirez f47607af02 Fix preprocessor if/else chain broken in order to fix #3927. 2017-04-11 16:54:27 +02:00
antirez 74720ea993 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2017-04-11 16:45:49 +02:00
antirez aa5b4be02e Fix zmalloc_get_memory_size() ifdefs to actually use the else branch. 
Close #3927.
 2017-04-11 16:45:11 +02:00
Salvatore Sanfilippo 69ce5c5d10 Merge pull request #3924 from lorneli/unstable 
Expire: Update comment of activeExpireCycle function
 2017-04-11 16:31:55 +02:00
antirez 531647bb1b Make more obvious why there was issue #3843. 2017-04-10 13:17:05 +02:00
Salvatore Sanfilippo 01b6966afc Merge pull request #3843 from dvirsky/fix_bc_free 
fixed free of blocked client before refering to it
 2017-04-10 13:14:52 +02:00
antirez ffefc9f92d Fix modules blocking commands awake delay. 
If a thread unblocks a client blocked in a module command, by using the
RedisMdoule_UnblockClient() API, the event loop may not be awaken until
the next timeout of the multiplexing API or the next unrelated I/O
operation on other clients. We actually want the client to be served
ASAP, so a mechanism is needed in order for the unblocking API to inform
Redis that there is a client to serve ASAP.

This commit fixes the issue using the old trick of the pipe: when a
client needs to be unblocked, a byte is written in a pipe. When we run
the list of clients blocked in modules, we consume all the bytes
written in the pipe. Writes and reads are performed inside the context
of the mutex, so no race is possible in which we consume the bytes that
are actually related to an awake request for a client that should still
be put into the list of clients to unblock.

It was verified that after the fix the server handles the blocked
clients with the expected short delay.

Thanks to @dvirsky for understanding there was such a problem and
reporting it.
 2017-04-10 09:33:21 +02:00
antirez 91999fce40 Rax library updated. 
Important bugs fixed.
 2017-04-08 17:31:13 +02:00
lorneli 98db5739cc Expire: Update comment of activeExpireCycle function 
The macro REDIS_EXPIRELOOKUPS_TIME_PERC has been replaced by
ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC in commit
6500fabfb8.
 2017-04-08 15:15:24 +08:00
antirez 3f9e2322ec Rax library updated. 2017-04-07 08:46:39 +02:00
antirez 1409c545da Cluster: hash slots tracking using a radix tree. 2017-03-27 16:37:22 +02:00
Salvatore Sanfilippo 94751543b0 Merge pull request #3875 from oranagra/lfu_tests 
add LFU policies to the test suite, just for coverage
 2017-03-15 09:18:04 +01:00
Oran Agra 4acb4da1d1 add LFU policies to the test suite, just for coverage 2017-03-15 01:05:15 -07:00
antirez a62f786344 Use sha256 instead of sha1 to generate tarball hashes. 2017-03-09 13:49:36 +01:00
vienna 59bdd08214 fix #3847: add close socket before return ANET_ERR. 2017-03-07 16:14:05 +00:00
itamar 443f279a3a Sets up fake client to select current db in RM_Call() 2017-03-06 14:37:10 +02:00
Guy Benoish 71a8df6a2b Merge branch 'unstable' of https://github.com/antirez/redis into unstable 2017-03-02 13:25:05 +02:00
Dvir Volk 4b2229e4b8 fixed free of blocked client before refering to it 2017-03-01 16:51:01 +02:00
Salvatore Sanfilippo 9cc83d2ad9 Makefile: fix building with Solaris C compiler, 64 bit. 2017-02-23 16:53:39 +01:00
antirez ed7e331051 Merge branch 'sparc' of ssh://209.141.57.197:12222//export/home/antirez/redis into sparc 2017-02-23 15:35:01 +01:00
Salvatore Sanfilippo b3391fd853 Use ARM unaligned accesses ifdefs for SPARC as well. 2017-02-23 22:39:44 +08:00
Salvatore Sanfilippo d7826823c0 Fix BITPOS unaligned memory access. 2017-02-23 22:38:44 +08:00
oranagra 161a3a174b when a slave experiances an error on commands that come from master, print to the log 
since slave isn't replying to it's master, these errors go unnoticed.
since we don't expect the master to send garbadge to the slave, this should be safe.
(as long as we don't log OOM errors there)
 2017-02-23 03:44:42 -08:00
oranagra f86df924b0 add SDS_NOINIT option to sdsnewlen to avoid unnecessary memsets. 
this commit also contains small bugfix in rdbLoadLzfStringObject
a bug that currently has no implications.
 2017-02-23 03:04:08 -08:00
antirez 95883313b5 Solaris fixes about tail usage and atomic vars. 
Testing with Solaris C compiler (SunOS 5.11 11.2 sun4v sparc sun4v)
there were issues compiling due to atomicvar.h and running the
tests also failed because of "tail" usage not conform with Solaris
tail implementation. This commit fixes both the issues.
 2017-02-22 13:08:21 +01:00
antirez 2b36706a48 Test: replication-psync, wait more to detect write load. 
Slow systems like the original Raspberry PI need more time
than 5 seconds to start the script and detect writes.
After fixing the Raspberry PI can pass the unit without issues.
 2017-02-22 12:27:01 +01:00
antirez 7c8ddab4f8 Test: fix conditional execution of HINCRBYFLOAT representation test. 2017-02-22 12:00:09 +01:00
antirez 06263485d4 Merge branch 'siphash' into unstable 2017-02-21 17:10:10 +01:00
antirez e084b5a39f Merge branch 'arm' into unstable 2017-02-21 17:10:06 +01:00
antirez 0285c2714b SipHash 2-4 -> SipHash 1-2. 
For performance reasons we use a reduced rounds variant of
SipHash. This should still provide enough protection and the
effects in the hash table distribution are non existing.
If some real world attack on SipHash 1-2 will be found we can
trivially switch to something more secure. Anyway it is a
big step forward from Murmurhash, for which it is trivial to
generate *seed independent* colliding keys... The speed
penatly introduced by SipHash 2-4, around 4%, was a too big
price to pay compared to the effectiveness of the HashDoS
attack against SipHash 1-2, and considering so far in the
Redis history, no such an incident ever happened even while
using trivially to collide hash functions.
 2017-02-21 17:07:28 +01:00
antirez cd90389b30 freeMemoryIfNeeded(): improve code and lazyfree handling. 
1. Refactor memory overhead computation into a function.
2. Every 10 keys evicted, check if memory usage already reached
   the target value directly, since we otherwise don't count all
   the memory reclaimed by the background thread right now.
 2017-02-21 12:55:59 +01:00
antirez 84fa8230e5 Use locale agnostic tolower() in dict.c hash function. 2017-02-20 17:39:44 +01:00
antirez 05ea8c6122 SipHash x86 optimizations. 2017-02-20 17:32:46 +01:00
antirez adeed29a99 Use SipHash hash function to mitigate HashDos attempts. 
This change attempts to switch to an hash function which mitigates
the effects of the HashDoS attack (denial of service attack trying
to force data structures to worst case behavior) while at the same time
providing Redis with an hash function that does not expect the input
data to be word aligned, a condition no longer true now that sds.c
strings have a varialbe length header.

Note that it is possible sometimes that even using an hash function
for which collisions cannot be generated without knowing the seed,
special implementation details or the exposure of the seed in an
indirect way (for example the ability to add elements to a Set and
check the return in which Redis returns them with SMEMBERS) may
make the attacker's life simpler in the process of trying to guess
the correct seed, however the next step would be to switch to a
log(N) data structure when too many items in a single bucket are
detected: this seems like an overkill in the case of Redis.

SPEED REGRESION TESTS:

In order to verify that switching from MurmurHash to SipHash had
no impact on speed, a set of benchmarks involving fast insertion
of 5 million of keys were performed.

The result shows Redis with SipHash in high pipelining conditions
to be about 4% slower compared to using the previous hash function.
However this could partially be related to the fact that the current
implementation does not attempt to hash whole words at a time but
reads single bytes, in order to have an output which is endian-netural
and at the same time working on systems where unaligned memory accesses
are a problem.

Further X86 specific optimizations should be tested, the function
may easily get at the same level of MurMurHash2 if a few optimizations
are performed.
 2017-02-20 17:29:17 +01:00
John.Koepi 9b05aafb50 fix #2883, #2857 pipe fds leak when fork() failed on bg aof rw 2017-02-20 10:22:57 +01:00
antirez 76d87f47c7 Don't leak file descriptor on syncWithMaster(). 
Close #3804.
 2017-02-20 10:18:41 +01:00
Salvatore Sanfilippo 7329cc3981 ARM: Avoid fast path for BITOP. 
GCC will produce certain unaligned multi load-store instructions
that will be trapped by the Linux kernel since ARM v6 cannot
handle them with unaligned addresses. Better to use the slower
but safer implementation instead of generating the exception which
should be anyway very slow.
 2017-02-19 15:07:08 +00:00
Salvatore Sanfilippo 4e9cf4cc7e ARM: Use libc malloc by default. 
I'm not sure how much test Jemalloc gets on ARM, moreover
compiling Redis with Jemalloc support in not very powerful
devices, like most ARMs people will build Redis on, is extremely
slow. It is possible to enable Jemalloc build anyway if needed
by using "make MALLOC=jemalloc".
 2017-02-19 15:02:37 +00:00
Salvatore Sanfilippo 72d6d64771 ARM: Avoid memcpy() in MurmurHash64A() if we are using 64 bit ARM. 
However note that in architectures supporting 64 bit unaligned
accesses memcpy(...,...,8) is likely translated to a simple
word memory movement anyway.
 2017-02-19 15:00:46 +00:00
Salvatore Sanfilippo 1e272a6b52 ARM: Fix 64 bit unaligned access in MurmurHash64A(). 2017-02-19 14:01:58 +00:00
minghang.zmh de07deb4d2 fix server.stat_net_output_bytes calc bug 2017-02-10 20:13:01 +08:00
flowly 1f72ec7dad Merge pull request #1 from antirez/unstable 
update to upstream
 2017-02-10 19:53:36 +08:00
antirez f917e0da4c Fix MIGRATE closing of cached socket on error. 
After investigating issue #3796, it was discovered that MIGRATE
could call migrateCloseSocket() after the original MIGRATE c->argv
was already rewritten as a DEL operation. As a result the host/port
passed to migrateCloseSocket() could be anything, often a NULL pointer
that gets deferenced crashing the server.

Now the socket is closed at an earlier time when there is a socket
error in a later stage where no retry will be performed, before we
rewrite the argument vector. Moreover a check was added so that later,
in the socket_err label, there is no further attempt at closing the
socket if the argument was rewritten.

This fix should resolve the bug reported in #3796.
 2017-02-09 09:58:38 +01:00
antirez 0dbfb1d154 Fix ziplist fix... 2017-02-01 17:01:31 +01:00
antirez c495d095ae Ziplist: insertion bug under particular conditions fixed. 
Ziplists had a bug that was discovered while investigating a different
issue, resulting in a corrupted ziplist representation, and a likely
segmentation foult and/or data corruption of the last element of the
ziplist, once the ziplist is accessed again.

The bug happens when a specific set of insertions / deletions is
performed so that an entry is encoded to have a "prevlen" field (the
length of the previous entry) of 5 bytes but with a count that could be
encoded in a "prevlen" field of a since byte. This could happen when the
"cascading update" process called by ziplistInsert()/ziplistDelete() in
certain contitious forces the prevlen to be bigger than necessary in
order to avoid too much data moving around.

Once such an entry is generated, inserting a very small entry
immediately before it will result in a resizing of the ziplist for a
count smaller than the current ziplist length (which is a violation,
inserting code expects the ziplist to get bigger actually). So an FF
byte is inserted in a misplaced position. Moreover a realloc() is
performed with a count smaller than the ziplist current length so the
final bytes could be trashed as well.

SECURITY IMPLICATIONS:

Currently it looks like an attacker can only crash a Redis server by
providing specifically choosen commands. However a FF byte is written
and there are other memory operations that depend on a wrong count, so
even if it is not immediately apparent how to mount an attack in order
to execute code remotely, it is not impossible at all that this could be
done. Attacks always get better... and we did not spent enough time in
order to think how to exploit this issue, but security researchers
or malicious attackers could.
 2017-02-01 15:01:59 +01:00
antirez 3a7410a8a6 ziplist: better comments, some refactoring. 2017-01-30 10:12:47 +01:00
antirez 27e29f4fe6 Jemalloc updated to 4.4.0. 
The original jemalloc source tree was modified to:

1. Remove the configure error that prevents nested builds.
2. Insert the Redis private Jemalloc API in order to allow the
Redis fragmentation function to work.
 2017-01-30 09:58:34 +01:00
Salvatore Sanfilippo 713fe0b7d7 Merge pull request #3784 from badboy/dont-divide-by-zero 
Don't divide by zero
 2017-01-27 16:26:24 +01:00
Jan-Erik Rediger 3c9b817217 Don't divide by zero 
Previously Redis crashed on `MEMORY DOCTOR` when it has no slaves attached.

Fixes #3783
 2017-01-27 16:24:14 +01:00
Salvatore Sanfilippo 2e4f53a711 Merge pull request #3778 from MiterV1/unstable 
Change switch statment to if statment
 2017-01-26 13:41:59 +01:00
miter 3ec1a001fb Change switch statment to if statment 2017-01-26 21:36:26 +09:00
Salvatore Sanfilippo 41d16f7a4a Merge pull request #3657 from itamarhaber/patch-9 
Verify pairs are provided after ZADD's subcommands
 2017-01-25 09:31:47 +01:00
Salvatore Sanfilippo 432699845c Merge pull request #3712 from oranagra/fix_assert_debug_digest 
fix rare assertion in DEBUG DIGEST
 2017-01-20 11:01:43 +01:00
antirez 17ac46ea78 Add panic() into redisassert.h. 
This header file is for libs, like ziplist.c, that we want to leave
almost separted from the core. The panic() calls will be easy to delete
in order to use such files outside, but the debugging info we gain are
very valuable compared to simple assertions where it is not possible to
print debugging info.
 2017-01-18 17:12:07 +01:00
antirez 53b8bf2c89 serverPanic(): allow printf() alike formatting. 
This is of great interest because allows us to print debugging
informations that could be of useful when debugging, like in the
following example:

    serverPanic("Unexpected encoding for object %d, %d",
        obj->type, obj->encoding);
 2017-01-18 17:05:10 +01:00
antirez 2cd1ae736f Ziplist: remove static from functions, they prevent good crash reports. 2017-01-13 11:55:13 +01:00
Salvatore Sanfilippo d21aabcedc Merge pull request #3734 from badboy/avoid-command 
Initialize help only in repl mode
 2017-01-13 11:32:22 +01:00
antirez 636c693f44 Use const in modules types mem_usage method. 
As suggested by @itamarhaber.
 2017-01-12 12:47:46 +01:00
antirez 3f79b2f883 Defrag: don't crash when a module value is encountered. 2017-01-12 09:50:40 +01:00
antirez baa9898821 MEMORY USAGE: support for modules data types. 
As a side effect of supporting it, we no longer crash when MEMORY USAGE
is called against a module data type.

Close #3637.
 2017-01-12 09:47:57 +01:00
Guy Benoish 56c01c959a enlarged buffer given to ld2string 2017-01-11 19:24:19 +02:00
antirez 634b096610 Defrag: document the feature in redis.conf. 2017-01-11 17:24:49 +01:00
antirez 6ad34a4b78 Defrag: not enabled by default. Error on CONFIG SET if not available. 2017-01-11 15:43:08 +01:00
antirez 86192f3038 Defrag: fix function name typo defarg -> defrag. 2017-01-11 15:38:12 +01:00
antirez 4186879675 Defrag: do not crash on empty quicklist. 2017-01-11 15:38:09 +01:00
antirez e91f0ea1b3 Defrag: fix comments & code to conform to the Redis code base. 
Don't go over 80 cols. Start with captial letter, capital letter afer
point, end comment with a point and so forth. No actual code behavior
touched at all.
 2017-01-10 11:33:50 +01:00
antirez 173d692bc2 Defrag: activate it only if running modified version of Jemalloc. 
This commit also includes minor aesthetic changes like removal of
trailing spaces.
 2017-01-10 11:25:39 +01:00
Jan-Erik Rediger afaaa91885 Initialize help only in repl mode 2017-01-08 18:29:22 +01:00
oranagra 5ab6a54cc6 active defrag improvements 2017-01-02 09:42:32 +02:00
oranagra 7aa9e6d2ae active memory defragmentation 2016-12-30 03:37:52 +02:00
oranagra b2da5ea773 fix rare assertion in DEBUG DIGEST 
getExpire calls dictFind which can do rehashing.
found by calling computeDatasetDigest from serverCron and running the test suite.
 2016-12-24 17:27:58 +02:00
antirez 6712bce92c deps/hiredis updated to latest version. 
Close #3687.
 2016-12-21 12:12:36 +01:00
Salvatore Sanfilippo 0b7691201e Merge pull request #3242 from whatacold/unstable 
fix the wrong description of intsetGet().
 2016-12-20 15:39:56 +01:00
Salvatore Sanfilippo 43bccbc3f4 Merge pull request #3498 from ZhukovAlexander/patch-1 
Fix an article usage
 2016-12-20 13:39:29 +01:00
antirez a764c87b67 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2016-12-20 13:33:10 +01:00
Salvatore Sanfilippo 619317da6f Merge pull request #3696 from jstncarvalho/FixMissingBrackets_ZIP_DECODE_LENGTH 
Fix missing brackets around encoding variable in ZIP_DECODE_LENGTH macro
 2016-12-20 13:32:54 +01:00
antirez af0284926c Geo: fuzzy test inconsistency report fixed to show all points. 
We need to report all the points added into the set, not just the ones
matching the Tcl distance algo.
 2016-12-20 12:22:11 +01:00
antirez 0f72257049 Geo: fix GEOHASH return value for consistency. 
The same thing observed in #3551 by gnethercutt also fixed for
GEOHASH as the original PR did.
 2016-12-20 10:20:13 +01:00
antirez 913070a9e8 Geo: fix edge case return values for uniformity. 
There were two cases outlined in issue #3512 and PR #3551 where
the Geo API returned unexpected results: empty strings where NULL
replies were expected, or a single null reply where an array was
expected. This violates the Redis principle that Redis replies for
existing keys or elements should be indistinguishable.

This is technically an API breakage so will be merged only into 4.0 and
specified in the changelog in the list of breaking compatibilities, even
if it is not very likely that actual code will be affected, hopefully,
since with the past behavior basically there was to acconut for *both*
the possibilities, and the new behavior is always one of the two, but
in a consistent way.
 2016-12-20 10:12:38 +01:00
Justin Carvalho 7c64e88963 Fix missing brackets around encoding variable in ZIP_DECODE_LENGTH macro 2016-12-19 17:37:41 -05:00
antirez 8444b46d20 Fix test "server is up" detection after logging changes. 2016-12-19 16:49:58 +01:00
antirez 074383f850 Remove first version of ASCII wave, later discarded. 2016-12-19 16:45:18 +01:00
antirez 06bfeb482d Only show Redis logo if logging to stdout / TTY. 
You can still force the logo in the normal logs.
For motivations, check issue #3112. For me the reason is that actually
the logo is nice to have in interactive sessions, but inside the logs
kinda loses its usefulness, but for the ability of users to recognize
restarts easily: for this reason the new startup sequence shows a one
liner ASCII "wave" so that there is still a bit of visual clue.

Startup logging was modified in order to log events in more obvious
ways, and to log more events. Also certain important informations are
now more easy to parse/grep since they are printed in field=value style.

The option --always-show-logo in redis.conf was added, defaulting to no.
 2016-12-19 16:41:47 +01:00
antirez 90a6f7fc98 adjustOpenFilesLimit() comment made hopefully more clear. 2016-12-19 08:53:29 +01:00
Salvatore Sanfilippo 2988889db1 Merge pull request #3603 from oranagra/adjustOpenFilesLimit_overflow 
fix unsigned int overflow in adjustOpenFilesLimit
 2016-12-19 08:48:44 +01:00
Salvatore Sanfilippo ce9e36eb01 Merge pull request #3605 from hylepo/unstable 
Fixing typo in the usage of redis-benchmark
 2016-12-19 08:20:01 +01:00
Salvatore Sanfilippo 6cf1a325d6 Merge pull request #3643 from andyli028/unstable 
Modify MIN->MAX
 2016-12-19 08:19:10 +01:00
antirez 8e390a62ad Hopefully improve code comments for issue #3616. 
This commit also contains other changes in order to conform the code to
the Redis core style, specifically 80 chars max per line, smart
conditionals in the same line:

    if (that) do_this();
 2016-12-16 17:48:38 +01:00
Salvatore Sanfilippo ca4ca5073e Merge pull request #3616 from oranagra/stop_aofrw_before_rdbload 
CoW improvement, stop AOFRW before flushing and parsing slave RDB
 2016-12-16 17:43:20 +01:00
Salvatore Sanfilippo 151af73118 Merge pull request #3661 from itamarhaber/module-doc2 
Corrects a couple of omissions in the modules docs
 2016-12-16 16:53:13 +01:00
antirez 87538cb7fe Switch PFCOUNT to LogLog-Beta algorithm. 
The new algorithm provides the same speed with a smaller error for
cardinalities in the range 0-100k. Before switching, the new and old
algorithm behavior was studied in details in the context of
issue #3677. You can find a few graphs and motivations there.
 2016-12-16 11:07:30 +01:00
antirez 0224be8811 Use llroundl() before converting loglog-beta output to integer. 
Otherwise for small cardinalities the algorithm will output something
like, for example, 4.99 for a candinality of 5, that will be converted
to 4 producing a huge error.
 2016-12-16 11:07:30 +01:00
antirez 47dea01c85 Fix HLL gnuplot graph generator script for new redis-rb versions. 
The PFADD now takes an array and has mandatory two arguments.
 2016-12-16 11:07:30 +01:00
Harish Murthy c55e3fbae5 LogLog-Beta Algorithm support within HLL 
Config option to use LogLog-Beta Algorithm for Cardinality
 2016-12-16 11:07:30 +01:00
Salvatore Sanfilippo 5ad2a94a16 Merge pull request #3686 from dvirsky/fix_lowlevel_zrange 
fixed stop condition in RM_ZsetRangeNext and RM_ZsetRangePrev
 2016-12-16 09:20:47 +01:00
antirez d634c36253 ziplist.c explanation of format improved a bit. 2016-12-16 09:04:57 +01:00
antirez ac61f90625 DEBUG: new "ziplist" subcommand added. Dumps a ziplist on stdout. 
The commit improves ziplistRepr() and adds a new debugging subcommand so
that we can trigger the dump directly from the Redis API.
This command capability was used while investigating issue #3684.
 2016-12-16 09:02:50 +01:00
Dvir Volk 7f9b9512b8 fixed stop condition in RM_ZsetRangeNext and RM_ZsetRangePrev 2016-12-15 00:07:20 +02:00
antirez b53e73e159 MIGRATE: Remove upfront ttl initialization. 
After the fix for #3673 the ttl var is always initialized inside the
loop itself, so the early initialization is not needed.

Variables declaration also moved to a more local scope.
 2016-12-14 12:43:55 +01:00
Salvatore Sanfilippo c9f0456d81 Merge pull request #3673 from badboy/reset-ttl-on-migrating 
Reset the ttl for additional keys
 2016-12-14 12:41:00 +01:00
antirez dee11ebab8 Writable slaves expires: unit test. 2016-12-13 16:28:12 +01:00
antirez b6f871cf42 Writable slaves expires: fix leak in key tracking. 
We need to use a dictionary type that frees the key, since we copy the
keys in the dictionary we use to track expires created in the slave
side.
 2016-12-13 16:27:13 +01:00
antirez d1adc85aa6 INFO: show num of slave-expires keys tracked. 2016-12-13 16:02:29 +01:00
antirez 5b9ba26403 Fix created->created typo in expire.c 2016-12-13 12:21:15 +01:00
antirez 04542cff92 Replication: fix the infamous key leakage of writable slaves + EXPIRE. 
BACKGROUND AND USE CASEj

Redis slaves are normally write only, however the supprot a "writable"
mode which is very handy when scaling reads on slaves, that actually
need write operations in order to access data. For instance imagine
having slaves replicating certain Sets keys from the master. When
accessing the data on the slave, we want to peform intersections between
such Sets values. However we don't want to intersect each time: to cache
the intersection for some time often is a good idea.

To do so, it is possible to setup a slave as a writable slave, and
perform the intersection on the slave side, perhaps setting a TTL on the
resulting key so that it will expire after some time.

THE BUG

Problem: in order to have a consistent replication, expiring of keys in
Redis replication is up to the master, that synthesize DEL operations to
send in the replication stream. However slaves logically expire keys
by hiding them from read attempts from clients so that if the master did
not promptly sent a DEL, the client still see logically expired keys
as non existing.

Because slaves don't actively expire keys by actually evicting them but
just masking from the POV of read operations, if a key is created in a
writable slave, and an expire is set, the key will be leaked forever:

1. No DEL will be received from the master, which does not know about
such a key at all.

2. No eviction will be performed by the slave, since it needs to disable
eviction because it's up to masters, otherwise consistency of data is
lost.

THE FIX

In order to fix the problem, the slave should be able to tag keys that
were created in the slave side and have an expire set in some way.

My solution involved using an unique additional dictionary created by
the writable slave only if needed. The dictionary is obviously keyed by
the key name that we need to track: all the keys that are set with an
expire directly by a client writing to the slave are tracked.

The value in the dictionary is a bitmap of all the DBs where such a key
name need to be tracked, so that we can use a single dictionary to track
keys in all the DBs used by the slave (actually this limits the solution
to the first 64 DBs, but the default with Redis is to use 16 DBs).

This solution allows to pay both a small complexity and CPU penalty,
which is zero when the feature is not used, actually. The slave-side
eviction is encapsulated in code which is not coupled with the rest of
the Redis core, if not for the hook to track the keys.

TODO

I'm doing the first smoke tests to see if the feature works as expected:
so far so good. Unit tests should be added before merging into the
4.0 branch.
 2016-12-13 10:59:54 +01:00
Salvatore Sanfilippo 39f5c0713e Merge pull request #3680 from yossigo/fix_rediscli_command_crash 
Fix redis-cli rare crash.
 2016-12-12 19:36:15 +01:00
Yossi Gottlieb b6ab4d04b6 Fix redis-cli rare crash. 
This happens if the server (mysteriously) returns an unexpected response
to the COMMAND command.
 2016-12-12 20:18:40 +02:00
Jan-Erik Rediger 2a32f0371e Reset the ttl for additional keys 
Before, if a previous key had a TTL set but the current one didn't, the
TTL was reused and thus resulted in wrong expirations set.

This behaviour was experienced, when `MigrateDefaultPipeline` in
redis-trib was set to >1

Fixes #3655
 2016-12-08 14:27:21 +01:00
Salvatore Sanfilippo 978a790b85 Merge pull request #3663 from wshn13/add-LF-character-in-memory-doctor-output-message 
Add '\n' to MEMORY DOCTOR command output message
 2016-12-06 09:20:36 +01:00
wangshaonan 2d91fce970 Add '\n' to MEMORY DOCTOR command output message when num_reports 
is 0 or empty is 1
 2016-12-06 03:11:27 +00:00
itamar 94fe98666c Corrects a couple of omissions in the modules docs 2016-12-05 18:34:38 +02:00
antirez 16cce320c4 Modules: types doc updated to new API. 2016-12-05 14:40:51 +01:00
antirez 37b6e16ae1 Modules: API doc updated (auto generated). 2016-12-05 14:40:43 +01:00
antirez 3c85a88888 Merge branch 'unstable' of github.com:/antirez/redis into unstable 2016-12-05 14:17:11 +01:00
antirez b1fc06f7eb Geo: improve fuzz test. 
The test now uses more diverse radius sizes, especially sizes near or
greater the whole earth surface are used, that are known to trigger edge
cases. Moreover the PRNG seeding was probably resulting into the same
sequence tested over and over again, now seeding unsing the current unix
time in milliseconds.

Related to #3631.
 2016-12-05 14:16:46 +01:00
antirez 001138aec3 Geo: fix computation of bounding box. 
A bug was reported in the context in issue #3631. The root cause of the
bug was that certain neighbor boxes were zeroed after the "inside the
bounding box or not" check, simply because the bounding box computation
function was wrong.

A few debugging infos where enhanced and moved in other parts of the
code. A check to avoid steps=0 was added, but is unrelated to this
issue and I did not verified it was an actual bug in practice.
 2016-12-05 14:02:32 +01:00
cbgbt e5db99ad4a cli: Only print elapsed time on OUTPUT_STANDARD 2016-12-02 20:59:33 -08:00
Itamar Haber 5dc4fe1529 Verify pairs are provided after subcommands 
Fixes https://github.com/antirez/redis/issues/3639
 2016-12-02 18:19:36 +02:00
Chris Lamb 6eb0c52d4c src/rdb.c: Correct "whenver" -> "whenever" typo. 2016-12-01 13:16:30 +01:00
Salvatore Sanfilippo ff1d548738 Merge pull request #3651 from yossigo/datatype_methods_typo 
Fix typo in RedisModuleTypeMethods declaration.
 2016-12-01 09:09:37 +01:00
Yossi Gottlieb 5f5b4f1508 Fix typo in RedisModuleTypeMethods declaration. 2016-11-30 22:05:59 +02:00
Salvatore Sanfilippo 3c4fe59e09 Merge pull request #3648 from dvirsky/fix_reply_crash 
fix memory corruption on RM_FreeCallReply
 2016-11-30 11:21:10 +01:00
Dvir Volk 8521cde570 fix memory corruption on RM_FreeCallReply 2016-11-30 11:49:49 +02:00
andyli 8abf9729f0 Modify MIN->MAX 2016-11-29 16:34:41 +08:00
oranagra e3a61950a2 when a slave loads an RDB, stop an AOFRW fork before flusing db and parsing rdb file, to avoid a CoW disaster. 2016-11-16 21:30:59 +02:00
hylepo 0b57a19c1b Merge pull request #1 from hylepo/Fixing-typo-in-the-usage-of-redis-benchmark 
Fixing typo in the usage of redis-benchmark
 2016-11-11 10:34:37 +08:00
hylepo dbb6cb442a Update redis-benchmark.c 
Fixing typo in the usage of redis-benchmark
 2016-11-11 10:33:48 +08:00
oranagra a1a07225b3 fix unsigned int overflow in adjustOpenFilesLimit 2016-11-10 16:59:52 +02:00
jybaek c76b9b644c Remove Duplicate Processing 2016-10-13 15:17:07 +09:00
Alexander Zhukov 27d08fb8f0 Fix an article usage 2016-09-12 13:53:47 +03:00
Yusaku Kaneta b6c4822f7c Fix the firstkey, lastkey, and keystep of moduleCommand 2016-06-29 18:09:16 +09:00
whatacold 0f36e5e5d8 fix the wrong description of intsetGet(). 2016-05-18 16:39:52 +08:00
kmiku7 413d8239df fix boundary case for _dictNextPower 2015-08-23 16:47:42 +08:00
Jan-Erik Rediger 04607b5350 Check that the whole first argument is a number 
Fixes #2258
 2015-01-07 22:31:45 +01:00
Byron Grobe 087b19237c Fixed issue #1996 (Missing '-' in help message for redis-benchmark) 2014-09-11 10:21:05 -04:00
Jan-Erik Rediger 367035a983 Don't use extended Regexp Syntax 
It's not POSIX (BSD systems have -E instead) and we don't actually need it.

Closes #1922
 2014-08-13 19:34:03 +02:00
503 changed files with 76097 additions and 34555 deletions
Expand all files Collapse all files
00-RELEASENOTES
+2642 -11
View File
File diff suppressed because it is too large Load Diff
CONTRIBUTING
+4 -2
View File
@@ -8,7 +8,9 @@ each source file that you contribute.
# IMPORTANT: HOW TO USE REDIS GITHUB ISSUES
* Github issues SHOULD ONLY BE USED to report bugs, and for DETAILED feature
requests. Everything else belongs to the Redis Google Group.
requests. Everything else belongs to the Redis Google Group:
https://groups.google.com/forum/m/#!forum/Redis-db
PLEASE DO NOT POST GENERAL QUESTIONS that are not about bugs or suspected
bugs in the Github issues system. We'll be very happy to help you and provide
@@ -30,7 +32,7 @@ each source file that you contribute.
a. Fork Redis on github ( http://help.github.com/fork-a-repo/ )
b. Create a topic branch (git checkout -b my_branch)
c. Push to your branch (git push origin my_branch)
d. Initiate a pull request on github ( http://help.github.com/send-pull-requests/ )
d. Initiate a pull request on github ( https://help.github.com/articles/creating-a-pull-request/ )
e. Done :)
For minor fixes just open a pull request on Github.
README.md
+11 -11
View File
@@ -1,4 +1,4 @@
This README is just a fast *quick start* document. You can find more detailed documentation at http://redis.io.
This README is just a fast *quick start* document. You can find more detailed documentation at [redis.io](https://redis.io).
What is Redis?
--------------
@@ -119,7 +119,7 @@ parameter (the path of the configuration file):
It is possible to alter the Redis configuration by passing parameters directly
as options using the command line. Examples:
% ./redis-server --port 9999 --slaveof 127.0.0.1 6379
% ./redis-server --port 9999 --replicaof 127.0.0.1 6379
% ./redis-server /etc/redis/6379.conf --loglevel debug
All the options in redis.conf are also supported as options using the command
@@ -216,7 +216,7 @@ Inside the root are the following important directories:
* `src`: contains the Redis implementation, written in C.
* `tests`: contains the unit tests, implemented in Tcl.
* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `anitrez/redis`. An exception to this rule is `deps/geohash-int` which is the low level geocoding library used by Redis: it originated from a different project, but at this point it diverged so much that it is developed as a separated entity directly inside the Redis repository.
* `deps`: contains libraries Redis uses. Everything needed to compile Redis is inside this directory; your system just needs to provide `libc`, a POSIX compatible interface and a C compiler. Notably `deps` contains a copy of `jemalloc`, which is the default allocator of Redis under Linux. Note that under `deps` there are also things which started with the Redis project, but for which the main repository is not `antirez/redis`. An exception to this rule is `deps/geohash-int` which is the low level geocoding library used by Redis: it originated from a different project, but at this point it diverged so much that it is developed as a separated entity directly inside the Redis repository.
There are a few more directories but they are not very important for our goals
here. We'll focus mostly on `src`, where the Redis implementation is contained,
@@ -227,7 +227,7 @@ of complexity incrementally.
Note: lately Redis was refactored quite a bit. Function names and file
names have been changed, so you may find that this documentation reflects the
`unstable` branch more closely. For instance in Redis 3.0 the `server.c`
and `server.h` files were named to `redis.c` and `redis.h`. However the overall
and `server.h` files were named `redis.c` and `redis.h`. However the overall
structure is the same. Keep in mind that all the new developments and pull
requests should be performed against the `unstable` branch.
@@ -245,7 +245,7 @@ A few important fields in this structure are:
* `server.db` is an array of Redis databases, where data is stored.
* `server.commands` is the command table.
* `server.clients` is a linked list of clients connected to the server.
* `server.master` is a special client, the master, if the instance is a slave.
* `server.master` is a special client, the master, if the instance is a replica.
There are tons of other fields. Most fields are commented directly inside
the structure definition.
@@ -323,7 +323,7 @@ Inside server.c you can find code that handles other vital things of the Redis s
networking.c
---
This file defines all the I/O functions with clients, masters and slaves
This file defines all the I/O functions with clients, masters and replicas
(which in Redis are just special clients):
* `createClient()` allocates and initializes a new client.
@@ -390,16 +390,16 @@ replication.c
This is one of the most complex files inside Redis, it is recommended to
approach it only after getting a bit familiar with the rest of the code base.
In this file there is the implementation of both the master and slave role
In this file there is the implementation of both the master and replica role
of Redis.
One of the most important functions inside this file is `replicationFeedSlaves()` that writes commands to the clients representing slave instances connected
to our master, so that the slaves can get the writes performed by the clients:
One of the most important functions inside this file is `replicationFeedSlaves()` that writes commands to the clients representing replica instances connected
to our master, so that the replicas can get the writes performed by the clients:
this way their data set will remain synchronized with the one in the master.
This file also implements both the `SYNC` and `PSYNC` commands that are
used in order to perform the first synchronization between masters and
slaves, or to continue the replication after a disconnection.
replicas, or to continue the replication after a disconnection.
Other C files
---
@@ -435,7 +435,7 @@ top comment inside `server.c`.
After the command operates in some way, it returns a reply to the client,
usually using `addReply()` or a similar function defined inside `networking.c`.
There are tons of commands implementations inside th Redis source code
There are tons of commands implementations inside the Redis source code
that can serve as examples of actual commands implementations. To write
a few toy commands can be a good exercise to familiarize with the code base.
deps/Makefile
Vendored
+1 -1
View File
@@ -77,7 +77,7 @@ JEMALLOC_LDFLAGS= $(LDFLAGS)
jemalloc: .make-prerequisites
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
cd jemalloc && ./configure --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a
.PHONY: jemalloc
deps/README.md
Vendored
+27 -5
View File
@@ -13,12 +13,34 @@ How to upgrade the above dependencies
Jemalloc
---
Jemalloc is unmodified. We only change settings via the `configure` script of Jemalloc using the `--with-lg-quantum` option, setting it to the value of 3 instead of 4. This provides us with more size classes that better suit the Redis data structures, in order to gain memory efficiency.
So in order to upgrade jemalloc:
Jemalloc is modified with changes that allow us to implement the Redis
active defragmentation logic. However this feature of Redis is not mandatory
and Redis is able to understand if the Jemalloc version it is compiled
against supports such Redis-specific modifications. So in theory, if you
are not interested in the active defragmentation, you can replace Jemalloc
just following tose steps:
1. Remove the jemalloc directory.
2. Substitute it with the new jemalloc source tree.
3. Edit the Makefile localted in the same directory as the README you are
reading, and change the --with-version in the Jemalloc configure script
options with the version you are using. This is required because otherwise
Jemalloc configuration script is broken and will not work nested in another
git repository.
However note that we change Jemalloc settings via the `configure` script of Jemalloc using the `--with-lg-quantum` option, setting it to the value of 3 instead of 4. This provides us with more size classes that better suit the Redis data structures, in order to gain memory efficiency.
If you want to upgrade Jemalloc while also providing support for
active defragmentation, in addition to the above steps you need to perform
the following additional steps:
5. In Jemalloc three, file `include/jemalloc/jemalloc_macros.h.in`, make sure
to add `#define JEMALLOC_FRAG_HINT`.
6. Implement the function `je_get_defrag_hint()` inside `src/jemalloc.c`. You
can see how it is implemented in the current Jemalloc source tree shipped
with Redis, and rewrite it according to the new Jemalloc internals, if they
changed, otherwise you could just copy the old implementation if you are
upgrading just to a similar version of Jemalloc.
Geohash
---
@@ -28,7 +50,7 @@ This is never upgraded since it's part of the Redis project. If there are change
Hiredis
---
Hiredis uses the SDS string library, that must be the same version used inside Redis itself. Hiredis is also very critical for Sentinel. Historically Redis often used forked versions of hiredis in a way or the other. In order to upgrade it is adviced to take a lot of care:
Hiredis uses the SDS string library, that must be the same version used inside Redis itself. Hiredis is also very critical for Sentinel. Historically Redis often used forked versions of hiredis in a way or the other. In order to upgrade it is advised to take a lot of care:
1. Check with diff if hiredis API changed and what impact it could have in Redis.
2. Make sure thet the SDS library inside Hiredis and inside Redis are compatible.
@@ -61,6 +83,6 @@ and our version:
1. Makefile is modified to allow a different compiler than GCC.
2. We have the implementation source code, and directly link to the following external libraries: `lua_cjson.o`, `lua_struct.o`, `lua_cmsgpack.o` and `lua_bit.o`.
3. There is a security fix in `ldo.c`, line 498: The check for `LUA_SIGNATURE[0]` is removed in order toa void direct bytecode exectuion.
3. There is a security fix in `ldo.c`, line 498: The check for `LUA_SIGNATURE[0]` is removed in order toa void direct bytecode execution.
deps/hiredis/.gitignore
+1
View File
@@ -4,3 +4,4 @@
/*.so
/*.dylib
/*.a
/*.pc
deps/hiredis/.travis.yml
Vendored
+34 -1
View File
@@ -1,6 +1,39 @@
language: c
sudo: false
compiler:
- gcc
- clang
script: make && make check
os:
- linux
- osx
before_script:
- if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update; brew install redis; fi
addons:
apt:
packages:
- libc6-dbg
- libc6-dev
- libc6:i386
- libc6-dev-i386
- libc6-dbg:i386
- gcc-multilib
- valgrind
env:
- CFLAGS="-Werror"
- PRE="valgrind --track-origins=yes --leak-check=full"
- TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror"
- TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full"
matrix:
exclude:
- os: osx
env: PRE="valgrind --track-origins=yes --leak-check=full"
- os: osx
env: TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full"
script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example
deps/hiredis/CHANGELOG.md
Vendored
+125
View File
@@ -1,3 +1,128 @@
### 1.0.0 (unreleased)
**Fixes**:
* Catch a buffer overflow when formatting the error message
* Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13
* Fix warnings, when compiled with -Wshadow
* Make hiredis compile in Cygwin on Windows, now CI-tested
**BREAKING CHANGES**:
* Change `redisReply.len` to `size_t`, as it denotes the the size of a string
User code should compare this to `size_t` values as well.
If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before.
* Remove backwards compatibility macro's
This removes the following old function aliases, use the new name now:
| Old | New |
| --------------------------- | ---------------------- |
| redisReplyReaderCreate | redisReaderCreate |
| redisReplyReaderCreate | redisReaderCreate |
| redisReplyReaderFree | redisReaderFree |
| redisReplyReaderFeed | redisReaderFeed |
| redisReplyReaderGetReply | redisReaderGetReply |
| redisReplyReaderSetPrivdata | redisReaderSetPrivdata |
| redisReplyReaderGetObject | redisReaderGetObject |
| redisReplyReaderGetError | redisReaderGetError |
* The `DEBUG` variable in the Makefile was renamed to `DEBUG_FLAGS`
Previously it broke some builds for people that had `DEBUG` set to some arbitrary value,
due to debugging other software.
By renaming we avoid unintentional name clashes.
Simply rename `DEBUG` to `DEBUG_FLAGS` in your environment to make it working again.
### 0.13.3 (2015-09-16)
* Revert "Clear `REDIS_CONNECTED` flag when connection is closed".
* Make tests pass on FreeBSD (Thanks, Giacomo Olgeni)
If the `REDIS_CONNECTED` flag is cleared,
the async onDisconnect callback function will never be called.
This causes problems as the disconnect is never reported back to the user.
### 0.13.2 (2015-08-25)
* Prevent crash on pending replies in async code (Thanks, @switch-st)
* Clear `REDIS_CONNECTED` flag when connection is closed (Thanks, Jerry Jacobs)
* Add MacOS X addapter (Thanks, @dizzus)
* Add Qt adapter (Thanks, Pietro Cerutti)
* Add Ivykis adapter (Thanks, Gergely Nagy)
All adapters are provided as is and are only tested where possible.
### 0.13.1 (2015-05-03)
This is a bug fix release.
The new `reconnect` method introduced new struct members, which clashed with pre-defined names in pre-C99 code.
Another commit forced C99 compilation just to make it work, but of course this is not desirable for outside projects.
Other non-C99 code can now use hiredis as usual again.
Sorry for the inconvenience.
* Fix memory leak in async reply handling (Salvatore Sanfilippo)
* Rename struct member to avoid name clash with pre-c99 code (Alex Balashov, ncopa)
### 0.13.0 (2015-04-16)
This release adds a minimal Windows compatibility layer.
The parser, standalone since v0.12.0, can now be compiled on Windows
(and thus used in other client libraries as well)
* Windows compatibility layer for parser code (tzickel)
* Properly escape data printed to PKGCONF file (Dan Skorupski)
* Fix tests when assert() undefined (Keith Bennett, Matt Stancliff)
* Implement a reconnect method for the client context, this changes the structure of `redisContext` (Aaron Bedra)
### 0.12.1 (2015-01-26)
* Fix `make install`: DESTDIR support, install all required files, install PKGCONF in proper location
* Fix `make test` as 32 bit build on 64 bit platform
### 0.12.0 (2015-01-22)
* Add optional KeepAlive support
* Try again on EINTR errors
* Add libuv adapter
* Add IPv6 support
* Remove possiblity of multiple close on same fd
* Add ability to bind source address on connect
* Add redisConnectFd() and redisFreeKeepFd()
* Fix getaddrinfo() memory leak
* Free string if it is unused (fixes memory leak)
* Improve redisAppendCommandArgv performance 2.5x
* Add support for SO_REUSEADDR
* Fix redisvFormatCommand format parsing
* Add GLib 2.0 adapter
* Refactor reading code into read.c
* Fix errno error buffers to not clobber errors
* Generate pkgconf during build
* Silence _BSD_SOURCE warnings
* Improve digit counting for multibulk creation
### 0.11.0
* Increase the maximum multi-bulk reply depth to 7.
deps/hiredis/Makefile
Vendored
+79 -28
View File
@@ -3,13 +3,25 @@
# Copyright (C) 2010-2011 Pieter Noordhuis <pcnoordhuis at gmail dot com>
# This file is released under the BSD license, see the COPYING file
OBJ=net.o hiredis.o sds.o async.o
EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev
OBJ=net.o hiredis.o sds.o async.o read.o
EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib
TESTS=hiredis-test
LIBNAME=libhiredis
PKGCONFNAME=hiredis.pc
HIREDIS_MAJOR=0
HIREDIS_MINOR=11
HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}')
HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}')
HIREDIS_PATCH=$(shell grep HIREDIS_PATCH hiredis.h | awk '{print $$3}')
HIREDIS_SONAME=$(shell grep HIREDIS_SONAME hiredis.h | awk '{print $$3}')
# Installation related variables and target
PREFIX?=/usr/local
INCLUDE_PATH?=include/hiredis
LIBRARY_PATH?=lib
PKGCONF_PATH?=pkgconfig
INSTALL_INCLUDE_PATH= $(DESTDIR)$(PREFIX)/$(INCLUDE_PATH)
INSTALL_LIBRARY_PATH= $(DESTDIR)$(PREFIX)/$(LIBRARY_PATH)
INSTALL_PKGCONF_PATH= $(INSTALL_LIBRARY_PATH)/$(PKGCONF_PATH)
# redis-server configuration used for testing
REDIS_PORT=56379
@@ -25,15 +37,16 @@ export REDIS_TEST_CONFIG
# Fallback to gcc when $CC is not in $PATH.
CC:=$(shell sh -c 'type $(CC) >/dev/null 2>/dev/null && echo $(CC) || echo gcc')
CXX:=$(shell sh -c 'type $(CXX) >/dev/null 2>/dev/null && echo $(CXX) || echo g++')
OPTIMIZATION?=-O3
WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings
DEBUG?= -g -ggdb
REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CFLAGS) $(WARNINGS) $(DEBUG) $(ARCH)
DEBUG_FLAGS?= -g -ggdb
REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) $(ARCH)
REAL_LDFLAGS=$(LDFLAGS) $(ARCH)
DYLIBSUFFIX=so
STLIBSUFFIX=a
DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR).$(HIREDIS_MINOR)
DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME)
DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR)
DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX)
DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS)
@@ -49,19 +62,20 @@ ifeq ($(uname_S),SunOS)
endif
ifeq ($(uname_S),Darwin)
DYLIBSUFFIX=dylib
DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(DYLIBSUFFIX)
DYLIB_MAJOR_NAME=$(LIBNAME).$(HIREDIS_MAJOR).$(DYLIBSUFFIX)
DYLIB_MINOR_NAME=$(LIBNAME).$(HIREDIS_SONAME).$(DYLIBSUFFIX)
DYLIB_MAKE_CMD=$(CC) -shared -Wl,-install_name,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS)
endif
all: $(DYLIBNAME)
all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME)
# Deps (use make dep to generate this)
net.o: net.c fmacros.h net.h hiredis.h
async.o: async.c async.h hiredis.h sds.h dict.c dict.h
hiredis.o: hiredis.c fmacros.h hiredis.h net.h sds.h
async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h
dict.o: dict.c fmacros.h dict.h
hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h
net.o: net.c fmacros.h net.h hiredis.h read.h sds.h
read.o: read.c fmacros.h read.h sds.h
sds.o: sds.c sds.h
test.o: test.c hiredis.h
test.o: test.c fmacros.h hiredis.h read.h sds.h
$(DYLIBNAME): $(OBJ)
$(DYLIB_MAKE_CMD) $(OBJ)
@@ -79,6 +93,15 @@ hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLI
hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME)
hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) $(shell pkg-config --cflags --libs glib-2.0) -I. $< $(STLIBNAME)
hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME)
hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME)
ifndef AE_DIR
hiredis-example-ae:
@echo "Please specify AE_DIR (e.g. <redis repository>/src)"
@@ -94,7 +117,20 @@ hiredis-example-libuv:
@false
else
hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME)
endif
ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),)
hiredis-example-qt:
@echo "Please specify QT_MOC, QT_INCLUDE_DIR AND QT_LIBRARY_DIR"
@false
else
hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME)
$(QT_MOC) adapters/qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \
$(CXX) -x c++ -o qt-adapter-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore
$(QT_MOC) examples/example-qt.h -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore | \
$(CXX) -x c++ -o qt-example-moc.o -c - $(REAL_CFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore
$(CXX) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(QT_INCLUDE_DIR) -I$(QT_INCLUDE_DIR)/QtCore -L$(QT_LIBRARY_DIR) qt-adapter-moc.o qt-example-moc.o $< -pthread $(STLIBNAME) -lQtCore
endif
hiredis-example: examples/example.c $(STLIBNAME)
@@ -103,14 +139,16 @@ hiredis-example: examples/example.c $(STLIBNAME)
examples: $(EXAMPLES)
hiredis-test: test.o $(STLIBNAME)
$(CC) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME)
hiredis-%: %.o $(STLIBNAME)
$(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME)
test: hiredis-test
./hiredis-test
check: hiredis-test
@echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) -
./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \
$(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \
( kill `cat /tmp/hiredis-test-redis.pid` && false )
kill `cat /tmp/hiredis-test-redis.pid`
@@ -118,29 +156,38 @@ check: hiredis-test
$(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $<
clean:
rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) examples/hiredis-example* *.o *.gcda *.gcno *.gcov
rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov
dep:
$(CC) -MM *.c
# Installation related variables and target
PREFIX?=/usr/local
INSTALL_INCLUDE_PATH= $(PREFIX)/include/hiredis
INSTALL_LIBRARY_PATH= $(PREFIX)/lib
ifeq ($(uname_S),SunOS)
INSTALL?= cp -r
endif
INSTALL?= cp -a
install: $(DYLIBNAME) $(STLIBNAME)
$(PKGCONFNAME): hiredis.h
@echo "Generating $@ for pkgconfig..."
@echo prefix=$(PREFIX) > $@
@echo exec_prefix=\$${prefix} >> $@
@echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@
@echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@
@echo >> $@
@echo Name: hiredis >> $@
@echo Description: Minimalistic C client library for Redis. >> $@
@echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@
@echo Libs: -L\$${libdir} -lhiredis >> $@
@echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@
install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME)
mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH)
$(INSTALL) hiredis.h async.h adapters $(INSTALL_INCLUDE_PATH)
$(INSTALL) hiredis.h async.h read.h sds.h adapters $(INSTALL_INCLUDE_PATH)
$(INSTALL) $(DYLIBNAME) $(INSTALL_LIBRARY_PATH)/$(DYLIB_MINOR_NAME)
cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIB_MAJOR_NAME)
cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MAJOR_NAME) $(DYLIBNAME)
cd $(INSTALL_LIBRARY_PATH) && ln -sf $(DYLIB_MINOR_NAME) $(DYLIBNAME)
$(INSTALL) $(STLIBNAME) $(INSTALL_LIBRARY_PATH)
mkdir -p $(INSTALL_PKGCONF_PATH)
$(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH)
32bit:
@echo ""
@@ -148,6 +195,10 @@ install: $(DYLIBNAME) $(STLIBNAME)
@echo ""
$(MAKE) CFLAGS="-m32" LDFLAGS="-m32"
32bit-vars:
$(eval CFLAGS=-m32)
$(eval LDFLAGS=-m32)
gprof:
$(MAKE) CFLAGS="-pg" LDFLAGS="-pg"
@@ -163,4 +214,4 @@ coverage: gcov
noopt:
$(MAKE) OPTIMIZATION=""
.PHONY: all test check clean dep install 32bit gprof gcov noopt
.PHONY: all test check clean dep install 32bit 32bit-vars gprof gcov noopt
deps/hiredis/README.md
Vendored
+115 -88
View File
@@ -1,11 +1,13 @@
[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis)
**This Readme reflects the latest changed in the master branch. See [v0.13.3](https://github.com/redis/hiredis/tree/v0.13.3) for the Readme and documentation for the latest release.**
# HIREDIS
Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database.
It is minimalistic because it just adds minimal support for the protocol, but
at the same time it uses an high level printf-alike API in order to make it
at the same time it uses a high level printf-alike API in order to make it
much higher level than otherwise suggested by its minimal code base and the
lack of explicit bindings for every Redis command.
@@ -20,7 +22,15 @@ Redis version >= 1.2.0.
The library comes with multiple APIs. There is the
*synchronous API*, the *asynchronous API* and the *reply parsing API*.
## UPGRADING
## Upgrading to `1.0.0`
Version 1.0.0 marks a stable release of hiredis.
It includes some minor breaking changes, mostly to make the exposed API more uniform and self-explanatory.
It also bundles the updated `sds` library, to sync up with upstream and Redis.
For most applications a recompile against the new hiredis should be enough.
For code changes see the [Changelog](CHANGELOG.md).
## Upgrading from `<0.9.0`
Version 0.9.0 is a major overhaul of hiredis in every aspect. However, upgrading existing
code using hiredis should not be a big pain. The key thing to keep in mind when
@@ -31,51 +41,62 @@ the stateless 0.0.1 that only has a file descriptor to work with.
To consume the synchronous API, there are only a few function calls that need to be introduced:
redisContext *redisConnect(const char *ip, int port);
void *redisCommand(redisContext *c, const char *format, ...);
void freeReplyObject(void *reply);
```c
redisContext *redisConnect(const char *ip, int port);
void *redisCommand(redisContext *c, const char *format, ...);
void freeReplyObject(void *reply);
```
### Connecting
The function `redisConnect` is used to create a so-called `redisContext`. The
context is where Hiredis holds state for a connection. The `redisContext`
struct has an integer `err` field that is non-zero when an the connection is in
struct has an integer `err` field that is non-zero when the connection is in
an error state. The field `errstr` will contain a string with a description of
the error. More information on errors can be found in the **Errors** section.
After trying to connect to Redis using `redisConnect` you should
check the `err` field to see if establishing the connection was successful:
redisContext *c = redisConnect("127.0.0.1", 6379);
if (c != NULL && c->err) {
```c
redisContext *c = redisConnect("127.0.0.1", 6379);
if (c == NULL || c->err) {
if (c) {
printf("Error: %s\n", c->errstr);
// handle error
} else {
printf("Can't allocate redis context\n");
}
}
```
*Note: A `redisContext` is not thread-safe.*
### Sending commands
There are several ways to issue commands to Redis. The first that will be introduced is
`redisCommand`. This function takes a format similar to printf. In the simplest form,
it is used like this:
reply = redisCommand(context, "SET foo bar");
```c
reply = redisCommand(context, "SET foo bar");
```
The specifier `%s` interpolates a string in the command, and uses `strlen` to
determine the length of the string:
reply = redisCommand(context, "SET foo %s", value);
```c
reply = redisCommand(context, "SET foo %s", value);
```
When you need to pass binary safe strings in a command, the `%b` specifier can be
used. Together with a pointer to the string, it requires a `size_t` length argument
of the string:
reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen);
```c
reply = redisCommand(context, "SET foo %b", value, (size_t) valuelen);
```
Internally, Hiredis splits the command in different arguments and will
convert it to the protocol used to communicate with Redis.
One or more spaces separates arguments, so you can use the specifiers
anywhere in an argument:
reply = redisCommand(context, "SET key:%s %s", myid, value);
```c
reply = redisCommand(context, "SET key:%s %s", myid, value);
```
### Using replies
@@ -114,11 +135,11 @@ was received:
Redis may reply with nested arrays but this is fully supported.
Replies should be freed using the `freeReplyObject()` function.
Note that this function will take care of freeing sub-replies objects
Note that this function will take care of freeing sub-reply objects
contained in arrays and nested arrays, so there is no need for the user to
free the sub replies (it is actually harmful and will corrupt the memory).
**Important:** the current version of hiredis (0.10.0) free's replies when the
**Important:** the current version of hiredis (0.10.0) frees replies when the
asynchronous API is used. This means you should not call `freeReplyObject` when
you use this API. The reply is cleaned up by hiredis _after_ the callback
returns. This behavior will probably change in future releases, so make sure to
@@ -127,19 +148,19 @@ keep an eye on the changelog when upgrading (see issue #39).
### Cleaning up
To disconnect and free the context the following function can be used:
void redisFree(redisContext *c);
This function immediately closes the socket and then free's the allocations done in
```c
void redisFree(redisContext *c);
```
This function immediately closes the socket and then frees the allocations done in
creating the context.
### Sending commands (cont'd)
Together with `redisCommand`, the function `redisCommandArgv` can be used to issue commands.
It has the following prototype:
void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
```c
void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
```
It takes the number of arguments `argc`, an array of strings `argv` and the lengths of the
arguments `argvlen`. For convenience, `argvlen` may be set to `NULL` and the function will
use `strlen(3)` on every argument to determine its length. Obviously, when any of the arguments
@@ -169,10 +190,10 @@ The function `redisGetReply` is exported as part of the Hiredis API and can be u
is expected on the socket. To pipeline commands, the only things that needs to be done is
filling up the output buffer. For this cause, two commands can be used that are identical
to the `redisCommand` family, apart from not returning a reply:
void redisAppendCommand(redisContext *c, const char *format, ...);
void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
```c
void redisAppendCommand(redisContext *c, const char *format, ...);
void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen);
```
After calling either function one or more times, `redisGetReply` can be used to receive the
subsequent replies. The return value for this function is either `REDIS_OK` or `REDIS_ERR`, where
the latter means an error occurred while reading a reply. Just as with the other commands,
@@ -180,24 +201,24 @@ the `err` field in the context can be used to find out what the cause of this er
The following examples shows a simple pipeline (resulting in only a single call to `write(2)` and
a single call to `read(2)`):
redisReply *reply;
redisAppendCommand(context,"SET foo bar");
redisAppendCommand(context,"GET foo");
redisGetReply(context,&reply); // reply for SET
freeReplyObject(reply);
redisGetReply(context,&reply); // reply for GET
freeReplyObject(reply);
```c
redisReply *reply;
redisAppendCommand(context,"SET foo bar");
redisAppendCommand(context,"GET foo");
redisGetReply(context,&reply); // reply for SET
freeReplyObject(reply);
redisGetReply(context,&reply); // reply for GET
freeReplyObject(reply);
```
This API can also be used to implement a blocking subscriber:
reply = redisCommand(context,"SUBSCRIBE foo");
```c
reply = redisCommand(context,"SUBSCRIBE foo");
freeReplyObject(reply);
while(redisGetReply(context,&reply) == REDIS_OK) {
// consume message
freeReplyObject(reply);
while(redisGetReply(context,&reply) == REDIS_OK) {
// consume message
freeReplyObject(reply);
}
}
```
### Errors
When a function call is not successful, depending on the function either `NULL` or `REDIS_ERR` is
@@ -237,58 +258,62 @@ should be checked after creation to see if there were errors creating the connec
Because the connection that will be created is non-blocking, the kernel is not able to
instantly return if the specified host and port is able to accept a connection.
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
printf("Error: %s\n", c->errstr);
// handle error
}
*Note: A `redisAsyncContext` is not thread-safe.*
```c
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
printf("Error: %s\n", c->errstr);
// handle error
}
```
The asynchronous context can hold a disconnect callback function that is called when the
connection is disconnected (either because of an error or per user request). This function should
have the following prototype:
void(const redisAsyncContext *c, int status);
```c
void(const redisAsyncContext *c, int status);
```
On a disconnect, the `status` argument is set to `REDIS_OK` when disconnection was initiated by the
user, or `REDIS_ERR` when the disconnection was caused by an error. When it is `REDIS_ERR`, the `err`
field in the context can be accessed to find out the cause of the error.
The context object is always free'd after the disconnect callback fired. When a reconnect is needed,
The context object is always freed after the disconnect callback fired. When a reconnect is needed,
the disconnect callback is a good point to do so.
Setting the disconnect callback can only be done once per context. For subsequent calls it will
return `REDIS_ERR`. The function to set the disconnect callback has the following prototype:
int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
```c
int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
```
### Sending commands and their callbacks
In an asynchronous context, commands are automatically pipelined due to the nature of an event loop.
Therefore, unlike the synchronous API, there is only a single way to send commands.
Because commands are sent to Redis asynchronously, issuing a command requires a callback function
that is called when the reply is received. Reply callbacks should have the following prototype:
void(redisAsyncContext *c, void *reply, void *privdata);
```c
void(redisAsyncContext *c, void *reply, void *privdata);
```
The `privdata` argument can be used to curry arbitrary data to the callback from the point where
the command is initially queued for execution.
The functions that can be used to issue commands in an asynchronous context are:
int redisAsyncCommand(
redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
const char *format, ...);
int redisAsyncCommandArgv(
redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
int argc, const char **argv, const size_t *argvlen);
```c
int redisAsyncCommand(
redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
const char *format, ...);
int redisAsyncCommandArgv(
redisAsyncContext *ac, redisCallbackFn *fn, void *privdata,
int argc, const char **argv, const size_t *argvlen);
```
Both functions work like their blocking counterparts. The return value is `REDIS_OK` when the command
was successfully added to the output buffer and `REDIS_ERR` otherwise. Example: when the connection
is being disconnected per user-request, no new commands may be added to the output buffer and `REDIS_ERR` is
returned on calls to the `redisAsyncCommand` family.
If the reply for a command with a `NULL` callback is read, it is immediately free'd. When the callback
for a command is non-`NULL`, the memory is free'd immediately following the callback: the reply is only
If the reply for a command with a `NULL` callback is read, it is immediately freed. When the callback
for a command is non-`NULL`, the memory is freed immediately following the callback: the reply is only
valid for the duration of the callback.
All pending callbacks are called with a `NULL` reply when the context encountered an error.
@@ -296,14 +321,14 @@ All pending callbacks are called with a `NULL` reply when the context encountere
### Disconnecting
An asynchronous connection can be terminated using:
void redisAsyncDisconnect(redisAsyncContext *ac);
```c
void redisAsyncDisconnect(redisAsyncContext *ac);
```
When this function is called, the connection is **not** immediately terminated. Instead, new
commands are no longer accepted and the connection is only terminated when all pending commands
have been written to the socket, their respective replies have been read and their respective
callbacks have been executed. After this, the disconnection callback is executed with the
`REDIS_OK` status and the context object is free'd.
`REDIS_OK` status and the context object is freed.
### Hooking it up to event library *X*
@@ -316,12 +341,12 @@ Hiredis comes with a reply parsing API that makes it easy for writing higher
level language bindings.
The reply parsing API consists of the following functions:
redisReader *redisReaderCreate(void);
void redisReaderFree(redisReader *reader);
int redisReaderFeed(redisReader *reader, const char *buf, size_t len);
int redisReaderGetReply(redisReader *reader, void **reply);
```c
redisReader *redisReaderCreate(void);
void redisReaderFree(redisReader *reader);
int redisReaderFeed(redisReader *reader, const char *buf, size_t len);
int redisReaderGetReply(redisReader *reader, void **reply);
```
The same set of functions are used internally by hiredis when creating a
normal Redis context, the above API just exposes it to the user for a direct
usage.
@@ -361,7 +386,7 @@ Both when using the Reader API directly or when using it indirectly via a
normal Redis context, the redisReader structure uses a buffer in order to
accumulate data from the server.
Usually this buffer is destroyed when it is empty and is larger than 16
kb in order to avoid wasting memory in unused buffers
KiB in order to avoid wasting memory in unused buffers
However when working with very big payloads destroying the buffer may slow
down performances considerably, so it is possible to modify the max size of
@@ -371,9 +396,9 @@ value for an idle buffer, so the buffer will never get freed.
For instance if you have a normal Redis context you can set the maximum idle
buffer to zero (unlimited) just with:
context->reader->maxbuf = 0;
```c
context->reader->maxbuf = 0;
```
This should be done only in order to maximize performances when working with
large payloads. The context should be set back to `REDIS_READER_MAX_BUF` again
as soon as possible in order to prevent allocation of useless memory.
@@ -381,4 +406,6 @@ as soon as possible in order to prevent allocation of useless memory.
## AUTHORS
Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and
Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license.
Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license.
Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and
Jan-Erik Rediger (janerik at fnordig dot com)
deps/hiredis/adapters/glib.h
Vendored
+153
View File
@@ -0,0 +1,153 @@
#ifndef __HIREDIS_GLIB_H__
#define __HIREDIS_GLIB_H__
#include <glib.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct
{
GSource source;
redisAsyncContext *ac;
GPollFD poll_fd;
} RedisSource;
static void
redis_source_add_read (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
source->poll_fd.events |= G_IO_IN;
g_main_context_wakeup(g_source_get_context((GSource *)data));
}
static void
redis_source_del_read (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
source->poll_fd.events &= ~G_IO_IN;
g_main_context_wakeup(g_source_get_context((GSource *)data));
}
static void
redis_source_add_write (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
source->poll_fd.events |= G_IO_OUT;
g_main_context_wakeup(g_source_get_context((GSource *)data));
}
static void
redis_source_del_write (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
source->poll_fd.events &= ~G_IO_OUT;
g_main_context_wakeup(g_source_get_context((GSource *)data));
}
static void
redis_source_cleanup (gpointer data)
{
RedisSource *source = (RedisSource *)data;
g_return_if_fail(source);
redis_source_del_read(source);
redis_source_del_write(source);
/*
* It is not our responsibility to remove ourself from the
* current main loop. However, we will remove the GPollFD.
*/
if (source->poll_fd.fd >= 0) {
g_source_remove_poll((GSource *)data, &source->poll_fd);
source->poll_fd.fd = -1;
}
}
static gboolean
redis_source_prepare (GSource *source,
gint *timeout_)
{
RedisSource *redis = (RedisSource *)source;
*timeout_ = -1;
return !!(redis->poll_fd.events & redis->poll_fd.revents);
}
static gboolean
redis_source_check (GSource *source)
{
RedisSource *redis = (RedisSource *)source;
return !!(redis->poll_fd.events & redis->poll_fd.revents);
}
static gboolean
redis_source_dispatch (GSource *source,
GSourceFunc callback,
gpointer user_data)
{
RedisSource *redis = (RedisSource *)source;
if ((redis->poll_fd.revents & G_IO_OUT)) {
redisAsyncHandleWrite(redis->ac);
redis->poll_fd.revents &= ~G_IO_OUT;
}
if ((redis->poll_fd.revents & G_IO_IN)) {
redisAsyncHandleRead(redis->ac);
redis->poll_fd.revents &= ~G_IO_IN;
}
if (callback) {
return callback(user_data);
}
return TRUE;
}
static void
redis_source_finalize (GSource *source)
{
RedisSource *redis = (RedisSource *)source;
if (redis->poll_fd.fd >= 0) {
g_source_remove_poll(source, &redis->poll_fd);
redis->poll_fd.fd = -1;
}
}
static GSource *
redis_source_new (redisAsyncContext *ac)
{
static GSourceFuncs source_funcs = {
.prepare = redis_source_prepare,
.check = redis_source_check,
.dispatch = redis_source_dispatch,
.finalize = redis_source_finalize,
};
redisContext *c = &ac->c;
RedisSource *source;
g_return_val_if_fail(ac != NULL, NULL);
source = (RedisSource *)g_source_new(&source_funcs, sizeof *source);
source->ac = ac;
source->poll_fd.fd = c->fd;
source->poll_fd.events = 0;
source->poll_fd.revents = 0;
g_source_add_poll((GSource *)source, &source->poll_fd);
ac->ev.addRead = redis_source_add_read;
ac->ev.delRead = redis_source_del_read;
ac->ev.addWrite = redis_source_add_write;
ac->ev.delWrite = redis_source_del_write;
ac->ev.cleanup = redis_source_cleanup;
ac->ev.data = source;
return (GSource *)source;
}
#endif /* __HIREDIS_GLIB_H__ */
deps/hiredis/adapters/ivykis.h
Vendored
+81
View File
@@ -0,0 +1,81 @@
#ifndef __HIREDIS_IVYKIS_H__
#define __HIREDIS_IVYKIS_H__
#include <iv.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct redisIvykisEvents {
redisAsyncContext *context;
struct iv_fd fd;
} redisIvykisEvents;
static void redisIvykisReadEvent(void *arg) {
redisAsyncContext *context = (redisAsyncContext *)arg;
redisAsyncHandleRead(context);
}
static void redisIvykisWriteEvent(void *arg) {
redisAsyncContext *context = (redisAsyncContext *)arg;
redisAsyncHandleWrite(context);
}
static void redisIvykisAddRead(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_set_handler_in(&e->fd, redisIvykisReadEvent);
}
static void redisIvykisDelRead(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_set_handler_in(&e->fd, NULL);
}
static void redisIvykisAddWrite(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_set_handler_out(&e->fd, redisIvykisWriteEvent);
}
static void redisIvykisDelWrite(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_set_handler_out(&e->fd, NULL);
}
static void redisIvykisCleanup(void *privdata) {
redisIvykisEvents *e = (redisIvykisEvents*)privdata;
iv_fd_unregister(&e->fd);
free(e);
}
static int redisIvykisAttach(redisAsyncContext *ac) {
redisContext *c = &(ac->c);
redisIvykisEvents *e;
/* Nothing should be attached when something is already attached */
if (ac->ev.data != NULL)
return REDIS_ERR;
/* Create container for context and r/w events */
e = (redisIvykisEvents*)malloc(sizeof(*e));
e->context = ac;
/* Register functions to start/stop listening for events */
ac->ev.addRead = redisIvykisAddRead;
ac->ev.delRead = redisIvykisDelRead;
ac->ev.addWrite = redisIvykisAddWrite;
ac->ev.delWrite = redisIvykisDelWrite;
ac->ev.cleanup = redisIvykisCleanup;
ac->ev.data = e;
/* Initialize and install read/write events */
IV_FD_INIT(&e->fd);
e->fd.fd = c->fd;
e->fd.handler_in = redisIvykisReadEvent;
e->fd.handler_out = redisIvykisWriteEvent;
e->fd.handler_err = NULL;
e->fd.cookie = e->context;
iv_fd_register(&e->fd);
return REDIS_OK;
}
#endif
deps/hiredis/adapters/libevent.h
Vendored
+12 -12
View File
@@ -30,13 +30,13 @@
#ifndef __HIREDIS_LIBEVENT_H__
#define __HIREDIS_LIBEVENT_H__
#include <event.h>
#include <event2/event.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct redisLibeventEvents {
redisAsyncContext *context;
struct event rev, wev;
struct event *rev, *wev;
} redisLibeventEvents;
static void redisLibeventReadEvent(int fd, short event, void *arg) {
@@ -53,28 +53,28 @@ static void redisLibeventWriteEvent(int fd, short event, void *arg) {
static void redisLibeventAddRead(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_add(&e->rev,NULL);
event_add(e->rev,NULL);
}
static void redisLibeventDelRead(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_del(&e->rev);
event_del(e->rev);
}
static void redisLibeventAddWrite(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_add(&e->wev,NULL);
event_add(e->wev,NULL);
}
static void redisLibeventDelWrite(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_del(&e->wev);
event_del(e->wev);
}
static void redisLibeventCleanup(void *privdata) {
redisLibeventEvents *e = (redisLibeventEvents*)privdata;
event_del(&e->rev);
event_del(&e->wev);
event_del(e->rev);
event_del(e->wev);
free(e);
}
@@ -99,10 +99,10 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) {
ac->ev.data = e;
/* Initialize and install read/write events */
event_set(&e->rev,c->fd,EV_READ,redisLibeventReadEvent,e);
event_set(&e->wev,c->fd,EV_WRITE,redisLibeventWriteEvent,e);
event_base_set(base,&e->rev);
event_base_set(base,&e->wev);
e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e);
e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e);
event_add(e->rev, NULL);
event_add(e->wev, NULL);
return REDIS_OK;
}
#endif
deps/hiredis/adapters/libuv.h
Vendored
+4 -3
View File
@@ -1,5 +1,6 @@
#ifndef __HIREDIS_LIBUV_H__
#define __HIREDIS_LIBUV_H__
#include <stdlib.h>
#include <uv.h>
#include "../hiredis.h"
#include "../async.h"
@@ -11,7 +12,6 @@ typedef struct redisLibuvEvents {
int events;
} redisLibuvEvents;
int redisLibuvAttach(redisAsyncContext*, uv_loop_t*);
static void redisLibuvPoll(uv_poll_t* handle, int status, int events) {
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
@@ -20,10 +20,10 @@ static void redisLibuvPoll(uv_poll_t* handle, int status, int events) {
return;
}
if (events & UV_READABLE) {
if (p->context != NULL && (events & UV_READABLE)) {
redisAsyncHandleRead(p->context);
}
if (events & UV_WRITABLE) {
if (p->context != NULL && (events & UV_WRITABLE)) {
redisAsyncHandleWrite(p->context);
}
}
@@ -83,6 +83,7 @@ static void on_close(uv_handle_t* handle) {
static void redisLibuvCleanup(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->context = NULL; // indicate that context might no longer exist
uv_close((uv_handle_t*)&p->handle, on_close);
}
deps/hiredis/adapters/macosx.h
Vendored
+114
View File
@@ -0,0 +1,114 @@
//
// Created by Дмитрий Бахвалов on 13.07.15.
// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved.
//
#ifndef __HIREDIS_MACOSX_H__
#define __HIREDIS_MACOSX_H__
#include <CoreFoundation/CoreFoundation.h>
#include "../hiredis.h"
#include "../async.h"
typedef struct {
redisAsyncContext *context;
CFSocketRef socketRef;
CFRunLoopSourceRef sourceRef;
} RedisRunLoop;
static int freeRedisRunLoop(RedisRunLoop* redisRunLoop) {
if( redisRunLoop != NULL ) {
if( redisRunLoop->sourceRef != NULL ) {
CFRunLoopSourceInvalidate(redisRunLoop->sourceRef);
CFRelease(redisRunLoop->sourceRef);
}
if( redisRunLoop->socketRef != NULL ) {
CFSocketInvalidate(redisRunLoop->socketRef);
CFRelease(redisRunLoop->socketRef);
}
free(redisRunLoop);
}
return REDIS_ERR;
}
static void redisMacOSAddRead(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack);
}
static void redisMacOSDelRead(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketReadCallBack);
}
static void redisMacOSAddWrite(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
CFSocketEnableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack);
}
static void redisMacOSDelWrite(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
CFSocketDisableCallBacks(redisRunLoop->socketRef, kCFSocketWriteCallBack);
}
static void redisMacOSCleanup(void *privdata) {
RedisRunLoop *redisRunLoop = (RedisRunLoop*)privdata;
freeRedisRunLoop(redisRunLoop);
}
static void redisMacOSAsyncCallback(CFSocketRef __unused s, CFSocketCallBackType callbackType, CFDataRef __unused address, const void __unused *data, void *info) {
redisAsyncContext* context = (redisAsyncContext*) info;
switch (callbackType) {
case kCFSocketReadCallBack:
redisAsyncHandleRead(context);
break;
case kCFSocketWriteCallBack:
redisAsyncHandleWrite(context);
break;
default:
break;
}
}
static int redisMacOSAttach(redisAsyncContext *redisAsyncCtx, CFRunLoopRef runLoop) {
redisContext *redisCtx = &(redisAsyncCtx->c);
/* Nothing should be attached when something is already attached */
if( redisAsyncCtx->ev.data != NULL ) return REDIS_ERR;
RedisRunLoop* redisRunLoop = (RedisRunLoop*) calloc(1, sizeof(RedisRunLoop));
if( !redisRunLoop ) return REDIS_ERR;
/* Setup redis stuff */
redisRunLoop->context = redisAsyncCtx;
redisAsyncCtx->ev.addRead = redisMacOSAddRead;
redisAsyncCtx->ev.delRead = redisMacOSDelRead;
redisAsyncCtx->ev.addWrite = redisMacOSAddWrite;
redisAsyncCtx->ev.delWrite = redisMacOSDelWrite;
redisAsyncCtx->ev.cleanup = redisMacOSCleanup;
redisAsyncCtx->ev.data = redisRunLoop;
/* Initialize and install read/write events */
CFSocketContext socketCtx = { 0, redisAsyncCtx, NULL, NULL, NULL };
redisRunLoop->socketRef = CFSocketCreateWithNative(NULL, redisCtx->fd,
kCFSocketReadCallBack | kCFSocketWriteCallBack,
redisMacOSAsyncCallback,
&socketCtx);
if( !redisRunLoop->socketRef ) return freeRedisRunLoop(redisRunLoop);
redisRunLoop->sourceRef = CFSocketCreateRunLoopSource(NULL, redisRunLoop->socketRef, 0);
if( !redisRunLoop->sourceRef ) return freeRedisRunLoop(redisRunLoop);
CFRunLoopAddSource(runLoop, redisRunLoop->sourceRef, kCFRunLoopDefaultMode);
return REDIS_OK;
}
#endif
deps/hiredis/adapters/qt.h
Vendored
+135
View File
@@ -0,0 +1,135 @@
/*-
* Copyright (C) 2014 Pietro Cerutti <gahr@gahr.ch>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef __HIREDIS_QT_H__
#define __HIREDIS_QT_H__
#include <QSocketNotifier>
#include "../async.h"
static void RedisQtAddRead(void *);
static void RedisQtDelRead(void *);
static void RedisQtAddWrite(void *);
static void RedisQtDelWrite(void *);
static void RedisQtCleanup(void *);
class RedisQtAdapter : public QObject {
Q_OBJECT
friend
void RedisQtAddRead(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->addRead();
}
friend
void RedisQtDelRead(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->delRead();
}
friend
void RedisQtAddWrite(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->addWrite();
}
friend
void RedisQtDelWrite(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->delWrite();
}
friend
void RedisQtCleanup(void * adapter) {
RedisQtAdapter * a = static_cast<RedisQtAdapter *>(adapter);
a->cleanup();
}
public:
RedisQtAdapter(QObject * parent = 0)
: QObject(parent), m_ctx(0), m_read(0), m_write(0) { }
~RedisQtAdapter() {
if (m_ctx != 0) {
m_ctx->ev.data = NULL;
}
}
int setContext(redisAsyncContext * ac) {
if (ac->ev.data != NULL) {
return REDIS_ERR;
}
m_ctx = ac;
m_ctx->ev.data = this;
m_ctx->ev.addRead = RedisQtAddRead;
m_ctx->ev.delRead = RedisQtDelRead;
m_ctx->ev.addWrite = RedisQtAddWrite;
m_ctx->ev.delWrite = RedisQtDelWrite;
m_ctx->ev.cleanup = RedisQtCleanup;
return REDIS_OK;
}
private:
void addRead() {
if (m_read) return;
m_read = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Read, 0);
connect(m_read, SIGNAL(activated(int)), this, SLOT(read()));
}
void delRead() {
if (!m_read) return;
delete m_read;
m_read = 0;
}
void addWrite() {
if (m_write) return;
m_write = new QSocketNotifier(m_ctx->c.fd, QSocketNotifier::Write, 0);
connect(m_write, SIGNAL(activated(int)), this, SLOT(write()));
}
void delWrite() {
if (!m_write) return;
delete m_write;
m_write = 0;
}
void cleanup() {
delRead();
delWrite();
}
private slots:
void read() { redisAsyncHandleRead(m_ctx); }
void write() { redisAsyncHandleWrite(m_ctx); }
private:
redisAsyncContext * m_ctx;
QSocketNotifier * m_read;
QSocketNotifier * m_write;
};
#endif /* !__HIREDIS_QT_H__ */
deps/hiredis/appveyor.yml
Vendored
+36
View File
@@ -0,0 +1,36 @@
# Appveyor configuration file for CI build of hiredis on Windows (under Cygwin)
environment:
matrix:
- CYG_ROOT: C:\cygwin64
CYG_SETUP: setup-x86_64.exe
CYG_MIRROR: http://cygwin.mirror.constant.com
CYG_CACHE: C:\cygwin64\var\cache\setup
CYG_BASH: C:\cygwin64\bin\bash
CC: gcc
- CYG_ROOT: C:\cygwin
CYG_SETUP: setup-x86.exe
CYG_MIRROR: http://cygwin.mirror.constant.com
CYG_CACHE: C:\cygwin\var\cache\setup
CYG_BASH: C:\cygwin\bin\bash
CC: gcc
TARGET: 32bit
TARGET_VARS: 32bit-vars
# Cache Cygwin files to speed up build
cache:
- '%CYG_CACHE%'
clone_depth: 1
# Attempt to ensure we don't try to convert line endings to Win32 CRLF as this will cause build to fail
init:
- git config --global core.autocrlf input
# Install needed build dependencies
install:
- ps: 'Start-FileDownload "http://cygwin.com/$env:CYG_SETUP" -FileName "$env:CYG_SETUP"'
- '%CYG_SETUP% --quiet-mode --no-shortcuts --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" --packages automake,bison,gcc-core,libtool,make,gettext-devel,gettext,intltool,pkg-config,clang,llvm > NUL 2>&1'
- '%CYG_BASH% -lc "cygcheck -dc cygwin"'
build_script:
- 'echo building...'
- '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0</dev/null; make LDFLAGS=$LDFLAGS CC=$CC $TARGET CFLAGS=$CFLAGS && make LDFLAGS=$LDFLAGS CC=$CC $TARGET_VARS hiredis-example"'
deps/hiredis/async.c
Vendored
+38 -13
View File
@@ -58,7 +58,7 @@
} while(0);
/* Forward declaration of function in hiredis.c */
void __redisAppendCommand(redisContext *c, char *cmd, size_t len);
int __redisAppendCommand(redisContext *c, const char *cmd, size_t len);
/* Functions managing dictionary of callbacks for pub/sub. */
static unsigned int callbackHash(const void *key) {
@@ -142,6 +142,9 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) {
/* We want the error field to be accessible directly instead of requiring
* an indirection to the redisContext struct. */
static void __redisAsyncCopyError(redisAsyncContext *ac) {
if (!ac)
return;
redisContext *c = &(ac->c);
ac->err = c->err;
ac->errstr = c->errstr;
@@ -173,6 +176,14 @@ redisAsyncContext *redisAsyncConnectBind(const char *ip, int port,
return ac;
}
redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
const char *source_addr) {
redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr);
redisAsyncContext *ac = redisAsyncInitialize(c);
__redisAsyncCopyError(ac);
return ac;
}
redisAsyncContext *redisAsyncConnectUnix(const char *path) {
redisContext *c;
redisAsyncContext *ac;
@@ -407,7 +418,8 @@ void redisProcessCallbacks(redisAsyncContext *ac) {
if (reply == NULL) {
/* When the connection is being disconnected and there are
* no more replies, this is the cue to really disconnect. */
if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0) {
if (c->flags & REDIS_DISCONNECTING && sdslen(c->obuf) == 0
&& ac->replies.head == NULL) {
__redisAsyncDisconnect(ac);
return;
}
@@ -477,7 +489,7 @@ void redisProcessCallbacks(redisAsyncContext *ac) {
}
/* Internal helper function to detect socket status the first time a read or
* write event fires. When connecting was not succesful, the connect callback
* write event fires. When connecting was not successful, the connect callback
* is called with a REDIS_ERR status and the context is free'd. */
static int __redisAsyncHandleConnect(redisAsyncContext *ac) {
redisContext *c = &(ac->c);
@@ -551,8 +563,8 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) {
/* Sets a pointer to the first argument and its length starting at p. Returns
* the number of bytes to skip to get to the following argument. */
static char *nextArgument(char *start, char **str, size_t *len) {
char *p = start;
static const char *nextArgument(const char *start, const char **str, size_t *len) {
const char *p = start;
if (p[0] != '$') {
p = strchr(p,'$');
if (p == NULL) return NULL;
@@ -568,14 +580,15 @@ static char *nextArgument(char *start, char **str, size_t *len) {
/* Helper function for the redisAsyncCommand* family of functions. Writes a
* formatted command to the output buffer and registers the provided callback
* function with the context. */
static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, char *cmd, size_t len) {
static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
redisContext *c = &(ac->c);
redisCallback cb;
int pvariant, hasnext;
char *cstr, *astr;
const char *cstr, *astr;
size_t clen, alen;
char *p;
const char *p;
sds sname;
int ret;
/* Don't accept new commands when the connection is about to be closed. */
if (c->flags & (REDIS_DISCONNECTING | REDIS_FREEING)) return REDIS_ERR;
@@ -599,9 +612,11 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void
while ((p = nextArgument(p,&astr,&alen)) != NULL) {
sname = sdsnewlen(astr,alen);
if (pvariant)
dictReplace(ac->sub.patterns,sname,&cb);
ret = dictReplace(ac->sub.patterns,sname,&cb);
else
dictReplace(ac->sub.channels,sname,&cb);
ret = dictReplace(ac->sub.channels,sname,&cb);
if (ret == 0) sdsfree(sname);
}
} else if (strncasecmp(cstr,"unsubscribe\r\n",13) == 0) {
/* It is only useful to call (P)UNSUBSCRIBE when the context is
@@ -637,6 +652,11 @@ int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdat
int len;
int status;
len = redisvFormatCommand(&cmd,format,ap);
/* We don't want to pass -1 or -2 to future functions as a length. */
if (len < 0)
return REDIS_ERR;
status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
free(cmd);
return status;
@@ -652,11 +672,16 @@ int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata
}
int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) {
char *cmd;
sds cmd;
int len;
int status;
len = redisFormatCommandArgv(&cmd,argc,argv,argvlen);
len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
free(cmd);
sdsfree(cmd);
return status;
}
int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len) {
int status = __redisAsyncCommand(ac,fn,privdata,cmd,len);
return status;
}
deps/hiredis/async.h
Vendored
+3
View File
@@ -103,6 +103,8 @@ typedef struct redisAsyncContext {
/* Functions that proxy to hiredis */
redisAsyncContext *redisAsyncConnect(const char *ip, int port);
redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr);
redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port,
const char *source_addr);
redisAsyncContext *redisAsyncConnectUnix(const char *path);
int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn);
int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn);
@@ -118,6 +120,7 @@ void redisAsyncHandleWrite(redisAsyncContext *ac);
int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap);
int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...);
int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen);
int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *cmd, size_t len);
#ifdef __cplusplus
}
deps/hiredis/dict.c
Vendored
+2 -2
View File
@@ -161,7 +161,7 @@ static int dictReplace(dict *ht, void *key, void *val) {
dictEntry *entry, auxentry;
/* Try to add the element. If the key
* does not exists dictAdd will suceed. */
* does not exists dictAdd will succeed. */
if (dictAdd(ht, key, val) == DICT_OK)
return 1;
/* It already exists, get the entry */
@@ -293,7 +293,7 @@ static void dictReleaseIterator(dictIterator *iter) {
/* Expand the hash table if needed */
static int _dictExpandIfNeeded(dict *ht) {
/* If the hash table is empty expand it to the intial size,
/* If the hash table is empty expand it to the initial size,
* if the table is "full" dobule its size. */
if (ht->size == 0)
return dictExpand(ht, DICT_HT_INITIAL_SIZE);
deps/hiredis/examples/example-glib.c
Vendored
+73
View File
@@ -0,0 +1,73 @@
#include <stdlib.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/glib.h>
static GMainLoop *mainloop;
static void
connect_cb (const redisAsyncContext *ac G_GNUC_UNUSED,
int status)
{
if (status != REDIS_OK) {
g_printerr("Failed to connect: %s\n", ac->errstr);
g_main_loop_quit(mainloop);
} else {
g_printerr("Connected...\n");
}
}
static void
disconnect_cb (const redisAsyncContext *ac G_GNUC_UNUSED,
int status)
{
if (status != REDIS_OK) {
g_error("Failed to disconnect: %s", ac->errstr);
} else {
g_printerr("Disconnected...\n");
g_main_loop_quit(mainloop);
}
}
static void
command_cb(redisAsyncContext *ac,
gpointer r,
gpointer user_data G_GNUC_UNUSED)
{
redisReply *reply = r;
if (reply) {
g_print("REPLY: %s\n", reply->str);
}
redisAsyncDisconnect(ac);
}
gint
main (gint argc G_GNUC_UNUSED,
gchar *argv[] G_GNUC_UNUSED)
{
redisAsyncContext *ac;
GMainContext *context = NULL;
GSource *source;
ac = redisAsyncConnect("127.0.0.1", 6379);
if (ac->err) {
g_printerr("%s\n", ac->errstr);
exit(EXIT_FAILURE);
}
source = redis_source_new(ac);
mainloop = g_main_loop_new(context, FALSE);
g_source_attach(source, context);
redisAsyncSetConnectCallback(ac, connect_cb);
redisAsyncSetDisconnectCallback(ac, disconnect_cb);
redisAsyncCommand(ac, command_cb, NULL, "SET key 1234");
redisAsyncCommand(ac, command_cb, NULL, "GET key");
g_main_loop_run(mainloop);
return EXIT_SUCCESS;
}
deps/hiredis/examples/example-ivykis.c
Vendored
+58
View File
@@ -0,0 +1,58 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/ivykis.h>
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
}
int main (int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
iv_init();
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
redisIvykisAttach(c);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
iv_main();
iv_deinit();
return 0;
}
deps/hiredis/examples/example-macosx.c
Vendored
+66
View File
@@ -0,0 +1,66 @@
//
// Created by Дмитрий Бахвалов on 13.07.15.
// Copyright (c) 2015 Dmitry Bakhvalov. All rights reserved.
//
#include <stdio.h>
#include <hiredis.h>
#include <async.h>
#include <adapters/macosx.h>
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
}
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
return;
}
CFRunLoopStop(CFRunLoopGetCurrent());
printf("Disconnected...\n");
}
int main (int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
CFRunLoopRef loop = CFRunLoopGetCurrent();
if( !loop ) {
printf("Error: Cannot get current run loop\n");
return 1;
}
redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379);
if (c->err) {
/* Let *c leak for now... */
printf("Error: %s\n", c->errstr);
return 1;
}
redisMacOSAttach(c, loop);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
CFRunLoopRun();
return 0;
}
deps/hiredis/examples/example-qt.cpp
Vendored
+46
View File
@@ -0,0 +1,46 @@
#include <iostream>
using namespace std;
#include <QCoreApplication>
#include <QTimer>
#include "example-qt.h"
void getCallback(redisAsyncContext *, void * r, void * privdata) {
redisReply * reply = static_cast<redisReply *>(r);
ExampleQt * ex = static_cast<ExampleQt *>(privdata);
if (reply == nullptr || ex == nullptr) return;
cout << "key: " << reply->str << endl;
ex->finish();
}
void ExampleQt::run() {
m_ctx = redisAsyncConnect("localhost", 6379);
if (m_ctx->err) {
cerr << "Error: " << m_ctx->errstr << endl;
redisAsyncFree(m_ctx);
emit finished();
}
m_adapter.setContext(m_ctx);
redisAsyncCommand(m_ctx, NULL, NULL, "SET key %s", m_value);
redisAsyncCommand(m_ctx, getCallback, this, "GET key");
}
int main (int argc, char **argv) {
QCoreApplication app(argc, argv);
ExampleQt example(argv[argc-1]);
QObject::connect(&example, SIGNAL(finished()), &app, SLOT(quit()));
QTimer::singleShot(0, &example, SLOT(run()));
return app.exec();
}
deps/hiredis/examples/example-qt.h
Vendored
+32
View File
@@ -0,0 +1,32 @@
#ifndef __HIREDIS_EXAMPLE_QT_H
#define __HIREDIS_EXAMPLE_QT_H
#include <adapters/qt.h>
class ExampleQt : public QObject {
Q_OBJECT
public:
ExampleQt(const char * value, QObject * parent = 0)
: QObject(parent), m_value(value) {}
signals:
void finished();
public slots:
void run();
private:
void finish() { emit finished(); }
private:
const char * m_value;
redisAsyncContext * m_ctx;
RedisQtAdapter m_adapter;
friend
void getCallback(redisAsyncContext *, void *, void *);
};
#endif /* !__HIREDIS_EXAMPLE_QT_H */
deps/hiredis/examples/example.c
Vendored
+1 -1
View File
@@ -57,7 +57,7 @@ int main(int argc, char **argv) {
for (j = 0; j < 10; j++) {
char buf[64];
snprintf(buf,64,"%d",j);
snprintf(buf,64,"%u",j);
reply = redisCommand(c,"LPUSH mylist element-%s", buf);
freeReplyObject(reply);
}
deps/hiredis/fmacros.h
Vendored
+8 -7
View File
@@ -1,23 +1,24 @@
#ifndef __HIREDIS_FMACRO_H
#define __HIREDIS_FMACRO_H
#if !defined(_BSD_SOURCE)
#if defined(__linux__)
#define _BSD_SOURCE
#define _DEFAULT_SOURCE
#endif
#if defined(_AIX)
#define _ALL_SOURCE
#if defined(__CYGWIN__)
#include <sys/cdefs.h>
#endif
#if defined(__sun__)
#define _POSIX_C_SOURCE 200112L
#elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__)
#define _XOPEN_SOURCE 600
#else
#define _XOPEN_SOURCE
#if !(defined(__APPLE__) && defined(__MACH__)) && !(defined(__FreeBSD__))
#define _XOPEN_SOURCE 600
#endif
#endif
#if __APPLE__ && __MACH__
#if defined(__APPLE__) && defined(__MACH__)
#define _OSX
#endif
deps/hiredis/hiredis.c
Vendored
+191 -528
View File
@@ -1,6 +1,8 @@
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
* Jan-Erik Rediger <janerik at fnordig dot com>
*
* All rights reserved.
*
@@ -73,6 +75,9 @@ void freeReplyObject(void *reply) {
redisReply *r = reply;
size_t j;
if (r == NULL)
return;
switch(r->type) {
case REDIS_REPLY_INTEGER:
break; /* Nothing to free */
@@ -183,504 +188,23 @@ static void *createNilObject(const redisReadTask *task) {
return r;
}
static void __redisReaderSetError(redisReader *r, int type, const char *str) {
size_t len;
if (r->reply != NULL && r->fn && r->fn->freeObject) {
r->fn->freeObject(r->reply);
r->reply = NULL;
}
/* Clear input buffer on errors. */
if (r->buf != NULL) {
sdsfree(r->buf);
r->buf = NULL;
r->pos = r->len = 0;
}
/* Reset task stack. */
r->ridx = -1;
/* Set error. */
r->err = type;
len = strlen(str);
len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1);
memcpy(r->errstr,str,len);
r->errstr[len] = '\0';
}
static size_t chrtos(char *buf, size_t size, char byte) {
size_t len = 0;
switch(byte) {
case '\\':
case '"':
len = snprintf(buf,size,"\"\\%c\"",byte);
break;
case '\n': len = snprintf(buf,size,"\"\\n\""); break;
case '\r': len = snprintf(buf,size,"\"\\r\""); break;
case '\t': len = snprintf(buf,size,"\"\\t\""); break;
case '\a': len = snprintf(buf,size,"\"\\a\""); break;
case '\b': len = snprintf(buf,size,"\"\\b\""); break;
default:
if (isprint(byte))
len = snprintf(buf,size,"\"%c\"",byte);
else
len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte);
break;
}
return len;
}
static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) {
char cbuf[8], sbuf[128];
chrtos(cbuf,sizeof(cbuf),byte);
snprintf(sbuf,sizeof(sbuf),
"Protocol error, got %s as reply type byte", cbuf);
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf);
}
static void __redisReaderSetErrorOOM(redisReader *r) {
__redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory");
}
static char *readBytes(redisReader *r, unsigned int bytes) {
char *p;
if (r->len-r->pos >= bytes) {
p = r->buf+r->pos;
r->pos += bytes;
return p;
}
return NULL;
}
/* Find pointer to \r\n. */
static char *seekNewline(char *s, size_t len) {
int pos = 0;
int _len = len-1;
/* Position should be < len-1 because the character at "pos" should be
* followed by a \n. Note that strchr cannot be used because it doesn't
* allow to search a limited length and the buffer that is being searched
* might not have a trailing NULL character. */
while (pos < _len) {
while(pos < _len && s[pos] != '\r') pos++;
if (s[pos] != '\r') {
/* Not found. */
return NULL;
} else {
if (s[pos+1] == '\n') {
/* Found. */
return s+pos;
} else {
/* Continue searching. */
pos++;
}
}
}
return NULL;
}
/* Read a long long value starting at *s, under the assumption that it will be
* terminated by \r\n. Ambiguously returns -1 for unexpected input. */
static long long readLongLong(char *s) {
long long v = 0;
int dec, mult = 1;
char c;
if (*s == '-') {
mult = -1;
s++;
} else if (*s == '+') {
mult = 1;
s++;
}
while ((c = *(s++)) != '\r') {
dec = c - '0';
if (dec >= 0 && dec < 10) {
v *= 10;
v += dec;
} else {
/* Should not happen... */
return -1;
}
}
return mult*v;
}
static char *readLine(redisReader *r, int *_len) {
char *p, *s;
int len;
p = r->buf+r->pos;
s = seekNewline(p,(r->len-r->pos));
if (s != NULL) {
len = s-(r->buf+r->pos);
r->pos += len+2; /* skip \r\n */
if (_len) *_len = len;
return p;
}
return NULL;
}
static void moveToNextTask(redisReader *r) {
redisReadTask *cur, *prv;
while (r->ridx >= 0) {
/* Return a.s.a.p. when the stack is now empty. */
if (r->ridx == 0) {
r->ridx--;
return;
}
cur = &(r->rstack[r->ridx]);
prv = &(r->rstack[r->ridx-1]);
assert(prv->type == REDIS_REPLY_ARRAY);
if (cur->idx == prv->elements-1) {
r->ridx--;
} else {
/* Reset the type because the next item can be anything */
assert(cur->idx < prv->elements);
cur->type = -1;
cur->elements = -1;
cur->idx++;
return;
}
}
}
static int processLineItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj;
char *p;
int len;
if ((p = readLine(r,&len)) != NULL) {
if (cur->type == REDIS_REPLY_INTEGER) {
if (r->fn && r->fn->createInteger)
obj = r->fn->createInteger(cur,readLongLong(p));
else
obj = (void*)REDIS_REPLY_INTEGER;
} else {
/* Type will be error or status. */
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,p,len);
else
obj = (void*)(size_t)(cur->type);
}
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
/* Set reply if this is the root object. */
if (r->ridx == 0) r->reply = obj;
moveToNextTask(r);
return REDIS_OK;
}
return REDIS_ERR;
}
static int processBulkItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj = NULL;
char *p, *s;
long len;
unsigned long bytelen;
int success = 0;
p = r->buf+r->pos;
s = seekNewline(p,r->len-r->pos);
if (s != NULL) {
p = r->buf+r->pos;
bytelen = s-(r->buf+r->pos)+2; /* include \r\n */
len = readLongLong(p);
if (len < 0) {
/* The nil object can always be created. */
if (r->fn && r->fn->createNil)
obj = r->fn->createNil(cur);
else
obj = (void*)REDIS_REPLY_NIL;
success = 1;
} else {
/* Only continue when the buffer contains the entire bulk item. */
bytelen += len+2; /* include \r\n */
if (r->pos+bytelen <= r->len) {
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,s+2,len);
else
obj = (void*)REDIS_REPLY_STRING;
success = 1;
}
}
/* Proceed when obj was created. */
if (success) {
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
r->pos += bytelen;
/* Set reply if this is the root object. */
if (r->ridx == 0) r->reply = obj;
moveToNextTask(r);
return REDIS_OK;
}
}
return REDIS_ERR;
}
static int processMultiBulkItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj;
char *p;
long elements;
int root = 0;
/* Set error for nested multi bulks with depth > 7 */
if (r->ridx == 8) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"No support for nested multi bulk replies with depth > 7");
return REDIS_ERR;
}
if ((p = readLine(r,NULL)) != NULL) {
elements = readLongLong(p);
root = (r->ridx == 0);
if (elements == -1) {
if (r->fn && r->fn->createNil)
obj = r->fn->createNil(cur);
else
obj = (void*)REDIS_REPLY_NIL;
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
moveToNextTask(r);
} else {
if (r->fn && r->fn->createArray)
obj = r->fn->createArray(cur,elements);
else
obj = (void*)REDIS_REPLY_ARRAY;
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
/* Modify task stack when there are more than 0 elements. */
if (elements > 0) {
cur->elements = elements;
cur->obj = obj;
r->ridx++;
r->rstack[r->ridx].type = -1;
r->rstack[r->ridx].elements = -1;
r->rstack[r->ridx].idx = 0;
r->rstack[r->ridx].obj = NULL;
r->rstack[r->ridx].parent = cur;
r->rstack[r->ridx].privdata = r->privdata;
} else {
moveToNextTask(r);
}
}
/* Set reply if this is the root object. */
if (root) r->reply = obj;
return REDIS_OK;
}
return REDIS_ERR;
}
static int processItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
char *p;
/* check if we need to read type */
if (cur->type < 0) {
if ((p = readBytes(r,1)) != NULL) {
switch (p[0]) {
case '-':
cur->type = REDIS_REPLY_ERROR;
break;
case '+':
cur->type = REDIS_REPLY_STATUS;
break;
case ':':
cur->type = REDIS_REPLY_INTEGER;
break;
case '$':
cur->type = REDIS_REPLY_STRING;
break;
case '*':
cur->type = REDIS_REPLY_ARRAY;
break;
default:
__redisReaderSetErrorProtocolByte(r,*p);
return REDIS_ERR;
}
} else {
/* could not consume 1 byte */
return REDIS_ERR;
}
}
/* process typed item */
switch(cur->type) {
case REDIS_REPLY_ERROR:
case REDIS_REPLY_STATUS:
case REDIS_REPLY_INTEGER:
return processLineItem(r);
case REDIS_REPLY_STRING:
return processBulkItem(r);
case REDIS_REPLY_ARRAY:
return processMultiBulkItem(r);
default:
assert(NULL);
return REDIS_ERR; /* Avoid warning. */
}
}
redisReader *redisReaderCreate(void) {
redisReader *r;
r = calloc(sizeof(redisReader),1);
if (r == NULL)
return NULL;
r->err = 0;
r->errstr[0] = '\0';
r->fn = &defaultFunctions;
r->buf = sdsempty();
r->maxbuf = REDIS_READER_MAX_BUF;
if (r->buf == NULL) {
free(r);
return NULL;
}
r->ridx = -1;
return r;
}
void redisReaderFree(redisReader *r) {
if (r->reply != NULL && r->fn && r->fn->freeObject)
r->fn->freeObject(r->reply);
if (r->buf != NULL)
sdsfree(r->buf);
free(r);
}
int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
sds newbuf;
/* Return early when this reader is in an erroneous state. */
if (r->err)
return REDIS_ERR;
/* Copy the provided buffer. */
if (buf != NULL && len >= 1) {
/* Destroy internal buffer when it is empty and is quite large. */
if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) {
sdsfree(r->buf);
r->buf = sdsempty();
r->pos = 0;
/* r->buf should not be NULL since we just free'd a larger one. */
assert(r->buf != NULL);
}
newbuf = sdscatlen(r->buf,buf,len);
if (newbuf == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
r->buf = newbuf;
r->len = sdslen(r->buf);
}
return REDIS_OK;
}
int redisReaderGetReply(redisReader *r, void **reply) {
/* Default target pointer to NULL. */
if (reply != NULL)
*reply = NULL;
/* Return early when this reader is in an erroneous state. */
if (r->err)
return REDIS_ERR;
/* When the buffer is empty, there will never be a reply. */
if (r->len == 0)
return REDIS_OK;
/* Set first item to process when the stack is empty. */
if (r->ridx == -1) {
r->rstack[0].type = -1;
r->rstack[0].elements = -1;
r->rstack[0].idx = -1;
r->rstack[0].obj = NULL;
r->rstack[0].parent = NULL;
r->rstack[0].privdata = r->privdata;
r->ridx = 0;
}
/* Process items in reply. */
while (r->ridx >= 0)
if (processItem(r) != REDIS_OK)
break;
/* Return ASAP when an error occurred. */
if (r->err)
return REDIS_ERR;
/* Discard part of the buffer when we've consumed at least 1k, to avoid
* doing unnecessary calls to memmove() in sds.c. */
if (r->pos >= 1024) {
sdsrange(r->buf,r->pos,-1);
r->pos = 0;
r->len = sdslen(r->buf);
}
/* Emit a reply when there is one. */
if (r->ridx == -1) {
if (reply != NULL)
*reply = r->reply;
r->reply = NULL;
}
return REDIS_OK;
}
/* Calculate the number of bytes needed to represent an integer as string. */
static int intlen(int i) {
int len = 0;
if (i < 0) {
len++;
i = -i;
}
do {
len++;
i /= 10;
} while(i);
return len;
/* Return the number of digits of 'v' when converted to string in radix 10.
* Implementation borrowed from link in redis/src/util.c:string2ll(). */
static uint32_t countDigits(uint64_t v) {
uint32_t result = 1;
for (;;) {
if (v < 10) return result;
if (v < 100) return result + 1;
if (v < 1000) return result + 2;
if (v < 10000) return result + 3;
v /= 10000U;
result += 4;
}
}
/* Helper that calculates the bulk length given a certain string length. */
static size_t bulklen(size_t len) {
return 1+intlen(len)+2+len+2;
return 1+countDigits(len)+2+len+2;
}
int redisvFormatCommand(char **target, const char *format, va_list ap) {
@@ -692,6 +216,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
char **curargv = NULL, **newargv = NULL;
int argc = 0;
int totlen = 0;
int error_type = 0; /* 0 = no error; -1 = memory error; -2 = format error */
int j;
/* Abort if there is not target to set */
@@ -708,19 +233,19 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
if (*c == ' ') {
if (touched) {
newargv = realloc(curargv,sizeof(char*)*(argc+1));
if (newargv == NULL) goto err;
if (newargv == NULL) goto memory_err;
curargv = newargv;
curargv[argc++] = curarg;
totlen += bulklen(sdslen(curarg));
/* curarg is put in argv so it can be overwritten. */
curarg = sdsempty();
if (curarg == NULL) goto err;
if (curarg == NULL) goto memory_err;
touched = 0;
}
} else {
newarg = sdscatlen(curarg,c,1);
if (newarg == NULL) goto err;
if (newarg == NULL) goto memory_err;
curarg = newarg;
touched = 1;
}
@@ -751,17 +276,14 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
/* Try to detect printf format */
{
static const char intfmts[] = "diouxX";
static const char flags[] = "#0-+ ";
char _format[16];
const char *_p = c+1;
size_t _l = 0;
va_list _cpy;
/* Flags */
if (*_p != '\0' && *_p == '#') _p++;
if (*_p != '\0' && *_p == '0') _p++;
if (*_p != '\0' && *_p == '-') _p++;
if (*_p != '\0' && *_p == ' ') _p++;
if (*_p != '\0' && *_p == '+') _p++;
while (*_p != '\0' && strchr(flags,*_p) != NULL) _p++;
/* Field width */
while (*_p != '\0' && isdigit(*_p)) _p++;
@@ -829,7 +351,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
fmt_invalid:
va_end(_cpy);
goto err;
goto format_err;
fmt_valid:
_l = (_p+1)-c;
@@ -848,7 +370,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
}
}
if (newarg == NULL) goto err;
if (newarg == NULL) goto memory_err;
curarg = newarg;
touched = 1;
@@ -860,7 +382,7 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
/* Add the last argument if needed */
if (touched) {
newargv = realloc(curargv,sizeof(char*)*(argc+1));
if (newargv == NULL) goto err;
if (newargv == NULL) goto memory_err;
curargv = newargv;
curargv[argc++] = curarg;
totlen += bulklen(sdslen(curarg));
@@ -872,11 +394,11 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
curarg = NULL;
/* Add bytes needed to hold multi bulk count */
totlen += 1+intlen(argc)+2;
totlen += 1+countDigits(argc)+2;
/* Build the command at protocol level */
cmd = malloc(totlen+1);
if (cmd == NULL) goto err;
if (cmd == NULL) goto memory_err;
pos = sprintf(cmd,"*%d\r\n",argc);
for (j = 0; j < argc; j++) {
@@ -894,20 +416,29 @@ int redisvFormatCommand(char **target, const char *format, va_list ap) {
*target = cmd;
return totlen;
err:
while(argc--)
sdsfree(curargv[argc]);
free(curargv);
format_err:
error_type = -2;
goto cleanup;
if (curarg != NULL)
sdsfree(curarg);
memory_err:
error_type = -1;
goto cleanup;
cleanup:
if (curargv) {
while(argc--)
sdsfree(curargv[argc]);
free(curargv);
}
sdsfree(curarg);
/* No need to check cmd since it is the last statement that can fail,
* but do it anyway to be as defensive as possible. */
if (cmd != NULL)
free(cmd);
return -1;
return error_type;
}
/* Format a command according to the Redis protocol. This function
@@ -928,9 +459,69 @@ int redisFormatCommand(char **target, const char *format, ...) {
va_start(ap,format);
len = redisvFormatCommand(target,format,ap);
va_end(ap);
/* The API says "-1" means bad result, but we now also return "-2" in some
* cases. Force the return value to always be -1. */
if (len < 0)
len = -1;
return len;
}
/* Format a command according to the Redis protocol using an sds string and
* sdscatfmt for the processing of arguments. This function takes the
* number of arguments, an array with arguments and an array with their
* lengths. If the latter is set to NULL, strlen will be used to compute the
* argument lengths.
*/
int redisFormatSdsCommandArgv(sds *target, int argc, const char **argv,
const size_t *argvlen)
{
sds cmd;
unsigned long long totlen;
int j;
size_t len;
/* Abort on a NULL target */
if (target == NULL)
return -1;
/* Calculate our total size */
totlen = 1+countDigits(argc)+2;
for (j = 0; j < argc; j++) {
len = argvlen ? argvlen[j] : strlen(argv[j]);
totlen += bulklen(len);
}
/* Use an SDS string for command construction */
cmd = sdsempty();
if (cmd == NULL)
return -1;
/* We already know how much storage we need */
cmd = sdsMakeRoomFor(cmd, totlen);
if (cmd == NULL)
return -1;
/* Construct command */
cmd = sdscatfmt(cmd, "*%i\r\n", argc);
for (j=0; j < argc; j++) {
len = argvlen ? argvlen[j] : strlen(argv[j]);
cmd = sdscatfmt(cmd, "$%u\r\n", len);
cmd = sdscatlen(cmd, argv[j], len);
cmd = sdscatlen(cmd, "\r\n", sizeof("\r\n")-1);
}
assert(sdslen(cmd)==totlen);
*target = cmd;
return totlen;
}
void redisFreeSdsCommand(sds cmd) {
sdsfree(cmd);
}
/* Format a command according to the Redis protocol. This function takes the
* number of arguments, an array with arguments and an array with their
* lengths. If the latter is set to NULL, strlen will be used to compute the
@@ -942,8 +533,12 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz
size_t len;
int totlen, j;
/* Abort on a NULL target */
if (target == NULL)
return -1;
/* Calculate number of bytes needed for the command */
totlen = 1+intlen(argc)+2;
totlen = 1+countDigits(argc)+2;
for (j = 0; j < argc; j++) {
len = argvlen ? argvlen[j] : strlen(argv[j]);
totlen += bulklen(len);
@@ -970,6 +565,10 @@ int redisFormatCommandArgv(char **target, int argc, const char **argv, const siz
return totlen;
}
void redisFreeCommand(char *cmd) {
free(cmd);
}
void __redisSetError(redisContext *c, int type, const char *str) {
size_t len;
@@ -982,10 +581,14 @@ void __redisSetError(redisContext *c, int type, const char *str) {
} else {
/* Only REDIS_ERR_IO may lack a description! */
assert(type == REDIS_ERR_IO);
strerror_r(errno,c->errstr,sizeof(c->errstr));
__redis_strerror_r(errno, c->errstr, sizeof(c->errstr));
}
}
redisReader *redisReaderCreate(void) {
return redisReaderCreateWithFunctions(&defaultFunctions);
}
static redisContext *redisContextInit(void) {
redisContext *c;
@@ -997,24 +600,72 @@ static redisContext *redisContextInit(void) {
c->errstr[0] = '\0';
c->obuf = sdsempty();
c->reader = redisReaderCreate();
c->tcp.host = NULL;
c->tcp.source_addr = NULL;
c->unix_sock.path = NULL;
c->timeout = NULL;
if (c->obuf == NULL || c->reader == NULL) {
redisFree(c);
return NULL;
}
return c;
}
void redisFree(redisContext *c) {
if (c == NULL)
return;
if (c->fd > 0)
close(c->fd);
if (c->obuf != NULL)
sdsfree(c->obuf);
if (c->reader != NULL)
redisReaderFree(c->reader);
if (c->tcp.host)
free(c->tcp.host);
if (c->tcp.source_addr)
free(c->tcp.source_addr);
if (c->unix_sock.path)
free(c->unix_sock.path);
if (c->timeout)
free(c->timeout);
free(c);
}
int redisFreeKeepFd(redisContext *c) {
int fd = c->fd;
c->fd = -1;
redisFree(c);
return fd;
int fd = c->fd;
c->fd = -1;
redisFree(c);
return fd;
}
int redisReconnect(redisContext *c) {
c->err = 0;
memset(c->errstr, '\0', strlen(c->errstr));
if (c->fd > 0) {
close(c->fd);
}
sdsfree(c->obuf);
redisReaderFree(c->reader);
c->obuf = sdsempty();
c->reader = redisReaderCreate();
if (c->connection_type == REDIS_CONN_TCP) {
return redisContextConnectBindTcp(c, c->tcp.host, c->tcp.port,
c->timeout, c->tcp.source_addr);
} else if (c->connection_type == REDIS_CONN_UNIX) {
return redisContextConnectUnix(c, c->unix_sock.path, c->timeout);
} else {
/* Something bad happened here and shouldn't have. There isn't
enough information in the context to reconnect. */
__redisSetError(c,REDIS_ERR_OTHER,"Not enough information to reconnect");
}
return REDIS_ERR;
}
/* Connect to a Redis instance. On error the field error in the returned
@@ -1064,6 +715,15 @@ redisContext *redisConnectBindNonBlock(const char *ip, int port,
return c;
}
redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
const char *source_addr) {
redisContext *c = redisContextInit();
c->flags &= ~REDIS_BLOCK;
c->flags |= REDIS_REUSEADDR;
redisContextConnectBindTcp(c,ip,port,NULL,source_addr);
return c;
}
redisContext *redisConnectUnix(const char *path) {
redisContext *c;
@@ -1162,10 +822,10 @@ int redisBufferRead(redisContext *c) {
/* Write the output buffer to the socket.
*
* Returns REDIS_OK when the buffer is empty, or (a part of) the buffer was
* succesfully written to the socket. When the buffer is empty after the
* successfully written to the socket. When the buffer is empty after the
* write operation, "done" is set to 1 (if given).
*
* Returns REDIS_ERR if an error occured trying to write and sets
* Returns REDIS_ERR if an error occurred trying to write and sets
* c->errstr to hold the appropriate error string.
*/
int redisBufferWrite(redisContext *c, int *done) {
@@ -1274,6 +934,9 @@ int redisvAppendCommand(redisContext *c, const char *format, va_list ap) {
if (len == -1) {
__redisSetError(c,REDIS_ERR_OOM,"Out of memory");
return REDIS_ERR;
} else if (len == -2) {
__redisSetError(c,REDIS_ERR_OTHER,"Invalid format string");
return REDIS_ERR;
}
if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
@@ -1296,21 +959,21 @@ int redisAppendCommand(redisContext *c, const char *format, ...) {
}
int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
char *cmd;
sds cmd;
int len;
len = redisFormatCommandArgv(&cmd,argc,argv,argvlen);
len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
if (len == -1) {
__redisSetError(c,REDIS_ERR_OOM,"Out of memory");
return REDIS_ERR;
}
if (__redisAppendCommand(c,cmd,len) != REDIS_OK) {
free(cmd);
sdsfree(cmd);
return REDIS_ERR;
}
free(cmd);
sdsfree(cmd);
return REDIS_OK;
}
@@ -1321,7 +984,7 @@ int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const s
* context is non-blocking, the "reply" pointer will not be used and the
* command is simply appended to the write buffer.
*
* Returns the reply when a reply was succesfully retrieved. Returns NULL
* Returns the reply when a reply was successfully retrieved. Returns NULL
* otherwise. When NULL is returned in a blocking context, the error field
* in the context will be set.
*/
deps/hiredis/hiredis.h
Vendored
+78 -75
View File
@@ -1,6 +1,8 @@
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
* Jan-Erik Rediger <janerik at fnordig dot com>
*
* All rights reserved.
*
@@ -31,26 +33,16 @@
#ifndef __HIREDIS_H
#define __HIREDIS_H
#include <stdio.h> /* for size_t */
#include "read.h"
#include <stdarg.h> /* for va_list */
#include <sys/time.h> /* for struct timeval */
#include <stdint.h> /* uintXX_t, etc */
#include "sds.h" /* for sds */
#define HIREDIS_MAJOR 0
#define HIREDIS_MINOR 11
#define HIREDIS_PATCH 0
#define REDIS_ERR -1
#define REDIS_OK 0
/* When an error occurs, the err flag in a context is set to hold the type of
* error that occured. REDIS_ERR_IO means there was an I/O error and you
* should use the "errno" variable to find out what is wrong.
* For other values, the "errstr" field will hold a description. */
#define REDIS_ERR_IO 1 /* Error in read or write */
#define REDIS_ERR_EOF 3 /* End of file */
#define REDIS_ERR_PROTOCOL 4 /* Protocol error */
#define REDIS_ERR_OOM 5 /* Out of memory */
#define REDIS_ERR_OTHER 2 /* Everything else... */
#define HIREDIS_MINOR 13
#define HIREDIS_PATCH 3
#define HIREDIS_SONAME 0.13
/* Connection type can be blocking or non-blocking and is set in the
* least significant bit of the flags field in redisContext. */
@@ -79,17 +71,39 @@
/* Flag that is set when monitor mode is active */
#define REDIS_MONITORING 0x40
#define REDIS_REPLY_STRING 1
#define REDIS_REPLY_ARRAY 2
#define REDIS_REPLY_INTEGER 3
#define REDIS_REPLY_NIL 4
#define REDIS_REPLY_STATUS 5
#define REDIS_REPLY_ERROR 6
#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */
/* Flag that is set when we should set SO_REUSEADDR before calling bind() */
#define REDIS_REUSEADDR 0x80
#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */
/* number of times we retry to connect in the case of EADDRNOTAVAIL and
* SO_REUSEADDR is being used. */
#define REDIS_CONNECT_RETRIES 10
/* strerror_r has two completely different prototypes and behaviors
* depending on system issues, so we need to operate on the error buffer
* differently depending on which strerror_r we're using. */
#ifndef _GNU_SOURCE
/* "regular" POSIX strerror_r that does the right thing. */
#define __redis_strerror_r(errno, buf, len) \
do { \
strerror_r((errno), (buf), (len)); \
} while (0)
#else
/* "bad" GNU strerror_r we need to clean up after. */
#define __redis_strerror_r(errno, buf, len) \
do { \
char *err_str = strerror_r((errno), (buf), (len)); \
/* If return value _isn't_ the start of the buffer we passed in, \
* then GNU strerror_r returned an internal static buffer and we \
* need to copy the result into our private buffer. */ \
if (err_str != (buf)) { \
strncpy((buf), err_str, ((len) - 1)); \
buf[(len)-1] = '\0'; \
} \
} while (0)
#endif
#ifdef __cplusplus
extern "C" {
#endif
@@ -98,61 +112,13 @@ extern "C" {
typedef struct redisReply {
int type; /* REDIS_REPLY_* */
long long integer; /* The integer when type is REDIS_REPLY_INTEGER */
int len; /* Length of string */
size_t len; /* Length of string */
char *str; /* Used for both REDIS_REPLY_ERROR and REDIS_REPLY_STRING */
size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */
struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */
} redisReply;
typedef struct redisReadTask {
int type;
int elements; /* number of elements in multibulk container */
int idx; /* index in parent (array) object */
void *obj; /* holds user-generated value for a read task */
struct redisReadTask *parent; /* parent task */
void *privdata; /* user-settable arbitrary field */
} redisReadTask;
typedef struct redisReplyObjectFunctions {
void *(*createString)(const redisReadTask*, char*, size_t);
void *(*createArray)(const redisReadTask*, int);
void *(*createInteger)(const redisReadTask*, long long);
void *(*createNil)(const redisReadTask*);
void (*freeObject)(void*);
} redisReplyObjectFunctions;
/* State for the protocol parser */
typedef struct redisReader {
int err; /* Error flags, 0 when there is no error */
char errstr[128]; /* String representation of error when applicable */
char *buf; /* Read buffer */
size_t pos; /* Buffer cursor */
size_t len; /* Buffer length */
size_t maxbuf; /* Max length of unused buffer */
redisReadTask rstack[9];
int ridx; /* Index of current read task */
void *reply; /* Temporary reply pointer */
redisReplyObjectFunctions *fn;
void *privdata;
} redisReader;
/* Public API for the protocol parser. */
redisReader *redisReaderCreate(void);
void redisReaderFree(redisReader *r);
int redisReaderFeed(redisReader *r, const char *buf, size_t len);
int redisReaderGetReply(redisReader *r, void **reply);
/* Backwards compatibility, can be removed on big version bump. */
#define redisReplyReaderCreate redisReaderCreate
#define redisReplyReaderFree redisReaderFree
#define redisReplyReaderFeed redisReaderFeed
#define redisReplyReaderGetReply redisReaderGetReply
#define redisReplyReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p))
#define redisReplyReaderGetObject(_r) (((redisReader*)(_r))->reply)
#define redisReplyReaderGetError(_r) (((redisReader*)(_r))->errstr)
/* Function to free the reply objects hiredis returns by default. */
void freeReplyObject(void *reply);
@@ -161,6 +127,14 @@ void freeReplyObject(void *reply);
int redisvFormatCommand(char **target, const char *format, va_list ap);
int redisFormatCommand(char **target, const char *format, ...);
int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen);
int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen);
void redisFreeCommand(char *cmd);
void redisFreeSdsCommand(sds cmd);
enum redisConnectionType {
REDIS_CONN_TCP,
REDIS_CONN_UNIX
};
/* Context for a connection to Redis */
typedef struct redisContext {
@@ -170,16 +144,45 @@ typedef struct redisContext {
int flags;
char *obuf; /* Write buffer */
redisReader *reader; /* Protocol reader */
enum redisConnectionType connection_type;
struct timeval *timeout;
struct {
char *host;
char *source_addr;
int port;
} tcp;
struct {
char *path;
} unix_sock;
} redisContext;
redisContext *redisConnect(const char *ip, int port);
redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv);
redisContext *redisConnectNonBlock(const char *ip, int port);
redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr);
redisContext *redisConnectBindNonBlock(const char *ip, int port,
const char *source_addr);
redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port,
const char *source_addr);
redisContext *redisConnectUnix(const char *path);
redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv);
redisContext *redisConnectUnixNonBlock(const char *path);
redisContext *redisConnectFd(int fd);
/**
* Reconnect the given context using the saved information.
*
* This re-uses the exact same connect options as in the initial connection.
* host, ip (or path), timeout and bind address are reused,
* flags are used unmodified from the existing context.
*
* Returns REDIS_OK on successful connect or REDIS_ERR otherwise.
*/
int redisReconnect(redisContext *c);
int redisSetTimeout(redisContext *c, const struct timeval tv);
int redisEnableKeepAlive(redisContext *c);
void redisFree(redisContext *c);
deps/hiredis/net.c
Vendored
+113 -19
View File
@@ -1,7 +1,9 @@
/* Extracted from anet.c to work properly with Hiredis error reporting.
*
* Copyright (c) 2006-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
* Jan-Erik Rediger <janerik at fnordig dot com>
*
* All rights reserved.
*
@@ -47,6 +49,7 @@
#include <stdio.h>
#include <poll.h>
#include <limits.h>
#include <stdlib.h>
#include "net.h"
#include "sds.h"
@@ -67,7 +70,7 @@ static void __redisSetErrorFromErrno(redisContext *c, int type, const char *pref
if (prefix != NULL)
len = snprintf(buf,sizeof(buf),"%s: ",prefix);
strerror_r(errno,buf+len,sizeof(buf)-len);
__redis_strerror_r(errno, (char *)(buf + len), sizeof(buf) - len);
__redisSetError(c,type,buf);
}
@@ -138,7 +141,7 @@ int redisKeepAlive(redisContext *c, int interval) {
return REDIS_ERR;
}
#else
#ifndef __sun
#if defined(__GLIBC__) && !defined(__FreeBSD_kernel__)
val = interval;
if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) {
__redisSetError(c,REDIS_ERR_OTHER,strerror(errno));
@@ -175,19 +178,15 @@ static int redisSetTcpNoDelay(redisContext *c) {
#define __MAX_MSEC (((LONG_MAX) - 999) / 1000)
static int redisContextWaitReady(redisContext *c, const struct timeval *timeout) {
struct pollfd wfd[1];
long msec;
msec = -1;
wfd[0].fd = c->fd;
wfd[0].events = POLLOUT;
static int redisContextTimeoutMsec(redisContext *c, long *result)
{
const struct timeval *timeout = c->timeout;
long msec = -1;
/* Only use timeout when not NULL. */
if (timeout != NULL) {
if (timeout->tv_usec > 1000000 || timeout->tv_sec > __MAX_MSEC) {
__redisSetErrorFromErrno(c, REDIS_ERR_IO, NULL);
redisContextCloseFd(c);
*result = msec;
return REDIS_ERR;
}
@@ -198,6 +197,16 @@ static int redisContextWaitReady(redisContext *c, const struct timeval *timeout)
}
}
*result = msec;
return REDIS_OK;
}
static int redisContextWaitReady(redisContext *c, long msec) {
struct pollfd wfd[1];
wfd[0].fd = c->fd;
wfd[0].events = POLLOUT;
if (errno == EINPROGRESS) {
int res;
@@ -256,10 +265,57 @@ int redisContextSetTimeout(redisContext *c, const struct timeval tv) {
static int _redisContextConnectTcp(redisContext *c, const char *addr, int port,
const struct timeval *timeout,
const char *source_addr) {
int s, rv;
int s, rv, n;
char _port[6]; /* strlen("65535"); */
struct addrinfo hints, *servinfo, *bservinfo, *p, *b;
int blocking = (c->flags & REDIS_BLOCK);
int reuseaddr = (c->flags & REDIS_REUSEADDR);
int reuses = 0;
long timeout_msec = -1;
servinfo = NULL;
c->connection_type = REDIS_CONN_TCP;
c->tcp.port = port;
/* We need to take possession of the passed parameters
* to make them reusable for a reconnect.
* We also carefully check we don't free data we already own,
* as in the case of the reconnect method.
*
* This is a bit ugly, but atleast it works and doesn't leak memory.
**/
if (c->tcp.host != addr) {
if (c->tcp.host)
free(c->tcp.host);
c->tcp.host = strdup(addr);
}
if (timeout) {
if (c->timeout != timeout) {
if (c->timeout == NULL)
c->timeout = malloc(sizeof(struct timeval));
memcpy(c->timeout, timeout, sizeof(struct timeval));
}
} else {
if (c->timeout)
free(c->timeout);
c->timeout = NULL;
}
if (redisContextTimeoutMsec(c, &timeout_msec) != REDIS_OK) {
__redisSetError(c, REDIS_ERR_IO, "Invalid timeout specified");
goto error;
}
if (source_addr == NULL) {
free(c->tcp.source_addr);
c->tcp.source_addr = NULL;
} else if (c->tcp.source_addr != source_addr) {
free(c->tcp.source_addr);
c->tcp.source_addr = strdup(source_addr);
}
snprintf(_port, 6, "%d", port);
memset(&hints,0,sizeof(hints));
@@ -271,7 +327,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port,
* as this would add latency to every connect. Otherwise a more sensible
* route could be: Use IPv6 if both addresses are available and there is IPv6
* connectivity. */
if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) {
if ((rv = getaddrinfo(c->tcp.host,_port,&hints,&servinfo)) != 0) {
hints.ai_family = AF_INET6;
if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) {
__redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv));
@@ -279,21 +335,31 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port,
}
}
for (p = servinfo; p != NULL; p = p->ai_next) {
addrretry:
if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1)
continue;
c->fd = s;
if (redisSetBlocking(c,0) != REDIS_OK)
goto error;
if (source_addr) {
if (c->tcp.source_addr) {
int bound = 0;
/* Using getaddrinfo saves us from self-determining IPv4 vs IPv6 */
if ((rv = getaddrinfo(source_addr, NULL, &hints, &bservinfo)) != 0) {
if ((rv = getaddrinfo(c->tcp.source_addr, NULL, &hints, &bservinfo)) != 0) {
char buf[128];
snprintf(buf,sizeof(buf),"Can't get addr: %s",gai_strerror(rv));
__redisSetError(c,REDIS_ERR_OTHER,buf);
goto error;
}
if (reuseaddr) {
n = 1;
if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char*) &n,
sizeof(n)) < 0) {
goto error;
}
}
for (b = bservinfo; b != NULL; b = b->ai_next) {
if (bind(s,b->ai_addr,b->ai_addrlen) != -1) {
bound = 1;
@@ -314,8 +380,15 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port,
continue;
} else if (errno == EINPROGRESS && !blocking) {
/* This is ok. */
} else if (errno == EADDRNOTAVAIL && reuseaddr) {
if (++reuses >= REDIS_CONNECT_RETRIES) {
goto error;
} else {
redisContextCloseFd(c);
goto addrretry;
}
} else {
if (redisContextWaitReady(c,timeout) != REDIS_OK)
if (redisContextWaitReady(c,timeout_msec) != REDIS_OK)
goto error;
}
}
@@ -356,19 +429,40 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port,
int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) {
int blocking = (c->flags & REDIS_BLOCK);
struct sockaddr_un sa;
long timeout_msec = -1;
if (redisCreateSocket(c,AF_LOCAL) < 0)
return REDIS_ERR;
if (redisSetBlocking(c,0) != REDIS_OK)
return REDIS_ERR;
c->connection_type = REDIS_CONN_UNIX;
if (c->unix_sock.path != path)
c->unix_sock.path = strdup(path);
if (timeout) {
if (c->timeout != timeout) {
if (c->timeout == NULL)
c->timeout = malloc(sizeof(struct timeval));
memcpy(c->timeout, timeout, sizeof(struct timeval));
}
} else {
if (c->timeout)
free(c->timeout);
c->timeout = NULL;
}
if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK)
return REDIS_ERR;
sa.sun_family = AF_LOCAL;
strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1);
if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) {
if (errno == EINPROGRESS && !blocking) {
/* This is ok. */
} else {
if (redisContextWaitReady(c,timeout) != REDIS_OK)
if (redisContextWaitReady(c,timeout_msec) != REDIS_OK)
return REDIS_ERR;
}
}
deps/hiredis/net.h
Vendored
+5 -3
View File
@@ -1,7 +1,9 @@
/* Extracted from anet.c to work properly with Hiredis error reporting.
*
* Copyright (c) 2006-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2014, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2015, Matt Stancliff <matt at genges dot com>,
* Jan-Erik Rediger <janerik at fnordig dot com>
*
* All rights reserved.
*
@@ -35,7 +37,7 @@
#include "hiredis.h"
#if defined(__sun) || defined(_AIX)
#if defined(__sun)
#define AF_LOCAL AF_UNIX
#endif
deps/hiredis/read.c
Vendored
+525
View File
@@ -0,0 +1,525 @@
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "fmacros.h"
#include <string.h>
#include <stdlib.h>
#ifndef _MSC_VER
#include <unistd.h>
#endif
#include <assert.h>
#include <errno.h>
#include <ctype.h>
#include "read.h"
#include "sds.h"
static void __redisReaderSetError(redisReader *r, int type, const char *str) {
size_t len;
if (r->reply != NULL && r->fn && r->fn->freeObject) {
r->fn->freeObject(r->reply);
r->reply = NULL;
}
/* Clear input buffer on errors. */
if (r->buf != NULL) {
sdsfree(r->buf);
r->buf = NULL;
r->pos = r->len = 0;
}
/* Reset task stack. */
r->ridx = -1;
/* Set error. */
r->err = type;
len = strlen(str);
len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1);
memcpy(r->errstr,str,len);
r->errstr[len] = '\0';
}
static size_t chrtos(char *buf, size_t size, char byte) {
size_t len = 0;
switch(byte) {
case '\\':
case '"':
len = snprintf(buf,size,"\"\\%c\"",byte);
break;
case '\n': len = snprintf(buf,size,"\"\\n\""); break;
case '\r': len = snprintf(buf,size,"\"\\r\""); break;
case '\t': len = snprintf(buf,size,"\"\\t\""); break;
case '\a': len = snprintf(buf,size,"\"\\a\""); break;
case '\b': len = snprintf(buf,size,"\"\\b\""); break;
default:
if (isprint(byte))
len = snprintf(buf,size,"\"%c\"",byte);
else
len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte);
break;
}
return len;
}
static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) {
char cbuf[8], sbuf[128];
chrtos(cbuf,sizeof(cbuf),byte);
snprintf(sbuf,sizeof(sbuf),
"Protocol error, got %s as reply type byte", cbuf);
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf);
}
static void __redisReaderSetErrorOOM(redisReader *r) {
__redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory");
}
static char *readBytes(redisReader *r, unsigned int bytes) {
char *p;
if (r->len-r->pos >= bytes) {
p = r->buf+r->pos;
r->pos += bytes;
return p;
}
return NULL;
}
/* Find pointer to \r\n. */
static char *seekNewline(char *s, size_t len) {
int pos = 0;
int _len = len-1;
/* Position should be < len-1 because the character at "pos" should be
* followed by a \n. Note that strchr cannot be used because it doesn't
* allow to search a limited length and the buffer that is being searched
* might not have a trailing NULL character. */
while (pos < _len) {
while(pos < _len && s[pos] != '\r') pos++;
if (pos==_len) {
/* Not found. */
return NULL;
} else {
if (s[pos+1] == '\n') {
/* Found. */
return s+pos;
} else {
/* Continue searching. */
pos++;
}
}
}
return NULL;
}
/* Read a long long value starting at *s, under the assumption that it will be
* terminated by \r\n. Ambiguously returns -1 for unexpected input. */
static long long readLongLong(char *s) {
long long v = 0;
int dec, mult = 1;
char c;
if (*s == '-') {
mult = -1;
s++;
} else if (*s == '+') {
mult = 1;
s++;
}
while ((c = *(s++)) != '\r') {
dec = c - '0';
if (dec >= 0 && dec < 10) {
v *= 10;
v += dec;
} else {
/* Should not happen... */
return -1;
}
}
return mult*v;
}
static char *readLine(redisReader *r, int *_len) {
char *p, *s;
int len;
p = r->buf+r->pos;
s = seekNewline(p,(r->len-r->pos));
if (s != NULL) {
len = s-(r->buf+r->pos);
r->pos += len+2; /* skip \r\n */
if (_len) *_len = len;
return p;
}
return NULL;
}
static void moveToNextTask(redisReader *r) {
redisReadTask *cur, *prv;
while (r->ridx >= 0) {
/* Return a.s.a.p. when the stack is now empty. */
if (r->ridx == 0) {
r->ridx--;
return;
}
cur = &(r->rstack[r->ridx]);
prv = &(r->rstack[r->ridx-1]);
assert(prv->type == REDIS_REPLY_ARRAY);
if (cur->idx == prv->elements-1) {
r->ridx--;
} else {
/* Reset the type because the next item can be anything */
assert(cur->idx < prv->elements);
cur->type = -1;
cur->elements = -1;
cur->idx++;
return;
}
}
}
static int processLineItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj;
char *p;
int len;
if ((p = readLine(r,&len)) != NULL) {
if (cur->type == REDIS_REPLY_INTEGER) {
if (r->fn && r->fn->createInteger)
obj = r->fn->createInteger(cur,readLongLong(p));
else
obj = (void*)REDIS_REPLY_INTEGER;
} else {
/* Type will be error or status. */
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,p,len);
else
obj = (void*)(size_t)(cur->type);
}
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
/* Set reply if this is the root object. */
if (r->ridx == 0) r->reply = obj;
moveToNextTask(r);
return REDIS_OK;
}
return REDIS_ERR;
}
static int processBulkItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj = NULL;
char *p, *s;
long len;
unsigned long bytelen;
int success = 0;
p = r->buf+r->pos;
s = seekNewline(p,r->len-r->pos);
if (s != NULL) {
p = r->buf+r->pos;
bytelen = s-(r->buf+r->pos)+2; /* include \r\n */
len = readLongLong(p);
if (len < 0) {
/* The nil object can always be created. */
if (r->fn && r->fn->createNil)
obj = r->fn->createNil(cur);
else
obj = (void*)REDIS_REPLY_NIL;
success = 1;
} else {
/* Only continue when the buffer contains the entire bulk item. */
bytelen += len+2; /* include \r\n */
if (r->pos+bytelen <= r->len) {
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,s+2,len);
else
obj = (void*)REDIS_REPLY_STRING;
success = 1;
}
}
/* Proceed when obj was created. */
if (success) {
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
r->pos += bytelen;
/* Set reply if this is the root object. */
if (r->ridx == 0) r->reply = obj;
moveToNextTask(r);
return REDIS_OK;
}
}
return REDIS_ERR;
}
static int processMultiBulkItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj;
char *p;
long elements;
int root = 0;
/* Set error for nested multi bulks with depth > 7 */
if (r->ridx == 8) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"No support for nested multi bulk replies with depth > 7");
return REDIS_ERR;
}
if ((p = readLine(r,NULL)) != NULL) {
elements = readLongLong(p);
root = (r->ridx == 0);
if (elements == -1) {
if (r->fn && r->fn->createNil)
obj = r->fn->createNil(cur);
else
obj = (void*)REDIS_REPLY_NIL;
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
moveToNextTask(r);
} else {
if (r->fn && r->fn->createArray)
obj = r->fn->createArray(cur,elements);
else
obj = (void*)REDIS_REPLY_ARRAY;
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
/* Modify task stack when there are more than 0 elements. */
if (elements > 0) {
cur->elements = elements;
cur->obj = obj;
r->ridx++;
r->rstack[r->ridx].type = -1;
r->rstack[r->ridx].elements = -1;
r->rstack[r->ridx].idx = 0;
r->rstack[r->ridx].obj = NULL;
r->rstack[r->ridx].parent = cur;
r->rstack[r->ridx].privdata = r->privdata;
} else {
moveToNextTask(r);
}
}
/* Set reply if this is the root object. */
if (root) r->reply = obj;
return REDIS_OK;
}
return REDIS_ERR;
}
static int processItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
char *p;
/* check if we need to read type */
if (cur->type < 0) {
if ((p = readBytes(r,1)) != NULL) {
switch (p[0]) {
case '-':
cur->type = REDIS_REPLY_ERROR;
break;
case '+':
cur->type = REDIS_REPLY_STATUS;
break;
case ':':
cur->type = REDIS_REPLY_INTEGER;
break;
case '$':
cur->type = REDIS_REPLY_STRING;
break;
case '*':
cur->type = REDIS_REPLY_ARRAY;
break;
default:
__redisReaderSetErrorProtocolByte(r,*p);
return REDIS_ERR;
}
} else {
/* could not consume 1 byte */
return REDIS_ERR;
}
}
/* process typed item */
switch(cur->type) {
case REDIS_REPLY_ERROR:
case REDIS_REPLY_STATUS:
case REDIS_REPLY_INTEGER:
return processLineItem(r);
case REDIS_REPLY_STRING:
return processBulkItem(r);
case REDIS_REPLY_ARRAY:
return processMultiBulkItem(r);
default:
assert(NULL);
return REDIS_ERR; /* Avoid warning. */
}
}
redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) {
redisReader *r;
r = calloc(sizeof(redisReader),1);
if (r == NULL)
return NULL;
r->err = 0;
r->errstr[0] = '\0';
r->fn = fn;
r->buf = sdsempty();
r->maxbuf = REDIS_READER_MAX_BUF;
if (r->buf == NULL) {
free(r);
return NULL;
}
r->ridx = -1;
return r;
}
void redisReaderFree(redisReader *r) {
if (r->reply != NULL && r->fn && r->fn->freeObject)
r->fn->freeObject(r->reply);
if (r->buf != NULL)
sdsfree(r->buf);
free(r);
}
int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
sds newbuf;
/* Return early when this reader is in an erroneous state. */
if (r->err)
return REDIS_ERR;
/* Copy the provided buffer. */
if (buf != NULL && len >= 1) {
/* Destroy internal buffer when it is empty and is quite large. */
if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) {
sdsfree(r->buf);
r->buf = sdsempty();
r->pos = 0;
/* r->buf should not be NULL since we just free'd a larger one. */
assert(r->buf != NULL);
}
newbuf = sdscatlen(r->buf,buf,len);
if (newbuf == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
r->buf = newbuf;
r->len = sdslen(r->buf);
}
return REDIS_OK;
}
int redisReaderGetReply(redisReader *r, void **reply) {
/* Default target pointer to NULL. */
if (reply != NULL)
*reply = NULL;
/* Return early when this reader is in an erroneous state. */
if (r->err)
return REDIS_ERR;
/* When the buffer is empty, there will never be a reply. */
if (r->len == 0)
return REDIS_OK;
/* Set first item to process when the stack is empty. */
if (r->ridx == -1) {
r->rstack[0].type = -1;
r->rstack[0].elements = -1;
r->rstack[0].idx = -1;
r->rstack[0].obj = NULL;
r->rstack[0].parent = NULL;
r->rstack[0].privdata = r->privdata;
r->ridx = 0;
}
/* Process items in reply. */
while (r->ridx >= 0)
if (processItem(r) != REDIS_OK)
break;
/* Return ASAP when an error occurred. */
if (r->err)
return REDIS_ERR;
/* Discard part of the buffer when we've consumed at least 1k, to avoid
* doing unnecessary calls to memmove() in sds.c. */
if (r->pos >= 1024) {
sdsrange(r->buf,r->pos,-1);
r->pos = 0;
r->len = sdslen(r->buf);
}
/* Emit a reply when there is one. */
if (r->ridx == -1) {
if (reply != NULL)
*reply = r->reply;
r->reply = NULL;
}
return REDIS_OK;
}
deps/hiredis/read.h
Vendored
+111
View File
@@ -0,0 +1,111 @@
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __HIREDIS_READ_H
#define __HIREDIS_READ_H
#include <stdio.h> /* for size_t */
#define REDIS_ERR -1
#define REDIS_OK 0
/* When an error occurs, the err flag in a context is set to hold the type of
* error that occurred. REDIS_ERR_IO means there was an I/O error and you
* should use the "errno" variable to find out what is wrong.
* For other values, the "errstr" field will hold a description. */
#define REDIS_ERR_IO 1 /* Error in read or write */
#define REDIS_ERR_EOF 3 /* End of file */
#define REDIS_ERR_PROTOCOL 4 /* Protocol error */
#define REDIS_ERR_OOM 5 /* Out of memory */
#define REDIS_ERR_OTHER 2 /* Everything else... */
#define REDIS_REPLY_STRING 1
#define REDIS_REPLY_ARRAY 2
#define REDIS_REPLY_INTEGER 3
#define REDIS_REPLY_NIL 4
#define REDIS_REPLY_STATUS 5
#define REDIS_REPLY_ERROR 6
#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */
#ifdef __cplusplus
extern "C" {
#endif
typedef struct redisReadTask {
int type;
int elements; /* number of elements in multibulk container */
int idx; /* index in parent (array) object */
void *obj; /* holds user-generated value for a read task */
struct redisReadTask *parent; /* parent task */
void *privdata; /* user-settable arbitrary field */
} redisReadTask;
typedef struct redisReplyObjectFunctions {
void *(*createString)(const redisReadTask*, char*, size_t);
void *(*createArray)(const redisReadTask*, int);
void *(*createInteger)(const redisReadTask*, long long);
void *(*createNil)(const redisReadTask*);
void (*freeObject)(void*);
} redisReplyObjectFunctions;
typedef struct redisReader {
int err; /* Error flags, 0 when there is no error */
char errstr[128]; /* String representation of error when applicable */
char *buf; /* Read buffer */
size_t pos; /* Buffer cursor */
size_t len; /* Buffer length */
size_t maxbuf; /* Max length of unused buffer */
redisReadTask rstack[9];
int ridx; /* Index of current read task */
void *reply; /* Temporary reply pointer */
redisReplyObjectFunctions *fn;
void *privdata;
} redisReader;
/* Public API for the protocol parser. */
redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn);
void redisReaderFree(redisReader *r);
int redisReaderFeed(redisReader *r, const char *buf, size_t len);
int redisReaderGetReply(redisReader *r, void **reply);
#define redisReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p))
#define redisReaderGetObject(_r) (((redisReader*)(_r))->reply)
#define redisReaderGetError(_r) (((redisReader*)(_r))->errstr)
#ifdef __cplusplus
}
#endif
#endif
deps/hiredis/sds.c
Vendored
+2 -4
View File
@@ -89,9 +89,9 @@ sds sdsnewlen(const void *init, size_t initlen) {
unsigned char *fp; /* flags pointer. */
sh = s_malloc(hdrlen+initlen+1);
if (sh == NULL) return NULL;
if (!init)
memset(sh, 0, hdrlen+initlen+1);
if (sh == NULL) return NULL;
s = (char*)sh+hdrlen;
fp = ((unsigned char*)s)-1;
switch(type) {
@@ -577,14 +577,12 @@ sds sdscatprintf(sds s, const char *fmt, ...) {
* %% - Verbatim "%" character.
*/
sds sdscatfmt(sds s, char const *fmt, ...) {
size_t initlen = sdslen(s);
const char *f = fmt;
int i;
va_list ap;
va_start(ap,fmt);
f = fmt; /* Next format specifier byte to process. */
i = initlen; /* Position of the next byte to write to dest str. */
i = sdslen(s); /* Position of the next byte to write to dest str. */
while(*f) {
char next, *str;
size_t l;
deps/hiredis/sds.h
Vendored
+1 -1
View File
@@ -79,7 +79,7 @@ struct __attribute__ ((__packed__)) sdshdr64 {
#define SDS_TYPE_64 4
#define SDS_TYPE_MASK 7
#define SDS_TYPE_BITS 3
#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (void*)((s)-(sizeof(struct sdshdr##T)));
#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)));
#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
deps/hiredis/sdsalloc.h
Vendored
+4 -4
View File
@@ -1,6 +1,7 @@
/* SDSLib 2.0 -- A C dynamic strings library
*
* Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2015, Oran Agra
* Copyright (c) 2015, Redis Labs, Inc
* All rights reserved.
*
@@ -36,7 +37,6 @@
* the include of your alternate allocator if needed (not needed in order
* to use the default libc allocator). */
#include "zmalloc.h"
#define s_malloc zmalloc
#define s_realloc zrealloc
#define s_free zfree
#define s_malloc malloc
#define s_realloc realloc
#define s_free free
deps/hiredis/test.c
Vendored
+105 -15
View File
@@ -11,6 +11,7 @@
#include <limits.h>
#include "hiredis.h"
#include "net.h"
enum connection_type {
CONN_TCP,
@@ -29,7 +30,7 @@ struct config {
struct {
const char *path;
} unix;
} unix_sock;
};
/* The following lines make up our testing "framework" :) */
@@ -43,6 +44,13 @@ static long long usec(void) {
return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
}
/* The assert() calls below have side effects, so we need assert()
* even if we are compiling without asserts (-DNDEBUG). */
#ifdef NDEBUG
#undef assert
#define assert(e) (void)(e)
#endif
static redisContext *select_database(redisContext *c) {
redisReply *reply;
@@ -51,7 +59,7 @@ static redisContext *select_database(redisContext *c) {
assert(reply != NULL);
freeReplyObject(reply);
/* Make sure the DB is empty */
/* Make sure the DB is emtpy */
reply = redisCommand(c,"DBSIZE");
assert(reply != NULL);
if (reply->type == REDIS_REPLY_INTEGER && reply->integer == 0) {
@@ -89,10 +97,10 @@ static redisContext *connect(struct config config) {
if (config.type == CONN_TCP) {
c = redisConnect(config.tcp.host, config.tcp.port);
} else if (config.type == CONN_UNIX) {
c = redisConnectUnix(config.unix.path);
c = redisConnectUnix(config.unix_sock.path);
} else if (config.type == CONN_FD) {
/* Create a dummy connection just to get an fd to inherit */
redisContext *dummy_ctx = redisConnectUnix(config.unix.path);
redisContext *dummy_ctx = redisConnectUnix(config.unix_sock.path);
if (dummy_ctx) {
int fd = disconnect(dummy_ctx, 1);
printf("Connecting to inherited fd %d\n", fd);
@@ -107,6 +115,7 @@ static redisContext *connect(struct config config) {
exit(1);
} else if (c->err) {
printf("Connection error: %s\n", c->errstr);
redisFree(c);
exit(1);
}
@@ -215,6 +224,22 @@ static void test_format_commands(void) {
test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(7+2)+4+(3+2));
free(cmd);
sds sds_cmd;
sds_cmd = sdsempty();
test("Format command into sds by passing argc/argv without lengths: ");
len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL);
test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(3+2)+4+(3+2));
sdsfree(sds_cmd);
sds_cmd = sdsempty();
test("Format command into sds by passing argc/argv with lengths: ");
len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens);
test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(7+2)+4+(3+2));
sdsfree(sds_cmd);
}
static void test_append_formatted_commands(struct config config) {
@@ -318,16 +343,31 @@ static void test_reply_reader(void) {
redisReaderFree(reader);
}
static void test_free_null(void) {
void *redisCtx = NULL;
void *reply = NULL;
test("Don't fail when redisFree is passed a NULL value: ");
redisFree(redisCtx);
test_cond(redisCtx == NULL);
test("Don't fail when freeReplyObject is passed a NULL value: ");
freeReplyObject(reply);
test_cond(reply == NULL);
}
static void test_blocking_connection_errors(void) {
redisContext *c;
test("Returns error when host cannot be resolved: ");
c = redisConnect((char*)"idontexist.local", 6379);
c = redisConnect((char*)"idontexist.test", 6379);
test_cond(c->err == REDIS_ERR_OTHER &&
(strcmp(c->errstr,"Name or service not known") == 0 ||
strcmp(c->errstr,"Can't resolve: idontexist.local") == 0 ||
strcmp(c->errstr,"Can't resolve: idontexist.test") == 0 ||
strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 ||
strcmp(c->errstr,"No address associated with hostname") == 0 ||
strcmp(c->errstr,"Temporary failure in name resolution") == 0 ||
strcmp(c->errstr,"hostname nor servname provided, or not known") == 0 ||
strcmp(c->errstr,"no address associated with name") == 0));
redisFree(c);
@@ -337,7 +377,7 @@ static void test_blocking_connection_errors(void) {
strcmp(c->errstr,"Connection refused") == 0);
redisFree(c);
test("Returns error when the unix socket path doesn't accept connections: ");
test("Returns error when the unix_sock socket path doesn't accept connections: ");
c = redisConnectUnix((char*)"/tmp/idontexist.sock");
test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */
redisFree(c);
@@ -421,6 +461,52 @@ static void test_blocking_connection(struct config config) {
disconnect(c, 0);
}
static void test_blocking_connection_timeouts(struct config config) {
redisContext *c;
redisReply *reply;
ssize_t s;
const char *cmd = "DEBUG SLEEP 3\r\n";
struct timeval tv;
c = connect(config);
test("Successfully completes a command when the timeout is not exceeded: ");
reply = redisCommand(c,"SET foo fast");
freeReplyObject(reply);
tv.tv_sec = 0;
tv.tv_usec = 10000;
redisSetTimeout(c, tv);
reply = redisCommand(c, "GET foo");
test_cond(reply != NULL && reply->type == REDIS_REPLY_STRING && memcmp(reply->str, "fast", 4) == 0);
freeReplyObject(reply);
disconnect(c, 0);
c = connect(config);
test("Does not return a reply when the command times out: ");
s = write(c->fd, cmd, strlen(cmd));
tv.tv_sec = 0;
tv.tv_usec = 10000;
redisSetTimeout(c, tv);
reply = redisCommand(c, "GET foo");
test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0);
freeReplyObject(reply);
test("Reconnect properly reconnects after a timeout: ");
redisReconnect(c);
reply = redisCommand(c, "PING");
test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
freeReplyObject(reply);
test("Reconnect properly uses owned parameters: ");
config.tcp.host = "foo";
config.unix_sock.path = "foo";
redisReconnect(c);
reply = redisCommand(c, "PING");
test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
freeReplyObject(reply);
disconnect(c, 0);
}
static void test_blocking_io_errors(struct config config) {
redisContext *c;
redisReply *reply;
@@ -444,7 +530,7 @@ static void test_blocking_io_errors(struct config config) {
test("Returns I/O error when the connection is lost: ");
reply = redisCommand(c,"QUIT");
if (major >= 2 && minor > 0) {
if (major > 2 || (major == 2 && minor > 0)) {
/* > 2.0 returns OK on QUIT and read() should be issued once more
* to know the descriptor is at EOF. */
test_cond(strcasecmp(reply->str,"OK") == 0 &&
@@ -482,7 +568,8 @@ static void test_invalid_timeout_errors(struct config config) {
c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
test_cond(c->err == REDIS_ERR_IO);
test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
redisFree(c);
test("Set error when an invalid timeout sec value is given to redisConnectWithTimeout: ");
@@ -491,8 +578,7 @@ static void test_invalid_timeout_errors(struct config config) {
c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
test_cond(c->err == REDIS_ERR_IO);
test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
redisFree(c);
}
@@ -666,7 +752,7 @@ int main(int argc, char **argv) {
.host = "127.0.0.1",
.port = 6379
},
.unix = {
.unix_sock = {
.path = "/tmp/redis.sock"
}
};
@@ -687,7 +773,7 @@ int main(int argc, char **argv) {
cfg.tcp.port = atoi(argv[0]);
} else if (argc >= 2 && !strcmp(argv[0],"-s")) {
argv++; argc--;
cfg.unix.path = argv[0];
cfg.unix_sock.path = argv[0];
} else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) {
throughput = 0;
} else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) {
@@ -702,27 +788,31 @@ int main(int argc, char **argv) {
test_format_commands();
test_reply_reader();
test_blocking_connection_errors();
test_free_null();
printf("\nTesting against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port);
cfg.type = CONN_TCP;
test_blocking_connection(cfg);
test_blocking_connection_timeouts(cfg);
test_blocking_io_errors(cfg);
test_invalid_timeout_errors(cfg);
test_append_formatted_commands(cfg);
if (throughput) test_throughput(cfg);
printf("\nTesting against Unix socket connection (%s):\n", cfg.unix.path);
printf("\nTesting against Unix socket connection (%s):\n", cfg.unix_sock.path);
cfg.type = CONN_UNIX;
test_blocking_connection(cfg);
test_blocking_connection_timeouts(cfg);
test_blocking_io_errors(cfg);
if (throughput) test_throughput(cfg);
if (test_inherit_fd) {
printf("\nTesting against inherited fd (%s):\n", cfg.unix.path);
printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path);
cfg.type = CONN_FD;
test_blocking_connection(cfg);
}
if (fails) {
printf("*** %d TESTS FAILED ***\n", fails);
return 1;
deps/hiredis/win32.h
Vendored
+42
View File
@@ -0,0 +1,42 @@
#ifndef _WIN32_HELPER_INCLUDE
#define _WIN32_HELPER_INCLUDE
#ifdef _MSC_VER
#ifndef inline
#define inline __inline
#endif
#ifndef va_copy
#define va_copy(d,s) ((d) = (s))
#endif
#ifndef snprintf
#define snprintf c99_snprintf
__inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap)
{
int count = -1;
if (size != 0)
count = _vsnprintf_s(str, size, _TRUNCATE, format, ap);
if (count == -1)
count = _vscprintf(format, ap);
return count;
}
__inline int c99_snprintf(char* str, size_t size, const char* format, ...)
{
int count;
va_list ap;
va_start(ap, format);
count = c99_vsnprintf(str, size, format, ap);
va_end(ap);
return count;
}
#endif
#endif
#endif
deps/hiredis/zmalloc.h
Vendored
-13
View File
@@ -1,13 +0,0 @@
/* Drop in replacement for zmalloc.h in order to just use libc malloc without
* any wrappering. */
#ifndef ZMALLOC_H
#define ZMALLOC_H
#define zmalloc malloc
#define zrealloc realloc
#define zcalloc(x) calloc(x,1)
#define zfree free
#define zstrdup strdup
#endif
deps/jemalloc/.appveyor.yml
Vendored
+42
View File
@@ -0,0 +1,42 @@
version: '{build}'
environment:
matrix:
- MSYSTEM: MINGW64
CPU: x86_64
MSVC: amd64
- MSYSTEM: MINGW32
CPU: i686
MSVC: x86
- MSYSTEM: MINGW64
CPU: x86_64
- MSYSTEM: MINGW32
CPU: i686
- MSYSTEM: MINGW64
CPU: x86_64
MSVC: amd64
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW32
CPU: i686
MSVC: x86
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW64
CPU: x86_64
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW32
CPU: i686
CONFIG_FLAGS: --enable-debug
install:
- set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
- if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC%
- if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc
- pacman --noconfirm -Suy mingw-w64-%CPU%-make
build_script:
- bash -c "autoconf"
- bash -c "./configure $CONFIG_FLAGS"
- mingw32-make
- file lib/jemalloc.dll
- mingw32-make tests
- mingw32-make -k check
deps/jemalloc/.gitignore
+32 -14
View File
@@ -1,5 +1,3 @@
/*.gcov.*
/bin/jemalloc-config
/bin/jemalloc.sh
/bin/jeprof
@@ -21,10 +19,14 @@
/Makefile
/include/jemalloc/internal/jemalloc_internal.h
/include/jemalloc/internal/jemalloc_preamble.h
/include/jemalloc/internal/jemalloc_internal_defs.h
/include/jemalloc/internal/private_namespace.gen.h
/include/jemalloc/internal/private_namespace.h
/include/jemalloc/internal/private_unnamespace.h
/include/jemalloc/internal/private_namespace_jet.gen.h
/include/jemalloc/internal/private_namespace_jet.h
/include/jemalloc/internal/private_symbols.awk
/include/jemalloc/internal/private_symbols_jet.awk
/include/jemalloc/internal/public_namespace.h
/include/jemalloc/internal/public_symbols.txt
/include/jemalloc/internal/public_unnamespace.h
@@ -40,8 +42,9 @@
/include/jemalloc/jemalloc_typedefs.h
/src/*.[od]
/src/*.gcda
/src/*.gcno
/src/*.sym
/run_tests.out/
/test/test.sh
test/include/test/jemalloc_test.h
@@ -50,26 +53,41 @@ test/include/test/jemalloc_test_defs.h
/test/integration/[A-Za-z]*
!/test/integration/[A-Za-z]*.*
/test/integration/*.[od]
/test/integration/*.gcda
/test/integration/*.gcno
/test/integration/*.out
/test/integration/cpp/[A-Za-z]*
!/test/integration/cpp/[A-Za-z]*.*
/test/integration/cpp/*.[od]
/test/integration/cpp/*.out
/test/src/*.[od]
/test/src/*.gcda
/test/src/*.gcno
/test/stress/[A-Za-z]*
!/test/stress/[A-Za-z]*.*
/test/stress/*.[od]
/test/stress/*.gcda
/test/stress/*.gcno
/test/stress/*.out
/test/unit/[A-Za-z]*
!/test/unit/[A-Za-z]*.*
/test/unit/*.[od]
/test/unit/*.gcda
/test/unit/*.gcno
/test/unit/*.out
/VERSION
*.pdb
*.sdf
*.opendb
*.VC.db
*.opensdf
*.cachefile
*.suo
*.user
*.sln.docstates
*.tmp
.vs/
/msvc/Win32/
/msvc/x64/
/msvc/projects/*/*/Debug*/
/msvc/projects/*/*/Release*/
/msvc/projects/*/*/Win32/
/msvc/projects/*/*/x64/
deps/jemalloc/.travis.yml
Vendored
+156
View File
@@ -0,0 +1,156 @@
language: generic
dist: precise
matrix:
include:
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false,dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false,percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary,percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
before_script:
- autoconf
- ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS
- make -j3
- make -j3 tests
script:
- make check
deps/jemalloc/COPYING
Vendored
+2 -2
View File
@@ -1,10 +1,10 @@
Unless otherwise specified, files in the jemalloc source distribution are
subject to the following license:
--------------------------------------------------------------------------------
Copyright (C) 2002-2015 Jason Evans <jasone@canonware.com>.
Copyright (C) 2002-2018 Jason Evans <jasone@canonware.com>.
All rights reserved.
Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
Copyright (C) 2009-2015 Facebook, Inc. All rights reserved.
Copyright (C) 2009-2018 Facebook, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
deps/jemalloc/ChangeLog
Vendored
+595 -1
View File
@@ -4,6 +4,600 @@ brevity. Much more detail can be found in the git revision history:
https://github.com/jemalloc/jemalloc
* 5.1.0 (May 4th, 2018)
This release is primarily about fine-tuning, ranging from several new features
to numerous notable performance and portability enhancements. The release and
prior dev versions have been running in multiple large scale applications for
months, and the cumulative improvements are substantial in many cases.
Given the long and successful production runs, this release is likely a good
candidate for applications to upgrade, from both jemalloc 5.0 and before. For
performance-critical applications, the newly added TUNING.md provides
guidelines on jemalloc tuning.
New features:
- Implement transparent huge page support for internal metadata. (@interwq)
- Add opt.thp to allow enabling / disabling transparent huge pages for all
mappings. (@interwq)
- Add maximum background thread count option. (@djwatson)
- Allow prof_active to control opt.lg_prof_interval and prof.gdump.
(@interwq)
- Allow arena index lookup based on allocation addresses via mallctl.
(@lionkov)
- Allow disabling initial-exec TLS model. (@davidtgoldblatt, @KenMacD)
- Add opt.lg_extent_max_active_fit to set the max ratio between the size of
the active extent selected (to split off from) and the size of the requested
allocation. (@interwq, @davidtgoldblatt)
- Add retain_grow_limit to set the max size when growing virtual address
space. (@interwq)
- Add mallctl interfaces:
+ arena.<i>.retain_grow_limit (@interwq)
+ arenas.lookup (@lionkov)
+ max_background_threads (@djwatson)
+ opt.lg_extent_max_active_fit (@interwq)
+ opt.max_background_threads (@djwatson)
+ opt.metadata_thp (@interwq)
+ opt.thp (@interwq)
+ stats.metadata_thp (@interwq)
Portability improvements:
- Support GNU/kFreeBSD configuration. (@paravoid)
- Support m68k, nios2 and SH3 architectures. (@paravoid)
- Fall back to FD_CLOEXEC when O_CLOEXEC is unavailable. (@zonyitoo)
- Fix symbol listing for cross-compiling. (@tamird)
- Fix high bits computation on ARM. (@davidtgoldblatt, @paravoid)
- Disable the CPU_SPINWAIT macro for Power. (@davidtgoldblatt, @marxin)
- Fix MSVC 2015 & 2017 builds. (@rustyx)
- Improve RISC-V support. (@EdSchouten)
- Set name mangling script in strict mode. (@nicolov)
- Avoid MADV_HUGEPAGE on ARM. (@marxin)
- Modify configure to determine return value of strerror_r.
(@davidtgoldblatt, @cferris1000)
- Make sure CXXFLAGS is tested with CPP compiler. (@nehaljwani)
- Fix 32-bit build on MSVC. (@rustyx)
- Fix external symbol on MSVC. (@maksqwe)
- Avoid a printf format specifier warning. (@jasone)
- Add configure option --disable-initial-exec-tls which can allow jemalloc to
be dynamically loaded after program startup. (@davidtgoldblatt, @KenMacD)
- AArch64: Add ILP32 support. (@cmuellner)
- Add --with-lg-vaddr configure option to support cross compiling.
(@cmuellner, @davidtgoldblatt)
Optimizations and refactors:
- Improve active extent fit with extent_max_active_fit. This considerably
reduces fragmentation over time and improves virtual memory and metadata
usage. (@davidtgoldblatt, @interwq)
- Eagerly coalesce large extents to reduce fragmentation. (@interwq)
- sdallocx: only read size info when page aligned (i.e. possibly sampled),
which speeds up the sized deallocation path significantly. (@interwq)
- Avoid attempting new mappings for in place expansion with retain, since
it rarely succeeds in practice and causes high overhead. (@interwq)
- Refactor OOM handling in newImpl. (@wqfish)
- Add internal fine-grained logging functionality for debugging use.
(@davidtgoldblatt)
- Refactor arena / tcache interactions. (@davidtgoldblatt)
- Refactor extent management with dumpable flag. (@davidtgoldblatt)
- Add runtime detection of lazy purging. (@interwq)
- Use pairing heap instead of red-black tree for extents_avail. (@djwatson)
- Use sysctl on startup in FreeBSD. (@trasz)
- Use thread local prng state instead of atomic. (@djwatson)
- Make decay to always purge one more extent than before, because in
practice large extents are usually the ones that cross the decay threshold.
Purging the additional extent helps save memory as well as reduce VM
fragmentation. (@interwq)
- Fast division by dynamic values. (@davidtgoldblatt)
- Improve the fit for aligned allocation. (@interwq, @edwinsmith)
- Refactor extent_t bitpacking. (@rkmisra)
- Optimize the generated assembly for ticker operations. (@davidtgoldblatt)
- Convert stats printing to use a structured text emitter. (@davidtgoldblatt)
- Remove preserve_lru feature for extents management. (@djwatson)
- Consolidate two memory loads into one on the fast deallocation path.
(@davidtgoldblatt, @interwq)
Bug fixes (most of the issues are only relevant to jemalloc 5.0):
- Fix deadlock with multithreaded fork in OS X. (@davidtgoldblatt)
- Validate returned file descriptor before use. (@zonyitoo)
- Fix a few background thread initialization and shutdown issues. (@interwq)
- Fix an extent coalesce + decay race by taking both coalescing extents off
the LRU list. (@interwq)
- Fix potentially unbound increase during decay, caused by one thread keep
stashing memory to purge while other threads generating new pages. The
number of pages to purge is checked to prevent this. (@interwq)
- Fix a FreeBSD bootstrap assertion. (@strejda, @interwq)
- Handle 32 bit mutex counters. (@rkmisra)
- Fix a indexing bug when creating background threads. (@davidtgoldblatt,
@binliu19)
- Fix arguments passed to extent_init. (@yuleniwo, @interwq)
- Fix addresses used for ordering mutexes. (@rkmisra)
- Fix abort_conf processing during bootstrap. (@interwq)
- Fix include path order for out-of-tree builds. (@cmuellner)
Incompatible changes:
- Remove --disable-thp. (@interwq)
- Remove mallctl interfaces:
+ config.thp (@interwq)
Documentation:
- Add TUNING.md. (@interwq, @davidtgoldblatt, @djwatson)
* 5.0.1 (July 1, 2017)
This bugfix release fixes several issues, most of which are obscure enough
that typical applications are not impacted.
Bug fixes:
- Update decay->nunpurged before purging, in order to avoid potential update
races and subsequent incorrect purging volume. (@interwq)
- Only abort on dlsym(3) error if the failure impacts an enabled feature (lazy
locking and/or background threads). This mitigates an initialization
failure bug for which we still do not have a clear reproduction test case.
(@interwq)
- Modify tsd management so that it neither crashes nor leaks if a thread's
only allocation activity is to call free() after TLS destructors have been
executed. This behavior was observed when operating with GNU libc, and is
unlikely to be an issue with other libc implementations. (@interwq)
- Mask signals during background thread creation. This prevents signals from
being inadvertently delivered to background threads. (@jasone,
@davidtgoldblatt, @interwq)
- Avoid inactivity checks within background threads, in order to prevent
recursive mutex acquisition. (@interwq)
- Fix extent_grow_retained() to use the specified hooks when the
arena.<i>.extent_hooks mallctl is used to override the default hooks.
(@interwq)
- Add missing reentrancy support for custom extent hooks which allocate.
(@interwq)
- Post-fork(2), re-initialize the list of tcaches associated with each arena
to contain no tcaches except the forking thread's. (@interwq)
- Add missing post-fork(2) mutex reinitialization for extent_grow_mtx. This
fixes potential deadlocks after fork(2). (@interwq)
- Enforce minimum autoconf version (currently 2.68), since 2.63 is known to
generate corrupt configure scripts. (@jasone)
- Ensure that the configured page size (--with-lg-page) is no larger than the
configured huge page size (--with-lg-hugepage). (@jasone)
* 5.0.0 (June 13, 2017)
Unlike all previous jemalloc releases, this release does not use naturally
aligned "chunks" for virtual memory management, and instead uses page-aligned
"extents". This change has few externally visible effects, but the internal
impacts are... extensive. Many other internal changes combine to make this
the most cohesively designed version of jemalloc so far, with ample
opportunity for further enhancements.
Continuous integration is now an integral aspect of development thanks to the
efforts of @davidtgoldblatt, and the dev branch tends to remain reasonably
stable on the tested platforms (Linux, FreeBSD, macOS, and Windows). As a
side effect the official release frequency may decrease over time.
New features:
- Implement optional per-CPU arena support; threads choose which arena to use
based on current CPU rather than on fixed thread-->arena associations.
(@interwq)
- Implement two-phase decay of unused dirty pages. Pages transition from
dirty-->muzzy-->clean, where the first phase transition relies on
madvise(... MADV_FREE) semantics, and the second phase transition discards
pages such that they are replaced with demand-zeroed pages on next access.
(@jasone)
- Increase decay time resolution from seconds to milliseconds. (@jasone)
- Implement opt-in per CPU background threads, and use them for asynchronous
decay-driven unused dirty page purging. (@interwq)
- Add mutex profiling, which collects a variety of statistics useful for
diagnosing overhead/contention issues. (@interwq)
- Add C++ new/delete operator bindings. (@djwatson)
- Support manually created arena destruction, such that all data and metadata
are discarded. Add MALLCTL_ARENAS_DESTROYED for accessing merged stats
associated with destroyed arenas. (@jasone)
- Add MALLCTL_ARENAS_ALL as a fixed index for use in accessing
merged/destroyed arena statistics via mallctl. (@jasone)
- Add opt.abort_conf to optionally abort if invalid configuration options are
detected during initialization. (@interwq)
- Add opt.stats_print_opts, so that e.g. JSON output can be selected for the
stats dumped during exit if opt.stats_print is true. (@jasone)
- Add --with-version=VERSION for use when embedding jemalloc into another
project's git repository. (@jasone)
- Add --disable-thp to support cross compiling. (@jasone)
- Add --with-lg-hugepage to support cross compiling. (@jasone)
- Add mallctl interfaces (various authors):
+ background_thread
+ opt.abort_conf
+ opt.retain
+ opt.percpu_arena
+ opt.background_thread
+ opt.{dirty,muzzy}_decay_ms
+ opt.stats_print_opts
+ arena.<i>.initialized
+ arena.<i>.destroy
+ arena.<i>.{dirty,muzzy}_decay_ms
+ arena.<i>.extent_hooks
+ arenas.{dirty,muzzy}_decay_ms
+ arenas.bin.<i>.slab_size
+ arenas.nlextents
+ arenas.lextent.<i>.size
+ arenas.create
+ stats.background_thread.{num_threads,num_runs,run_interval}
+ stats.mutexes.{ctl,background_thread,prof,reset}.
{num_ops,num_spin_acq,num_wait,max_wait_time,total_wait_time,max_num_thds,
num_owner_switch}
+ stats.arenas.<i>.{dirty,muzzy}_decay_ms
+ stats.arenas.<i>.uptime
+ stats.arenas.<i>.{pmuzzy,base,internal,resident}
+ stats.arenas.<i>.{dirty,muzzy}_{npurge,nmadvise,purged}
+ stats.arenas.<i>.bins.<j>.{nslabs,reslabs,curslabs}
+ stats.arenas.<i>.bins.<j>.mutex.
{num_ops,num_spin_acq,num_wait,max_wait_time,total_wait_time,max_num_thds,
num_owner_switch}
+ stats.arenas.<i>.lextents.<j>.{nmalloc,ndalloc,nrequests,curlextents}
+ stats.arenas.i.mutexes.{large,extent_avail,extents_dirty,extents_muzzy,
extents_retained,decay_dirty,decay_muzzy,base,tcache_list}.
{num_ops,num_spin_acq,num_wait,max_wait_time,total_wait_time,max_num_thds,
num_owner_switch}
Portability improvements:
- Improve reentrant allocation support, such that deadlock is less likely if
e.g. a system library call in turn allocates memory. (@davidtgoldblatt,
@interwq)
- Support static linking of jemalloc with glibc. (@djwatson)
Optimizations and refactors:
- Organize virtual memory as "extents" of virtual memory pages, rather than as
naturally aligned "chunks", and store all metadata in arbitrarily distant
locations. This reduces virtual memory external fragmentation, and will
interact better with huge pages (not yet explicitly supported). (@jasone)
- Fold large and huge size classes together; only small and large size classes
remain. (@jasone)
- Unify the allocation paths, and merge most fast-path branching decisions.
(@davidtgoldblatt, @interwq)
- Embed per thread automatic tcache into thread-specific data, which reduces
conditional branches and dereferences. Also reorganize tcache to increase
fast-path data locality. (@interwq)
- Rewrite atomics to closely model the C11 API, convert various
synchronization from mutex-based to atomic, and use the explicit memory
ordering control to resolve various hypothetical races without increasing
synchronization overhead. (@davidtgoldblatt)
- Extensively optimize rtree via various methods:
+ Add multiple layers of rtree lookup caching, since rtree lookups are now
part of fast-path deallocation. (@interwq)
+ Determine rtree layout at compile time. (@jasone)
+ Make the tree shallower for common configurations. (@jasone)
+ Embed the root node in the top-level rtree data structure, thus avoiding
one level of indirection. (@jasone)
+ Further specialize leaf elements as compared to internal node elements,
and directly embed extent metadata needed for fast-path deallocation.
(@jasone)
+ Ignore leading always-zero address bits (architecture-specific).
(@jasone)
- Reorganize headers (ongoing work) to make them hermetic, and disentangle
various module dependencies. (@davidtgoldblatt)
- Convert various internal data structures such as size class metadata from
boot-time-initialized to compile-time-initialized. Propagate resulting data
structure simplifications, such as making arena metadata fixed-size.
(@jasone)
- Simplify size class lookups when constrained to size classes that are
multiples of the page size. This speeds lookups, but the primary benefit is
complexity reduction in code that was the source of numerous regressions.
(@jasone)
- Lock individual extents when possible for localized extent operations,
rather than relying on a top-level arena lock. (@davidtgoldblatt, @jasone)
- Use first fit layout policy instead of best fit, in order to improve
packing. (@jasone)
- If munmap(2) is not in use, use an exponential series to grow each arena's
virtual memory, so that the number of disjoint virtual memory mappings
remains low. (@jasone)
- Implement per arena base allocators, so that arenas never share any virtual
memory pages. (@jasone)
- Automatically generate private symbol name mangling macros. (@jasone)
Incompatible changes:
- Replace chunk hooks with an expanded/normalized set of extent hooks.
(@jasone)
- Remove ratio-based purging. (@jasone)
- Remove --disable-tcache. (@jasone)
- Remove --disable-tls. (@jasone)
- Remove --enable-ivsalloc. (@jasone)
- Remove --with-lg-size-class-group. (@jasone)
- Remove --with-lg-tiny-min. (@jasone)
- Remove --disable-cc-silence. (@jasone)
- Remove --enable-code-coverage. (@jasone)
- Remove --disable-munmap (replaced by opt.retain). (@jasone)
- Remove Valgrind support. (@jasone)
- Remove quarantine support. (@jasone)
- Remove redzone support. (@jasone)
- Remove mallctl interfaces (various authors):
+ config.munmap
+ config.tcache
+ config.tls
+ config.valgrind
+ opt.lg_chunk
+ opt.purge
+ opt.lg_dirty_mult
+ opt.decay_time
+ opt.quarantine
+ opt.redzone
+ opt.thp
+ arena.<i>.lg_dirty_mult
+ arena.<i>.decay_time
+ arena.<i>.chunk_hooks
+ arenas.initialized
+ arenas.lg_dirty_mult
+ arenas.decay_time
+ arenas.bin.<i>.run_size
+ arenas.nlruns
+ arenas.lrun.<i>.size
+ arenas.nhchunks
+ arenas.hchunk.<i>.size
+ arenas.extend
+ stats.cactive
+ stats.arenas.<i>.lg_dirty_mult
+ stats.arenas.<i>.decay_time
+ stats.arenas.<i>.metadata.{mapped,allocated}
+ stats.arenas.<i>.{npurge,nmadvise,purged}
+ stats.arenas.<i>.huge.{allocated,nmalloc,ndalloc,nrequests}
+ stats.arenas.<i>.bins.<j>.{nruns,reruns,curruns}
+ stats.arenas.<i>.lruns.<j>.{nmalloc,ndalloc,nrequests,curruns}
+ stats.arenas.<i>.hchunks.<j>.{nmalloc,ndalloc,nrequests,curhchunks}
Bug fixes:
- Improve interval-based profile dump triggering to dump only one profile when
a single allocation's size exceeds the interval. (@jasone)
- Use prefixed function names (as controlled by --with-jemalloc-prefix) when
pruning backtrace frames in jeprof. (@jasone)
* 4.5.0 (February 28, 2017)
This is the first release to benefit from much broader continuous integration
testing, thanks to @davidtgoldblatt. Had we had this testing infrastructure
in place for prior releases, it would have caught all of the most serious
regressions fixed by this release.
New features:
- Add --disable-thp and the opt.thp mallctl to provide opt-out mechanisms for
transparent huge page integration. (@jasone)
- Update zone allocator integration to work with macOS 10.12. (@glandium)
- Restructure *CFLAGS configuration, so that CFLAGS behaves typically, and
EXTRA_CFLAGS provides a way to specify e.g. -Werror during building, but not
during configuration. (@jasone, @ronawho)
Bug fixes:
- Fix DSS (sbrk(2)-based) allocation. This regression was first released in
4.3.0. (@jasone)
- Handle race in per size class utilization computation. This functionality
was first released in 4.0.0. (@interwq)
- Fix lock order reversal during gdump. (@jasone)
- Fix/refactor tcache synchronization. This regression was first released in
4.0.0. (@jasone)
- Fix various JSON-formatted malloc_stats_print() bugs. This functionality
was first released in 4.3.0. (@jasone)
- Fix huge-aligned allocation. This regression was first released in 4.4.0.
(@jasone)
- When transparent huge page integration is enabled, detect what state pages
start in according to the kernel's current operating mode, and only convert
arena chunks to non-huge during purging if that is not their initial state.
This functionality was first released in 4.4.0. (@jasone)
- Fix lg_chunk clamping for the --enable-cache-oblivious --disable-fill case.
This regression was first released in 4.0.0. (@jasone, @428desmo)
- Properly detect sparc64 when building for Linux. (@glaubitz)
* 4.4.0 (December 3, 2016)
New features:
- Add configure support for *-*-linux-android. (@cferris1000, @jasone)
- Add the --disable-syscall configure option, for use on systems that place
security-motivated limitations on syscall(2). (@jasone)
- Add support for Debian GNU/kFreeBSD. (@thesam)
Optimizations:
- Add extent serial numbers and use them where appropriate as a sort key that
is higher priority than address, so that the allocation policy prefers older
extents. This tends to improve locality (decrease fragmentation) when
memory grows downward. (@jasone)
- Refactor madvise(2) configuration so that MADV_FREE is detected and utilized
on Linux 4.5 and newer. (@jasone)
- Mark partially purged arena chunks as non-huge-page. This improves
interaction with Linux's transparent huge page functionality. (@jasone)
Bug fixes:
- Fix size class computations for edge conditions involving extremely large
allocations. This regression was first released in 4.0.0. (@jasone,
@ingvarha)
- Remove overly restrictive assertions related to the cactive statistic. This
regression was first released in 4.1.0. (@jasone)
- Implement a more reliable detection scheme for os_unfair_lock on macOS.
(@jszakmeister)
* 4.3.1 (November 7, 2016)
Bug fixes:
- Fix a severe virtual memory leak. This regression was first released in
4.3.0. (@interwq, @jasone)
- Refactor atomic and prng APIs to restore support for 32-bit platforms that
use pre-C11 toolchains, e.g. FreeBSD's mips. (@jasone)
* 4.3.0 (November 4, 2016)
This is the first release that passes the test suite for multiple Windows
configurations, thanks in large part to @glandium setting up continuous
integration via AppVeyor (and Travis CI for Linux and OS X).
New features:
- Add "J" (JSON) support to malloc_stats_print(). (@jasone)
- Add Cray compiler support. (@ronawho)
Optimizations:
- Add/use adaptive spinning for bootstrapping and radix tree node
initialization. (@jasone)
Bug fixes:
- Fix large allocation to search starting in the optimal size class heap,
which can substantially reduce virtual memory churn and fragmentation. This
regression was first released in 4.0.0. (@mjp41, @jasone)
- Fix stats.arenas.<i>.nthreads accounting. (@interwq)
- Fix and simplify decay-based purging. (@jasone)
- Make DSS (sbrk(2)-related) operations lockless, which resolves potential
deadlocks during thread exit. (@jasone)
- Fix over-sized allocation of radix tree leaf nodes. (@mjp41, @ogaun,
@jasone)
- Fix over-sized allocation of arena_t (plus associated stats) data
structures. (@jasone, @interwq)
- Fix EXTRA_CFLAGS to not affect configuration. (@jasone)
- Fix a Valgrind integration bug. (@ronawho)
- Disallow 0x5a junk filling when running in Valgrind. (@jasone)
- Fix a file descriptor leak on Linux. This regression was first released in
4.2.0. (@vsarunas, @jasone)
- Fix static linking of jemalloc with glibc. (@djwatson)
- Use syscall(2) rather than {open,read,close}(2) during boot on Linux. This
works around other libraries' system call wrappers performing reentrant
allocation. (@kspinka, @Whissi, @jasone)
- Fix OS X default zone replacement to work with OS X 10.12. (@glandium,
@jasone)
- Fix cached memory management to avoid needless commit/decommit operations
during purging, which resolves permanent virtual memory map fragmentation
issues on Windows. (@mjp41, @jasone)
- Fix TSD fetches to avoid (recursive) allocation. This is relevant to
non-TLS and Windows configurations. (@jasone)
- Fix malloc_conf overriding to work on Windows. (@jasone)
- Forcibly disable lazy-lock on Windows (was forcibly *enabled*). (@jasone)
* 4.2.1 (June 8, 2016)
Bug fixes:
- Fix bootstrapping issues for configurations that require allocation during
tsd initialization (e.g. --disable-tls). (@cferris1000, @jasone)
- Fix gettimeofday() version of nstime_update(). (@ronawho)
- Fix Valgrind regressions in calloc() and chunk_alloc_wrapper(). (@ronawho)
- Fix potential VM map fragmentation regression. (@jasone)
- Fix opt_zero-triggered in-place huge reallocation zeroing. (@jasone)
- Fix heap profiling context leaks in reallocation edge cases. (@jasone)
* 4.2.0 (May 12, 2016)
New features:
- Add the arena.<i>.reset mallctl, which makes it possible to discard all of
an arena's allocations in a single operation. (@jasone)
- Add the stats.retained and stats.arenas.<i>.retained statistics. (@jasone)
- Add the --with-version configure option. (@jasone)
- Support --with-lg-page values larger than actual page size. (@jasone)
Optimizations:
- Use pairing heaps rather than red-black trees for various hot data
structures. (@djwatson, @jasone)
- Streamline fast paths of rtree operations. (@jasone)
- Optimize the fast paths of calloc() and [m,d,sd]allocx(). (@jasone)
- Decommit unused virtual memory if the OS does not overcommit. (@jasone)
- Specify MAP_NORESERVE on Linux if [heuristic] overcommit is active, in order
to avoid unfortunate interactions during fork(2). (@jasone)
Bug fixes:
- Fix chunk accounting related to triggering gdump profiles. (@jasone)
- Link against librt for clock_gettime(2) if glibc < 2.17. (@jasone)
- Scale leak report summary according to sampling probability. (@jasone)
* 4.1.1 (May 3, 2016)
This bugfix release resolves a variety of mostly minor issues, though the
bitmap fix is critical for 64-bit Windows.
Bug fixes:
- Fix the linear scan version of bitmap_sfu() to shift by the proper amount
even when sizeof(long) is not the same as sizeof(void *), as on 64-bit
Windows. (@jasone)
- Fix hashing functions to avoid unaligned memory accesses (and resulting
crashes). This is relevant at least to some ARM-based platforms.
(@rkmisra)
- Fix fork()-related lock rank ordering reversals. These reversals were
unlikely to cause deadlocks in practice except when heap profiling was
enabled and active. (@jasone)
- Fix various chunk leaks in OOM code paths. (@jasone)
- Fix malloc_stats_print() to print opt.narenas correctly. (@jasone)
- Fix MSVC-specific build/test issues. (@rustyx, @yuslepukhin)
- Fix a variety of test failures that were due to test fragility rather than
core bugs. (@jasone)
* 4.1.0 (February 28, 2016)
This release is primarily about optimizations, but it also incorporates a lot
of portability-motivated refactoring and enhancements. Many people worked on
this release, to an extent that even with the omission here of minor changes
(see git revision history), and of the people who reported and diagnosed
issues, so much of the work was contributed that starting with this release,
changes are annotated with author credits to help reflect the collaborative
effort involved.
New features:
- Implement decay-based unused dirty page purging, a major optimization with
mallctl API impact. This is an alternative to the existing ratio-based
unused dirty page purging, and is intended to eventually become the sole
purging mechanism. New mallctls:
+ opt.purge
+ opt.decay_time
+ arena.<i>.decay
+ arena.<i>.decay_time
+ arenas.decay_time
+ stats.arenas.<i>.decay_time
(@jasone, @cevans87)
- Add --with-malloc-conf, which makes it possible to embed a default
options string during configuration. This was motivated by the desire to
specify --with-malloc-conf=purge:decay , since the default must remain
purge:ratio until the 5.0.0 release. (@jasone)
- Add MS Visual Studio 2015 support. (@rustyx, @yuslepukhin)
- Make *allocx() size class overflow behavior defined. The maximum
size class is now less than PTRDIFF_MAX to protect applications against
numerical overflow, and all allocation functions are guaranteed to indicate
errors rather than potentially crashing if the request size exceeds the
maximum size class. (@jasone)
- jeprof:
+ Add raw heap profile support. (@jasone)
+ Add --retain and --exclude for backtrace symbol filtering. (@jasone)
Optimizations:
- Optimize the fast path to combine various bootstrapping and configuration
checks and execute more streamlined code in the common case. (@interwq)
- Use linear scan for small bitmaps (used for small object tracking). In
addition to speeding up bitmap operations on 64-bit systems, this reduces
allocator metadata overhead by approximately 0.2%. (@djwatson)
- Separate arena_avail trees, which substantially speeds up run tree
operations. (@djwatson)
- Use memoization (boot-time-computed table) for run quantization. Separate
arena_avail trees reduced the importance of this optimization. (@jasone)
- Attempt mmap-based in-place huge reallocation. This can dramatically speed
up incremental huge reallocation. (@jasone)
Incompatible changes:
- Make opt.narenas unsigned rather than size_t. (@jasone)
Bug fixes:
- Fix stats.cactive accounting regression. (@rustyx, @jasone)
- Handle unaligned keys in hash(). This caused problems for some ARM systems.
(@jasone, @cferris1000)
- Refactor arenas array. In addition to fixing a fork-related deadlock, this
makes arena lookups faster and simpler. (@jasone)
- Move retained memory allocation out of the default chunk allocation
function, to a location that gets executed even if the application installs
a custom chunk allocation function. This resolves a virtual memory leak.
(@buchgr)
- Fix a potential tsd cleanup leak. (@cferris1000, @jasone)
- Fix run quantization. In practice this bug had no impact unless
applications requested memory with alignment exceeding one page.
(@jasone, @djwatson)
- Fix LinuxThreads-specific bootstrapping deadlock. (Cosmin Paraschiv)
- jeprof:
+ Don't discard curl options if timeout is not defined. (@djwatson)
+ Detect failed profile fetches. (@djwatson)
- Fix stats.arenas.<i>.{dss,lg_dirty_mult,decay_time,pactive,pdirty} for
--disable-stats case. (@jasone)
* 4.0.4 (October 24, 2015)
This bugfix release fixes another xallocx() regression. No other regressions
have come to light in over a month, so this is likely a good starting point
for people who prefer to wait for "dot one" releases with all the major issues
shaken out.
Bug fixes:
- Fix xallocx(..., MALLOCX_ZERO to zero the last full trailing page of large
allocations that have been randomly assigned an offset of 0 when
--enable-cache-oblivious configure option is enabled.
* 4.0.3 (September 24, 2015)
This bugfix release continues the trend of xallocx() and heap profiling fixes.
@@ -38,7 +632,7 @@ brevity. Much more detail can be found in the git revision history:
these fixes, xallocx() now tries harder to partially fulfill requests for
optional extra space. Note that a couple of minor heap profiling
optimizations are included, but these are better thought of as performance
fixes that were integral to disovering most of the other bugs.
fixes that were integral to discovering most of the other bugs.
Optimizations:
- Avoid a chunk metadata read in arena_prof_tctx_set(), since it is in the
deps/jemalloc/INSTALL → deps/jemalloc/INSTALL.md
Vendored
+175 -154
View File
@@ -18,16 +18,19 @@ would create a dependency on xsltproc in packaged releases, hence the
requirement to either run 'make dist' or avoid installing docs via the various
install_* targets documented below.
=== Advanced configuration =====================================================
## Advanced configuration
The 'configure' script supports numerous options that allow control of which
functionality is enabled, where jemalloc is installed, etc. Optionally, pass
any of the following arguments (not a definitive list) to 'configure':
--help
* `--help`
Print a definitive list of options.
--prefix=<install-root-dir>
* `--prefix=<install-root-dir>`
Set the base directory in which to install. For example:
./configure --prefix=/usr/local
@@ -35,11 +38,29 @@ any of the following arguments (not a definitive list) to 'configure':
will cause files to be installed into /usr/local/include, /usr/local/lib,
and /usr/local/man.
--with-rpath=<colon-separated-rpath>
* `--with-version=(<major>.<minor>.<bugfix>-<nrev>-g<gid>|VERSION)`
The VERSION file is mandatory for successful configuration, and the
following steps are taken to assure its presence:
1) If --with-version=<major>.<minor>.<bugfix>-<nrev>-g<gid> is specified,
generate VERSION using the specified value.
2) If --with-version is not specified in either form and the source
directory is inside a git repository, try to generate VERSION via 'git
describe' invocations that pattern-match release tags.
3) If VERSION is missing, generate it with a bogus version:
0.0.0-0-g0000000000000000000000000000000000000000
Note that --with-version=VERSION bypasses (1) and (2), which simplifies
VERSION configuration when embedding a jemalloc release into another
project's git repository.
* `--with-rpath=<colon-separated-rpath>`
Embed one or more library paths, so that libjemalloc can find the libraries
it is linked to. This works only on ELF-based systems.
--with-mangling=<map>
* `--with-mangling=<map>`
Mangle public symbols specified in <map> which is a comma-separated list of
name:mangled pairs.
@@ -52,7 +73,8 @@ any of the following arguments (not a definitive list) to 'configure':
--with-jemalloc-prefix, and mangled symbols are then ignored when applying
the prefix.
--with-jemalloc-prefix=<prefix>
* `--with-jemalloc-prefix=<prefix>`
Prefix all public APIs with <prefix>. For example, if <prefix> is
"prefix_", API changes like the following occur:
@@ -68,55 +90,46 @@ any of the following arguments (not a definitive list) to 'configure':
jemalloc overlays the default malloc zone, but makes no attempt to actually
replace the "malloc", "calloc", etc. symbols.
--without-export
* `--without-export`
Don't export public APIs. This can be useful when building jemalloc as a
static library, or to avoid exporting public APIs when using the zone
allocator on OSX.
--with-private-namespace=<prefix>
* `--with-private-namespace=<prefix>`
Prefix all library-private APIs with <prefix>je_. For shared libraries,
symbol visibility mechanisms prevent these symbols from being exported, but
for static libraries, naming collisions are a real possibility. By
default, <prefix> is empty, which results in a symbol prefix of je_ .
--with-install-suffix=<suffix>
* `--with-install-suffix=<suffix>`
Append <suffix> to the base name of all installed files, such that multiple
versions of jemalloc can coexist in the same installation directory. For
example, libjemalloc.so.0 becomes libjemalloc<suffix>.so.0.
--disable-cc-silence
Disable code that silences non-useful compiler warnings. This is mainly
useful during development when auditing the set of warnings that are being
silenced.
* `--with-malloc-conf=<malloc_conf>`
Embed `<malloc_conf>` as a run-time options string that is processed prior to
the malloc_conf global variable, the /etc/malloc.conf symlink, and the
MALLOC_CONF environment variable. For example, to change the default decay
time to 30 seconds:
--with-malloc-conf=decay_ms:30000
* `--enable-debug`
--enable-debug
Enable assertions and validation code. This incurs a substantial
performance hit, but is very useful during application development.
Implies --enable-ivsalloc.
--enable-code-coverage
Enable code coverage support, for use during jemalloc test development.
Additional testing targets are available if this option is enabled:
* `--disable-stats`
coverage
coverage_unit
coverage_integration
coverage_stress
These targets do not clear code coverage results from previous runs, and
there are interactions between the various coverage targets, so it is
usually advisable to run 'make clean' between repeated code coverage runs.
--disable-stats
Disable statistics gathering functionality. See the "opt.stats_print"
option documentation for usage details.
--enable-ivsalloc
Enable validation code, which verifies that pointers reside within
jemalloc-owned chunks before dereferencing them. This incurs a minor
performance hit.
* `--enable-prof`
--enable-prof
Enable heap profiling and leak detection functionality. See the "opt.prof"
option documentation for usage details. When enabled, there are several
approaches to backtracing, and the configure script chooses the first one
@@ -126,66 +139,55 @@ any of the following arguments (not a definitive list) to 'configure':
+ libgcc (unless --disable-prof-libgcc)
+ gcc intrinsics (unless --disable-prof-gcc)
--enable-prof-libunwind
* `--enable-prof-libunwind`
Use the libunwind library (http://www.nongnu.org/libunwind/) for stack
backtracing.
--disable-prof-libgcc
* `--disable-prof-libgcc`
Disable the use of libgcc's backtracing functionality.
--disable-prof-gcc
* `--disable-prof-gcc`
Disable the use of gcc intrinsics for backtracing.
--with-static-libunwind=<libunwind.a>
* `--with-static-libunwind=<libunwind.a>`
Statically link against the specified libunwind.a rather than dynamically
linking with -lunwind.
--disable-tcache
Disable thread-specific caches for small objects. Objects are cached and
released in bulk, thus reducing the total number of mutex operations. See
the "opt.tcache" option for usage details.
* `--disable-fill`
--disable-munmap
Disable virtual memory deallocation via munmap(2); instead keep track of
the virtual memory for later use. munmap() is disabled by default (i.e.
--disable-munmap is implied) on Linux, which has a quirk in its virtual
memory allocation algorithm that causes semi-permanent VM map holes under
normal jemalloc operation.
Disable support for junk/zero filling of memory. See the "opt.junk" and
"opt.zero" option documentation for usage details.
--disable-fill
Disable support for junk/zero filling of memory, quarantine, and redzones.
See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option
documentation for usage details.
* `--disable-zone-allocator`
--disable-valgrind
Disable support for Valgrind.
--disable-zone-allocator
Disable zone allocator for Darwin. This means jemalloc won't be hooked as
the default allocator on OSX/iOS.
--enable-utrace
* `--enable-utrace`
Enable utrace(2)-based allocation tracing. This feature is not broadly
portable (FreeBSD has it, but Linux and OS X do not).
--enable-xmalloc
* `--enable-xmalloc`
Enable support for optional immediate termination due to out-of-memory
errors, as is commonly implemented by "xmalloc" wrapper function for malloc.
See the "opt.xmalloc" option documentation for usage details.
--enable-lazy-lock
* `--enable-lazy-lock`
Enable code that wraps pthread_create() to detect when an application
switches from single-threaded to multi-threaded mode, so that it can avoid
mutex locking/unlocking operations while in single-threaded mode. In
practice, this feature usually has little impact on performance unless
thread-specific caching is disabled.
--disable-tls
Disable thread-local storage (TLS), which allows for fast access to
thread-local variables via the __thread keyword. If TLS is available,
jemalloc uses it for several purposes.
* `--disable-cache-oblivious`
--disable-cache-oblivious
Disable cache-oblivious large allocation alignment for large allocation
requests with no alignment constraints. If this feature is disabled, all
large allocations are page-aligned as an implementation artifact, which can
@@ -194,56 +196,51 @@ any of the following arguments (not a definitive list) to 'configure':
most extreme case increases physical memory usage for the 16 KiB size class
to 20 KiB.
--with-xslroot=<path>
* `--disable-syscall`
Disable use of syscall(2) rather than {open,read,write,close}(2). This is
intended as a workaround for systems that place security limitations on
syscall(2).
* `--disable-cxx`
Disable C++ integration. This will cause new and delete operator
implementations to be omitted.
* `--with-xslroot=<path>`
Specify where to find DocBook XSL stylesheets when building the
documentation.
--with-lg-page=<lg-page>
Specify the base 2 log of the system page size. This option is only useful
when cross compiling, since the configure script automatically determines
the host's page size by default.
* `--with-lg-page=<lg-page>`
Specify the base 2 log of the allocator page size, which must in turn be at
least as large as the system page size. By default the configure script
determines the host's page size and sets the allocator page size equal to
the system page size, so this option need not be specified unless the
system page size may change between configuration and execution, e.g. when
cross compiling.
* `--with-lg-page-sizes=<lg-page-sizes>`
--with-lg-page-sizes=<lg-page-sizes>
Specify the comma-separated base 2 logs of the page sizes to support. This
option may be useful when cross-compiling in combination with
--with-lg-page, but its primary use case is for integration with FreeBSD's
option may be useful when cross compiling in combination with
`--with-lg-page`, but its primary use case is for integration with FreeBSD's
libc, wherein jemalloc is embedded.
--with-lg-size-class-group=<lg-size-class-group>
Specify the base 2 log of how many size classes to use for each doubling in
size. By default jemalloc uses <lg-size-class-group>=2, which results in
e.g. the following size classes:
* `--with-lg-hugepage=<lg-hugepage>`
[...], 64,
80, 96, 112, 128,
160, [...]
Specify the base 2 log of the system huge page size. This option is useful
when cross compiling, or when overriding the default for systems that do
not explicitly support huge pages.
<lg-size-class-group>=3 results in e.g. the following size classes:
* `--with-lg-quantum=<lg-quantum>`
[...], 64,
72, 80, 88, 96, 104, 112, 120, 128,
144, [...]
The minimal <lg-size-class-group>=0 causes jemalloc to only provide size
classes that are powers of 2:
[...],
64,
128,
256,
[...]
An implementation detail currently limits the total number of small size
classes to 255, and a compilation error will result if the
<lg-size-class-group> you specify cannot be supported. The limit is
roughly <lg-size-class-group>=4, depending on page size.
--with-lg-quantum=<lg-quantum>
Specify the base 2 log of the minimum allocation alignment. jemalloc needs
to know the minimum alignment that meets the following C standard
requirement (quoted from the April 12, 2011 draft of the C11 standard):
The pointer returned if the allocation succeeds is suitably aligned so
> The pointer returned if the allocation succeeds is suitably aligned so
that it may be assigned to a pointer to any type of object with a
fundamental alignment requirement and then used to access such an object
or an array of such objects in the space allocated [...]
@@ -251,71 +248,82 @@ any of the following arguments (not a definitive list) to 'configure':
This setting is architecture-specific, and although jemalloc includes known
safe values for the most commonly used modern architectures, there is a
wrinkle related to GNU libc (glibc) that may impact your choice of
<lg-quantum>. On most modern architectures, this mandates 16-byte alignment
(<lg-quantum>=4), but the glibc developers chose not to meet this
<lg-quantum>. On most modern architectures, this mandates 16-byte
alignment (<lg-quantum>=4), but the glibc developers chose not to meet this
requirement for performance reasons. An old discussion can be found at
https://sourceware.org/bugzilla/show_bug.cgi?id=206 . Unlike glibc,
<https://sourceware.org/bugzilla/show_bug.cgi?id=206> . Unlike glibc,
jemalloc does follow the C standard by default (caveat: jemalloc
technically cheats if --with-lg-tiny-min is smaller than
--with-lg-quantum), but the fact that Linux systems already work around
this allocator noncompliance means that it is generally safe in practice to
let jemalloc's minimum alignment follow glibc's lead. If you specify
--with-lg-quantum=3 during configuration, jemalloc will provide additional
size classes that are not 16-byte-aligned (24, 40, and 56, assuming
--with-lg-size-class-group=2).
technically cheats for size classes smaller than the quantum), but the fact
that Linux systems already work around this allocator noncompliance means
that it is generally safe in practice to let jemalloc's minimum alignment
follow glibc's lead. If you specify `--with-lg-quantum=3` during
configuration, jemalloc will provide additional size classes that are not
16-byte-aligned (24, 40, and 56).
--with-lg-tiny-min=<lg-tiny-min>
Specify the base 2 log of the minimum tiny size class to support. Tiny
size classes are powers of 2 less than the quantum, and are only
incorporated if <lg-tiny-min> is less than <lg-quantum> (see
--with-lg-quantum). Tiny size classes technically violate the C standard
requirement for minimum alignment, and crashes could conceivably result if
the compiler were to generate instructions that made alignment assumptions,
both because illegal instruction traps could result, and because accesses
could straddle page boundaries and cause segmentation faults due to
accessing unmapped addresses.
* `--with-lg-vaddr=<lg-vaddr>`
The default of <lg-tiny-min>=3 works well in practice even on architectures
that technically require 16-byte alignment, probably for the same reason
--with-lg-quantum=3 works. Smaller tiny size classes can, and will, cause
crashes (see https://bugzilla.mozilla.org/show_bug.cgi?id=691003 for an
example).
Specify the number of significant virtual address bits. By default, the
configure script attempts to detect virtual address size on those platforms
where it knows how, and picks a default otherwise. This option may be
useful when cross-compiling.
This option is rarely useful, and is mainly provided as documentation of a
subtle implementation detail. If you do use this option, specify a
value in [3, ..., <lg-quantum>].
* `--disable-initial-exec-tls`
Disable the initial-exec TLS model for jemalloc's internal thread-local
storage (on those platforms that support explicit settings). This can allow
jemalloc to be dynamically loaded after program startup (e.g. using dlopen).
Note that in this case, there will be two malloc implementations operating
in the same process, which will almost certainly result in confusing runtime
crashes if pointers leak from one implementation to the other.
The following environment variables (not a definitive list) impact configure's
behavior:
CFLAGS="?"
Pass these flags to the compiler. You probably shouldn't define this unless
you know what you are doing. (Use EXTRA_CFLAGS instead.)
* `CFLAGS="?"`
* `CXXFLAGS="?"`
EXTRA_CFLAGS="?"
Append these flags to CFLAGS. This makes it possible to add flags such as
-Werror, while allowing the configure script to determine what other flags
are appropriate for the specified configuration.
Pass these flags to the C/C++ compiler. Any flags set by the configure
script are prepended, which means explicitly set flags generally take
precedence. Take care when specifying flags such as -Werror, because
configure tests may be affected in undesirable ways.
The configure script specifically checks whether an optimization flag (-O*)
is specified in EXTRA_CFLAGS, and refrains from specifying an optimization
level if it finds that one has already been specified.
* `EXTRA_CFLAGS="?"`
* `EXTRA_CXXFLAGS="?"`
Append these flags to CFLAGS/CXXFLAGS, without passing them to the
compiler(s) during configuration. This makes it possible to add flags such
as -Werror, while allowing the configure script to determine what other
flags are appropriate for the specified configuration.
* `CPPFLAGS="?"`
CPPFLAGS="?"
Pass these flags to the C preprocessor. Note that CFLAGS is not passed to
'cpp' when 'configure' is looking for include files, so you must use
CPPFLAGS instead if you need to help 'configure' find header files.
LD_LIBRARY_PATH="?"
* `LD_LIBRARY_PATH="?"`
'ld' uses this colon-separated list to find libraries.
LDFLAGS="?"
* `LDFLAGS="?"`
Pass these flags when linking.
PATH="?"
* `PATH="?"`
'configure' uses this to find programs.
=== Advanced compilation =======================================================
In some cases it may be necessary to work around configuration results that do
not match reality. For example, Linux 4.5 added support for the MADV_FREE flag
to madvise(2), which can cause problems if building on a host with MADV_FREE
support and deploying to a target without. To work around this, use a cache
file to override the relevant configuration variable defined in configure.ac,
e.g.:
echo "je_cv_madv_free=no" > config.cache && ./configure -C
## Advanced compilation
To build only parts of jemalloc, use the following targets:
@@ -332,6 +340,7 @@ To install only parts of jemalloc, use the following targets:
install_include
install_lib_shared
install_lib_static
install_lib_pc
install_lib
install_doc_html
install_doc_man
@@ -343,40 +352,51 @@ To clean up build results to varying degrees, use the following make targets:
distclean
relclean
=== Advanced installation ======================================================
## Advanced installation
Optionally, define make variables when invoking make, including (not
exclusively):
INCLUDEDIR="?"
* `INCLUDEDIR="?"`
Use this as the installation prefix for header files.
LIBDIR="?"
* `LIBDIR="?"`
Use this as the installation prefix for libraries.
MANDIR="?"
* `MANDIR="?"`
Use this as the installation prefix for man pages.
DESTDIR="?"
* `DESTDIR="?"`
Prepend DESTDIR to INCLUDEDIR, LIBDIR, DATADIR, and MANDIR. This is useful
when installing to a different path than was specified via --prefix.
CC="?"
* `CC="?"`
Use this to invoke the C compiler.
CFLAGS="?"
* `CFLAGS="?"`
Pass these flags to the compiler.
CPPFLAGS="?"
* `CPPFLAGS="?"`
Pass these flags to the C preprocessor.
LDFLAGS="?"
* `LDFLAGS="?"`
Pass these flags when linking.
PATH="?"
* `PATH="?"`
Use this to search for programs used during configuration and building.
=== Development ================================================================
## Development
If you intend to make non-trivial changes to jemalloc, use the 'autogen.sh'
script rather than 'configure'. This re-generates 'configure', enables
@@ -393,7 +413,8 @@ directory, issue configuration and build commands:
../configure --enable-autogen
make
=== Documentation ==============================================================
## Documentation
The manual page is generated in both html and roff formats. Any web browser
can be used to view the html manual. The roff manual page can be formatted
deps/jemalloc/Makefile.in
Vendored
+220 -109
View File
@@ -9,6 +9,7 @@ vpath % .
SHELL := /bin/sh
CC := @CC@
CXX := @CXX@
# Configuration parameters.
DESTDIR =
@@ -23,12 +24,18 @@ abs_srcroot := @abs_srcroot@
abs_objroot := @abs_objroot@
# Build parameters.
CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include
CFLAGS := @CFLAGS@
CPPFLAGS := @CPPFLAGS@ -I$(objroot)include -I$(srcroot)include
CONFIGURE_CFLAGS := @CONFIGURE_CFLAGS@
SPECIFIED_CFLAGS := @SPECIFIED_CFLAGS@
EXTRA_CFLAGS := @EXTRA_CFLAGS@
CFLAGS := $(strip $(CONFIGURE_CFLAGS) $(SPECIFIED_CFLAGS) $(EXTRA_CFLAGS))
CONFIGURE_CXXFLAGS := @CONFIGURE_CXXFLAGS@
SPECIFIED_CXXFLAGS := @SPECIFIED_CXXFLAGS@
EXTRA_CXXFLAGS := @EXTRA_CXXFLAGS@
CXXFLAGS := $(strip $(CONFIGURE_CXXFLAGS) $(SPECIFIED_CXXFLAGS) $(EXTRA_CXXFLAGS))
LDFLAGS := @LDFLAGS@
EXTRA_LDFLAGS := @EXTRA_LDFLAGS@
LIBS := @LIBS@
TESTLIBS := @TESTLIBS@
RPATH_EXTRA := @RPATH_EXTRA@
SO := @so@
IMPORTLIB := @importlib@
@@ -48,20 +55,24 @@ cfghdrs_out := @cfghdrs_out@
cfgoutputs_in := $(addprefix $(srcroot),@cfgoutputs_in@)
cfgoutputs_out := @cfgoutputs_out@
enable_autogen := @enable_autogen@
enable_code_coverage := @enable_code_coverage@
enable_prof := @enable_prof@
enable_valgrind := @enable_valgrind@
enable_zone_allocator := @enable_zone_allocator@
MALLOC_CONF := @JEMALLOC_CPREFIX@MALLOC_CONF
link_whole_archive := @link_whole_archive@
DSO_LDFLAGS = @DSO_LDFLAGS@
SOREV = @SOREV@
PIC_CFLAGS = @PIC_CFLAGS@
CTARGET = @CTARGET@
LDTARGET = @LDTARGET@
TEST_LD_MODE = @TEST_LD_MODE@
MKLIB = @MKLIB@
AR = @AR@
ARFLAGS = @ARFLAGS@
DUMP_SYMS = @DUMP_SYMS@
AWK := @AWK@
CC_MM = @CC_MM@
LM := @LM@
INSTALL = @INSTALL@
ifeq (macho, $(ABI))
TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH="$(objroot)lib"
@@ -78,18 +89,36 @@ LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
# Lists of files.
BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof
C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \
$(srcroot)src/atomic.c $(srcroot)src/base.c $(srcroot)src/bitmap.c \
$(srcroot)src/chunk.c $(srcroot)src/chunk_dss.c \
$(srcroot)src/chunk_mmap.c $(srcroot)src/ckh.c $(srcroot)src/ctl.c \
$(srcroot)src/extent.c $(srcroot)src/hash.c $(srcroot)src/huge.c \
$(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/pages.c \
$(srcroot)src/prof.c $(srcroot)src/quarantine.c $(srcroot)src/rtree.c \
$(srcroot)src/stats.c $(srcroot)src/tcache.c $(srcroot)src/util.c \
$(srcroot)src/tsd.c
ifeq ($(enable_valgrind), 1)
C_SRCS += $(srcroot)src/valgrind.c
endif
C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/arena.c \
$(srcroot)src/background_thread.c \
$(srcroot)src/base.c \
$(srcroot)src/bin.c \
$(srcroot)src/bitmap.c \
$(srcroot)src/ckh.c \
$(srcroot)src/ctl.c \
$(srcroot)src/div.c \
$(srcroot)src/extent.c \
$(srcroot)src/extent_dss.c \
$(srcroot)src/extent_mmap.c \
$(srcroot)src/hash.c \
$(srcroot)src/hooks.c \
$(srcroot)src/large.c \
$(srcroot)src/log.c \
$(srcroot)src/malloc_io.c \
$(srcroot)src/mutex.c \
$(srcroot)src/mutex_pool.c \
$(srcroot)src/nstime.c \
$(srcroot)src/pages.c \
$(srcroot)src/prng.c \
$(srcroot)src/prof.c \
$(srcroot)src/rtree.c \
$(srcroot)src/stats.c \
$(srcroot)src/sz.c \
$(srcroot)src/tcache.c \
$(srcroot)src/ticker.c \
$(srcroot)src/tsd.c \
$(srcroot)src/witness.c
ifeq ($(enable_zone_allocator), 1)
C_SRCS += $(srcroot)src/zone.c
endif
@@ -105,6 +134,11 @@ DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV)
ifneq ($(SOREV),$(SO))
DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO)
endif
ifeq (1, $(link_whole_archive))
LJEMALLOC := -Wl,--whole-archive -L$(objroot)lib -l$(LIBJEMALLOC) -Wl,--no-whole-archive
else
LJEMALLOC := $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
endif
PC := $(objroot)jemalloc.pc
MAN3 := $(objroot)doc/jemalloc$(install_suffix).3
DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml
@@ -116,53 +150,103 @@ C_TESTLIB_SRCS := $(srcroot)test/src/btalloc.c $(srcroot)test/src/btalloc_0.c \
$(srcroot)test/src/mtx.c $(srcroot)test/src/mq.c \
$(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \
$(srcroot)test/src/thd.c $(srcroot)test/src/timer.c
C_UTIL_INTEGRATION_SRCS := $(srcroot)src/util.c
TESTS_UNIT := $(srcroot)test/unit/atomic.c \
ifeq (1, $(link_whole_archive))
C_UTIL_INTEGRATION_SRCS :=
C_UTIL_CPP_SRCS :=
else
C_UTIL_INTEGRATION_SRCS := $(srcroot)src/nstime.c $(srcroot)src/malloc_io.c
C_UTIL_CPP_SRCS := $(srcroot)src/nstime.c $(srcroot)src/malloc_io.c
endif
TESTS_UNIT := \
$(srcroot)test/unit/a0.c \
$(srcroot)test/unit/arena_reset.c \
$(srcroot)test/unit/atomic.c \
$(srcroot)test/unit/background_thread.c \
$(srcroot)test/unit/background_thread_enable.c \
$(srcroot)test/unit/base.c \
$(srcroot)test/unit/bitmap.c \
$(srcroot)test/unit/ckh.c \
$(srcroot)test/unit/decay.c \
$(srcroot)test/unit/div.c \
$(srcroot)test/unit/emitter.c \
$(srcroot)test/unit/extent_quantize.c \
$(srcroot)test/unit/fork.c \
$(srcroot)test/unit/hash.c \
$(srcroot)test/unit/hooks.c \
$(srcroot)test/unit/junk.c \
$(srcroot)test/unit/junk_alloc.c \
$(srcroot)test/unit/junk_free.c \
$(srcroot)test/unit/lg_chunk.c \
$(srcroot)test/unit/log.c \
$(srcroot)test/unit/mallctl.c \
$(srcroot)test/unit/malloc_io.c \
$(srcroot)test/unit/math.c \
$(srcroot)test/unit/mq.c \
$(srcroot)test/unit/mtx.c \
$(srcroot)test/unit/pack.c \
$(srcroot)test/unit/pages.c \
$(srcroot)test/unit/ph.c \
$(srcroot)test/unit/prng.c \
$(srcroot)test/unit/prof_accum.c \
$(srcroot)test/unit/prof_active.c \
$(srcroot)test/unit/prof_gdump.c \
$(srcroot)test/unit/prof_idump.c \
$(srcroot)test/unit/prof_reset.c \
$(srcroot)test/unit/prof_tctx.c \
$(srcroot)test/unit/prof_thread_name.c \
$(srcroot)test/unit/ql.c \
$(srcroot)test/unit/qr.c \
$(srcroot)test/unit/quarantine.c \
$(srcroot)test/unit/rb.c \
$(srcroot)test/unit/retained.c \
$(srcroot)test/unit/rtree.c \
$(srcroot)test/unit/SFMT.c \
$(srcroot)test/unit/size_classes.c \
$(srcroot)test/unit/slab.c \
$(srcroot)test/unit/smoothstep.c \
$(srcroot)test/unit/spin.c \
$(srcroot)test/unit/stats.c \
$(srcroot)test/unit/stats_print.c \
$(srcroot)test/unit/ticker.c \
$(srcroot)test/unit/nstime.c \
$(srcroot)test/unit/tsd.c \
$(srcroot)test/unit/util.c \
$(srcroot)test/unit/witness.c \
$(srcroot)test/unit/zero.c
ifeq (@enable_prof@, 1)
TESTS_UNIT += \
$(srcroot)test/unit/arena_reset_prof.c
endif
TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
$(srcroot)test/integration/allocated.c \
$(srcroot)test/integration/sdallocx.c \
$(srcroot)test/integration/extent.c \
$(srcroot)test/integration/mallocx.c \
$(srcroot)test/integration/MALLOCX_ARENA.c \
$(srcroot)test/integration/overflow.c \
$(srcroot)test/integration/posix_memalign.c \
$(srcroot)test/integration/rallocx.c \
$(srcroot)test/integration/sdallocx.c \
$(srcroot)test/integration/thread_arena.c \
$(srcroot)test/integration/thread_tcache_enabled.c \
$(srcroot)test/integration/xallocx.c \
$(srcroot)test/integration/chunk.c
$(srcroot)test/integration/xallocx.c
ifeq (@enable_cxx@, 1)
CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp
else
CPP_SRCS :=
TESTS_INTEGRATION_CPP :=
endif
TESTS_STRESS := $(srcroot)test/stress/microbench.c
TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS)
TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_INTEGRATION_CPP) $(TESTS_STRESS)
PRIVATE_NAMESPACE_HDRS := $(objroot)include/jemalloc/internal/private_namespace.h $(objroot)include/jemalloc/internal/private_namespace_jet.h
PRIVATE_NAMESPACE_GEN_HDRS := $(PRIVATE_NAMESPACE_HDRS:%.h=%.gen.h)
C_SYM_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.sym.$(O))
C_SYMS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.sym)
C_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.$(O))
CPP_OBJS := $(CPP_SRCS:$(srcroot)%.cpp=$(objroot)%.$(O))
C_PIC_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.pic.$(O))
CPP_PIC_OBJS := $(CPP_SRCS:$(srcroot)%.cpp=$(objroot)%.pic.$(O))
C_JET_SYM_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.sym.$(O))
C_JET_SYMS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.sym)
C_JET_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.$(O))
C_TESTLIB_UNIT_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.unit.$(O))
C_TESTLIB_INTEGRATION_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O))
@@ -172,15 +256,17 @@ C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_
TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_INTEGRATION_CPP_OBJS := $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%.$(O))
TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS)
TESTS_CPP_OBJS := $(TESTS_INTEGRATION_CPP_OBJS)
.PHONY: all dist build_doc_html build_doc_man build_doc
.PHONY: install_bin install_include install_lib
.PHONY: install_doc_html install_doc_man install_doc install
.PHONY: tests check clean distclean relclean
.SECONDARY : $(TESTS_OBJS)
.SECONDARY : $(PRIVATE_NAMESPACE_GEN_HDRS) $(TESTS_OBJS) $(TESTS_CPP_OBJS)
# Default target.
all: build_lib
@@ -201,18 +287,32 @@ build_doc: $(DOCS)
# Include generated dependency files.
#
ifdef CC_MM
-include $(C_SYM_OBJS:%.$(O)=%.d)
-include $(C_OBJS:%.$(O)=%.d)
-include $(CPP_OBJS:%.$(O)=%.d)
-include $(C_PIC_OBJS:%.$(O)=%.d)
-include $(CPP_PIC_OBJS:%.$(O)=%.d)
-include $(C_JET_SYM_OBJS:%.$(O)=%.d)
-include $(C_JET_OBJS:%.$(O)=%.d)
-include $(C_TESTLIB_OBJS:%.$(O)=%.d)
-include $(TESTS_OBJS:%.$(O)=%.d)
-include $(TESTS_CPP_OBJS:%.$(O)=%.d)
endif
$(C_SYM_OBJS): $(objroot)src/%.sym.$(O): $(srcroot)src/%.c
$(C_SYM_OBJS): CPPFLAGS += -DJEMALLOC_NO_PRIVATE_NAMESPACE
$(C_SYMS): $(objroot)src/%.sym: $(objroot)src/%.sym.$(O)
$(C_OBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c
$(CPP_OBJS): $(objroot)src/%.$(O): $(srcroot)src/%.cpp
$(C_PIC_OBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c
$(C_PIC_OBJS): CFLAGS += $(PIC_CFLAGS)
$(CPP_PIC_OBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.cpp
$(CPP_PIC_OBJS): CXXFLAGS += $(PIC_CFLAGS)
$(C_JET_SYM_OBJS): $(objroot)src/%.jet.sym.$(O): $(srcroot)src/%.c
$(C_JET_SYM_OBJS): CPPFLAGS += -DJEMALLOC_JET -DJEMALLOC_NO_PRIVATE_NAMESPACE
$(C_JET_SYMS): $(objroot)src/%.jet.sym: $(objroot)src/%.jet.sym.$(O)
$(C_JET_OBJS): $(objroot)src/%.jet.$(O): $(srcroot)src/%.c
$(C_JET_OBJS): CFLAGS += -DJEMALLOC_JET
$(C_JET_OBJS): CPPFLAGS += -DJEMALLOC_JET
$(C_TESTLIB_UNIT_OBJS): $(objroot)test/src/%.unit.$(O): $(srcroot)test/src/%.c
$(C_TESTLIB_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
$(C_TESTLIB_INTEGRATION_OBJS): $(objroot)test/src/%.integration.$(O): $(srcroot)test/src/%.c
@@ -223,112 +323,146 @@ $(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_T
$(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
$(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
$(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
$(TESTS_INTEGRATION_CPP_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_CPP_TEST
$(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST
$(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c
$(TESTS_CPP_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.cpp
$(TESTS_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
$(TESTS_CPP_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
ifneq ($(IMPORTLIB),$(SO))
$(C_OBJS) $(C_JET_OBJS): CPPFLAGS += -DDLLEXPORT
$(CPP_OBJS) $(C_SYM_OBJS) $(C_OBJS) $(C_JET_SYM_OBJS) $(C_JET_OBJS): CPPFLAGS += -DDLLEXPORT
endif
ifndef CC_MM
# Dependencies.
ifndef CC_MM
HEADER_DIRS = $(srcroot)include/jemalloc/internal \
$(objroot)include/jemalloc $(objroot)include/jemalloc/internal
HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h))
$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): $(HEADERS)
$(TESTS_OBJS): $(objroot)test/include/test/jemalloc_test.h
HEADERS = $(filter-out $(PRIVATE_NAMESPACE_HDRS),$(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h)))
$(C_SYM_OBJS) $(C_OBJS) $(CPP_OBJS) $(C_PIC_OBJS) $(CPP_PIC_OBJS) $(C_JET_SYM_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS) $(TESTS_CPP_OBJS): $(HEADERS)
$(TESTS_OBJS) $(TESTS_CPP_OBJS): $(objroot)test/include/test/jemalloc_test.h
endif
$(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O):
$(C_OBJS) $(CPP_OBJS) $(C_PIC_OBJS) $(CPP_PIC_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_INTEGRATION_CPP_OBJS): $(objroot)include/jemalloc/internal/private_namespace.h
$(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_STRESS_OBJS) $(TESTS_UNIT_OBJS) $(TESTS_STRESS_OBJS): $(objroot)include/jemalloc/internal/private_namespace_jet.h
$(C_SYM_OBJS) $(C_OBJS) $(C_PIC_OBJS) $(C_JET_SYM_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O):
@mkdir -p $(@D)
$(CC) $(CFLAGS) -c $(CPPFLAGS) $(CTARGET) $<
ifdef CC_MM
@$(CC) -MM $(CPPFLAGS) -MT $@ -o $(@:%.$(O)=%.d) $<
endif
$(C_SYMS): %.sym:
@mkdir -p $(@D)
$(DUMP_SYMS) $< | $(AWK) -f $(objroot)include/jemalloc/internal/private_symbols.awk > $@
$(C_JET_SYMS): %.sym:
@mkdir -p $(@D)
$(DUMP_SYMS) $< | $(AWK) -f $(objroot)include/jemalloc/internal/private_symbols_jet.awk > $@
$(objroot)include/jemalloc/internal/private_namespace.gen.h: $(C_SYMS)
$(SHELL) $(srcroot)include/jemalloc/internal/private_namespace.sh $^ > $@
$(objroot)include/jemalloc/internal/private_namespace_jet.gen.h: $(C_JET_SYMS)
$(SHELL) $(srcroot)include/jemalloc/internal/private_namespace.sh $^ > $@
%.h: %.gen.h
@if ! `cmp -s $< $@` ; then echo "cp $< $<"; cp $< $@ ; fi
$(CPP_OBJS) $(CPP_PIC_OBJS) $(TESTS_CPP_OBJS): %.$(O):
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) -c $(CPPFLAGS) $(CTARGET) $<
ifdef CC_MM
@$(CXX) -MM $(CPPFLAGS) -MT $@ -o $(@:%.$(O)=%.d) $<
endif
ifneq ($(SOREV),$(SO))
%.$(SO) : %.$(SOREV)
@mkdir -p $(@D)
ln -sf $(<F) $@
endif
$(objroot)lib/$(LIBJEMALLOC).$(SOREV) : $(if $(PIC_CFLAGS),$(C_PIC_OBJS),$(C_OBJS))
$(objroot)lib/$(LIBJEMALLOC).$(SOREV) : $(if $(PIC_CFLAGS),$(C_PIC_OBJS),$(C_OBJS)) $(if $(PIC_CFLAGS),$(CPP_PIC_OBJS),$(CPP_OBJS))
@mkdir -p $(@D)
$(CC) $(DSO_LDFLAGS) $(call RPATH,$(RPATH_EXTRA)) $(LDTARGET) $+ $(LDFLAGS) $(LIBS) $(EXTRA_LDFLAGS)
$(objroot)lib/$(LIBJEMALLOC)_pic.$(A) : $(C_PIC_OBJS)
$(objroot)lib/$(LIBJEMALLOC).$(A) : $(C_OBJS)
$(objroot)lib/$(LIBJEMALLOC)_s.$(A) : $(C_OBJS)
$(objroot)lib/$(LIBJEMALLOC)_pic.$(A) : $(C_PIC_OBJS) $(CPP_PIC_OBJS)
$(objroot)lib/$(LIBJEMALLOC).$(A) : $(C_OBJS) $(CPP_OBJS)
$(objroot)lib/$(LIBJEMALLOC)_s.$(A) : $(C_OBJS) $(CPP_OBJS)
$(STATIC_LIBS):
@mkdir -p $(@D)
$(AR) $(ARFLAGS)@AROUT@ $+
$(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(TESTS_UNIT_LINK_OBJS) $(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS)
$(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS)
@mkdir -p $(@D)
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(TESTLIBS) $(EXTRA_LDFLAGS)
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS)
$(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
@mkdir -p $(@D)
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(filter -lpthread,$(LIBS))) -lm $(TESTLIBS) $(EXTRA_LDFLAGS)
$(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -lpthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS)
$(objroot)test/integration/cpp/%$(EXE): $(objroot)test/integration/cpp/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
@mkdir -p $(@D)
$(CXX) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS)
$(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_STRESS_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
@mkdir -p $(@D)
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(TESTLIBS) $(EXTRA_LDFLAGS)
$(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS)
build_lib_shared: $(DSOS)
build_lib_static: $(STATIC_LIBS)
build_lib: build_lib_shared build_lib_static
install_bin:
install -d $(BINDIR)
$(INSTALL) -d $(BINDIR)
@for b in $(BINS); do \
echo "install -m 755 $$b $(BINDIR)"; \
install -m 755 $$b $(BINDIR); \
echo "$(INSTALL) -m 755 $$b $(BINDIR)"; \
$(INSTALL) -m 755 $$b $(BINDIR); \
done
install_include:
install -d $(INCLUDEDIR)/jemalloc
$(INSTALL) -d $(INCLUDEDIR)/jemalloc
@for h in $(C_HDRS); do \
echo "install -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
install -m 644 $$h $(INCLUDEDIR)/jemalloc; \
echo "$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc; \
done
install_lib_shared: $(DSOS)
install -d $(LIBDIR)
install -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
$(INSTALL) -d $(LIBDIR)
$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
ifneq ($(SOREV),$(SO))
ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO)
endif
install_lib_static: $(STATIC_LIBS)
install -d $(LIBDIR)
$(INSTALL) -d $(LIBDIR)
@for l in $(STATIC_LIBS); do \
echo "install -m 755 $$l $(LIBDIR)"; \
install -m 755 $$l $(LIBDIR); \
echo "$(INSTALL) -m 755 $$l $(LIBDIR)"; \
$(INSTALL) -m 755 $$l $(LIBDIR); \
done
install_lib_pc: $(PC)
install -d $(LIBDIR)/pkgconfig
$(INSTALL) -d $(LIBDIR)/pkgconfig
@for l in $(PC); do \
echo "install -m 644 $$l $(LIBDIR)/pkgconfig"; \
install -m 644 $$l $(LIBDIR)/pkgconfig; \
echo "$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig"; \
$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \
done
install_lib: install_lib_shared install_lib_static install_lib_pc
install_doc_html:
install -d $(DATADIR)/doc/jemalloc$(install_suffix)
$(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix)
@for d in $(DOCS_HTML); do \
echo "install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
echo "$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
done
install_doc_man:
install -d $(MANDIR)/man3
$(INSTALL) -d $(MANDIR)/man3
@for d in $(DOCS_MAN3); do \
echo "install -m 644 $$d $(MANDIR)/man3"; \
install -m 644 $$d $(MANDIR)/man3; \
echo "$(INSTALL) -m 644 $$d $(MANDIR)/man3"; \
$(INSTALL) -m 644 $$d $(MANDIR)/man3; \
done
install_doc: install_doc_html install_doc_man
@@ -336,7 +470,7 @@ install_doc: install_doc_html install_doc_man
install: install_bin install_include install_lib install_doc
tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE))
tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE))
tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE)) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%$(EXE))
tests_stress: $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%$(EXE))
tests: tests_unit tests_integration tests_stress
@@ -352,71 +486,48 @@ check_unit: tests_unit check_unit_dir
$(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
check_integration_prof: tests_integration check_integration_dir
ifeq ($(enable_prof), 1)
$(MALLOC_CONF)="prof:true" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
$(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
$(MALLOC_CONF)="prof:true" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
$(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
endif
check_integration_decay: tests_integration check_integration_dir
$(MALLOC_CONF)="dirty_decay_ms:-1,muzzy_decay_ms:-1" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
$(MALLOC_CONF)="dirty_decay_ms:0,muzzy_decay_ms:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
check_integration: tests_integration check_integration_dir
$(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
$(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
stress: tests_stress stress_dir
$(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%)
check: tests check_dir check_integration_prof
$(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
ifeq ($(enable_code_coverage), 1)
coverage_unit: check_unit
$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS)
coverage_integration: check_integration
$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS)
coverage_stress: stress
$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress stress $(TESTS_STRESS_OBJS)
coverage: check
$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS) $(TESTS_UNIT_AUX_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress integration $(TESTS_STRESS_OBJS)
endif
check: check_unit check_integration check_integration_decay check_integration_prof
clean:
rm -f $(PRIVATE_NAMESPACE_HDRS)
rm -f $(PRIVATE_NAMESPACE_GEN_HDRS)
rm -f $(C_SYM_OBJS)
rm -f $(C_SYMS)
rm -f $(C_OBJS)
rm -f $(CPP_OBJS)
rm -f $(C_PIC_OBJS)
rm -f $(CPP_PIC_OBJS)
rm -f $(C_JET_SYM_OBJS)
rm -f $(C_JET_SYMS)
rm -f $(C_JET_OBJS)
rm -f $(C_TESTLIB_OBJS)
rm -f $(C_SYM_OBJS:%.$(O)=%.d)
rm -f $(C_OBJS:%.$(O)=%.d)
rm -f $(C_OBJS:%.$(O)=%.gcda)
rm -f $(C_OBJS:%.$(O)=%.gcno)
rm -f $(CPP_OBJS:%.$(O)=%.d)
rm -f $(C_PIC_OBJS:%.$(O)=%.d)
rm -f $(C_PIC_OBJS:%.$(O)=%.gcda)
rm -f $(C_PIC_OBJS:%.$(O)=%.gcno)
rm -f $(CPP_PIC_OBJS:%.$(O)=%.d)
rm -f $(C_JET_SYM_OBJS:%.$(O)=%.d)
rm -f $(C_JET_OBJS:%.$(O)=%.d)
rm -f $(C_JET_OBJS:%.$(O)=%.gcda)
rm -f $(C_JET_OBJS:%.$(O)=%.gcno)
rm -f $(C_TESTLIB_OBJS:%.$(O)=%.d)
rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcda)
rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcno)
rm -f $(TESTS_OBJS:%.$(O)=%$(EXE))
rm -f $(TESTS_OBJS)
rm -f $(TESTS_OBJS:%.$(O)=%.d)
rm -f $(TESTS_OBJS:%.$(O)=%.gcda)
rm -f $(TESTS_OBJS:%.$(O)=%.gcno)
rm -f $(TESTS_OBJS:%.$(O)=%.out)
rm -f $(TESTS_CPP_OBJS:%.$(O)=%$(EXE))
rm -f $(TESTS_CPP_OBJS)
rm -f $(TESTS_CPP_OBJS:%.$(O)=%.d)
rm -f $(TESTS_CPP_OBJS:%.$(O)=%.out)
rm -f $(DSOS) $(STATIC_LIBS)
rm -f $(objroot)*.gcov.*
distclean: clean
rm -f $(objroot)bin/jemalloc-config
deps/jemalloc/README
Vendored
+7 -7
View File
@@ -3,12 +3,12 @@ fragmentation avoidance and scalable concurrency support. jemalloc first came
into use as the FreeBSD libc allocator in 2005, and since then it has found its
way into numerous applications that rely on its predictable behavior. In 2010
jemalloc development efforts broadened to include developer support features
such as heap profiling, Valgrind integration, and extensive monitoring/tuning
hooks. Modern jemalloc releases continue to be integrated back into FreeBSD,
and therefore versatility remains critical. Ongoing development efforts trend
toward making jemalloc among the best allocators for a broad range of demanding
applications, and eliminating/mitigating weaknesses that have practical
repercussions for real world applications.
such as heap profiling and extensive monitoring/tuning hooks. Modern jemalloc
releases continue to be integrated back into FreeBSD, and therefore versatility
remains critical. Ongoing development efforts trend toward making jemalloc
among the best allocators for a broad range of demanding applications, and
eliminating/mitigating weaknesses that have practical repercussions for real
world applications.
The COPYING file contains copyright and licensing information.
@@ -17,4 +17,4 @@ jemalloc.
The ChangeLog file contains a brief summary of changes for each release.
URL: http://www.canonware.com/jemalloc/
URL: http://jemalloc.net/
deps/jemalloc/TUNING.md
Vendored
+129
View File
@@ -0,0 +1,129 @@
This document summarizes the common approaches for performance fine tuning with
jemalloc (as of 5.1.0). The default configuration of jemalloc tends to work
reasonably well in practice, and most applications should not have to tune any
options. However, in order to cover a wide range of applications and avoid
pathological cases, the default setting is sometimes kept conservative and
suboptimal, even for many common workloads. When jemalloc is properly tuned for
a specific application / workload, it is common to improve system level metrics
by a few percent, or make favorable trade-offs.
## Notable runtime options for performance tuning
Runtime options can be set via
[malloc_conf](http://jemalloc.net/jemalloc.3.html#tuning).
* [background_thread](http://jemalloc.net/jemalloc.3.html#background_thread)
Enabling jemalloc background threads generally improves the tail latency for
application threads, since unused memory purging is shifted to the dedicated
background threads. In addition, unintended purging delay caused by
application inactivity is avoided with background threads.
Suggested: `background_thread:true` when jemalloc managed threads can be
allowed.
* [metadata_thp](http://jemalloc.net/jemalloc.3.html#opt.metadata_thp)
Allowing jemalloc to utilize transparent huge pages for its internal
metadata usually reduces TLB misses significantly, especially for programs
with large memory footprint and frequent allocation / deallocation
activities. Metadata memory usage may increase due to the use of huge
pages.
Suggested for allocation intensive programs: `metadata_thp:auto` or
`metadata_thp:always`, which is expected to improve CPU utilization at a
small memory cost.
* [dirty_decay_ms](http://jemalloc.net/jemalloc.3.html#opt.dirty_decay_ms) and
[muzzy_decay_ms](http://jemalloc.net/jemalloc.3.html#opt.muzzy_decay_ms)
Decay time determines how fast jemalloc returns unused pages back to the
operating system, and therefore provides a fairly straightforward trade-off
between CPU and memory usage. Shorter decay time purges unused pages faster
to reduces memory usage (usually at the cost of more CPU cycles spent on
purging), and vice versa.
Suggested: tune the values based on the desired trade-offs.
* [narenas](http://jemalloc.net/jemalloc.3.html#opt.narenas)
By default jemalloc uses multiple arenas to reduce internal lock contention.
However high arena count may also increase overall memory fragmentation,
since arenas manage memory independently. When high degree of parallelism
is not expected at the allocator level, lower number of arenas often
improves memory usage.
Suggested: if low parallelism is expected, try lower arena count while
monitoring CPU and memory usage.
* [percpu_arena](http://jemalloc.net/jemalloc.3.html#opt.percpu_arena)
Enable dynamic thread to arena association based on running CPU. This has
the potential to improve locality, e.g. when thread to CPU affinity is
present.
Suggested: try `percpu_arena:percpu` or `percpu_arena:phycpu` if
thread migration between processors is expected to be infrequent.
Examples:
* High resource consumption application, prioritizing CPU utilization:
`background_thread:true,metadata_thp:auto` combined with relaxed decay time
(increased `dirty_decay_ms` and / or `muzzy_decay_ms`,
e.g. `dirty_decay_ms:30000,muzzy_decay_ms:30000`).
* High resource consumption application, prioritizing memory usage:
`background_thread:true` combined with shorter decay time (decreased
`dirty_decay_ms` and / or `muzzy_decay_ms`,
e.g. `dirty_decay_ms:5000,muzzy_decay_ms:5000`), and lower arena count
(e.g. number of CPUs).
* Low resource consumption application:
`narenas:1,lg_tcache_max:13` combined with shorter decay time (decreased
`dirty_decay_ms` and / or `muzzy_decay_ms`,e.g.
`dirty_decay_ms:1000,muzzy_decay_ms:0`).
* Extremely conservative -- minimize memory usage at all costs, only suitable when
allocation activity is very rare:
`narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0`
Note that it is recommended to combine the options with `abort_conf:true` which
aborts immediately on illegal options.
## Beyond runtime options
In addition to the runtime options, there are a number of programmatic ways to
improve application performance with jemalloc.
* [Explicit arenas](http://jemalloc.net/jemalloc.3.html#arenas.create)
Manually created arenas can help performance in various ways, e.g. by
managing locality and contention for specific usages. For example,
applications can explicitly allocate frequently accessed objects from a
dedicated arena with
[mallocx()](http://jemalloc.net/jemalloc.3.html#MALLOCX_ARENA) to improve
locality. In addition, explicit arenas often benefit from individually
tuned options, e.g. relaxed [decay
time](http://jemalloc.net/jemalloc.3.html#arena.i.dirty_decay_ms) if
frequent reuse is expected.
* [Extent hooks](http://jemalloc.net/jemalloc.3.html#arena.i.extent_hooks)
Extent hooks allow customization for managing underlying memory. One use
case for performance purpose is to utilize huge pages -- for example,
[HHVM](https://github.com/facebook/hhvm/blob/master/hphp/util/alloc.cpp)
uses explicit arenas with customized extent hooks to manage 1GB huge pages
for frequently accessed data, which reduces TLB misses significantly.
* [Explicit thread-to-arena
binding](http://jemalloc.net/jemalloc.3.html#thread.arena)
It is common for some threads in an application to have different memory
access / allocation patterns. Threads with heavy workloads often benefit
from explicit binding, e.g. binding very active threads to dedicated arenas
may reduce contention at the allocator level.
deps/jemalloc/VERSION
Vendored
+1 -1
View File
@@ -1 +1 @@
4.0.3-0-ge9192eacf8935e29fc62fddc2701f7942b1cc02c
5.1.0-0-g0
deps/jemalloc/bin/jemalloc-config.in
Vendored
+4
View File
@@ -18,6 +18,7 @@ Options:
--cc : Print compiler used to build jemalloc.
--cflags : Print compiler flags used to build jemalloc.
--cppflags : Print preprocessor flags used to build jemalloc.
--cxxflags : Print C++ compiler flags used to build jemalloc.
--ldflags : Print library flags used to build jemalloc.
--libs : Print libraries jemalloc was linked against.
EOF
@@ -67,6 +68,9 @@ case "$1" in
--cppflags)
echo "@CPPFLAGS@"
;;
--cxxflags)
echo "@CXXFLAGS@"
;;
--ldflags)
echo "@LDFLAGS@ @EXTRA_LDFLAGS@"
;;
deps/jemalloc/bin/jeprof.in
Vendored
+143 -29
View File
@@ -71,6 +71,7 @@
use strict;
use warnings;
use Getopt::Long;
use Cwd;
my $JEPROF_VERSION = "@jemalloc_version@";
my $PPROF_VERSION = "2.0";
@@ -95,7 +96,7 @@ my @EVINCE = ("evince"); # could also be xpdf or perhaps acroread
my @KCACHEGRIND = ("kcachegrind");
my @PS2PDF = ("ps2pdf");
# These are used for dynamic profiles
my @URL_FETCHER = ("curl", "-s");
my @URL_FETCHER = ("curl", "-s", "--fail");
# These are the web pages that servers need to support for dynamic profiles
my $HEAP_PAGE = "/pprof/heap";
@@ -223,12 +224,14 @@ Call-graph Options:
--nodefraction=<f> Hide nodes below <f>*total [default=.005]
--edgefraction=<f> Hide edges below <f>*total [default=.001]
--maxdegree=<n> Max incoming/outgoing edges per node [default=8]
--focus=<regexp> Focus on nodes matching <regexp>
--focus=<regexp> Focus on backtraces with nodes matching <regexp>
--thread=<n> Show profile for thread <n>
--ignore=<regexp> Ignore nodes matching <regexp>
--ignore=<regexp> Ignore backtraces with nodes matching <regexp>
--scale=<n> Set GV scaling [default=0]
--heapcheck Make nodes with non-0 object counts
(i.e. direct leak generators) more visible
--retain=<regexp> Retain only nodes that match <regexp>
--exclude=<regexp> Exclude all nodes that match <regexp>
Miscellaneous:
--tools=<prefix or binary:fullpath>[,...] \$PATH for object tool pathnames
@@ -339,6 +342,8 @@ sub Init() {
$main::opt_ignore = '';
$main::opt_scale = 0;
$main::opt_heapcheck = 0;
$main::opt_retain = '';
$main::opt_exclude = '';
$main::opt_seconds = 30;
$main::opt_lib = "";
@@ -410,6 +415,8 @@ sub Init() {
"ignore=s" => \$main::opt_ignore,
"scale=i" => \$main::opt_scale,
"heapcheck" => \$main::opt_heapcheck,
"retain=s" => \$main::opt_retain,
"exclude=s" => \$main::opt_exclude,
"inuse_space!" => \$main::opt_inuse_space,
"inuse_objects!" => \$main::opt_inuse_objects,
"alloc_space!" => \$main::opt_alloc_space,
@@ -1160,8 +1167,21 @@ sub PrintSymbolizedProfile {
}
print '---', "\n";
$PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash
my $profile_marker = $&;
my $profile_marker;
if ($main::profile_type eq 'heap') {
$HEAP_PAGE =~ m,[^/]+$,; # matches everything after the last slash
$profile_marker = $&;
} elsif ($main::profile_type eq 'growth') {
$GROWTH_PAGE =~ m,[^/]+$,; # matches everything after the last slash
$profile_marker = $&;
} elsif ($main::profile_type eq 'contention') {
$CONTENTION_PAGE =~ m,[^/]+$,; # matches everything after the last slash
$profile_marker = $&;
} else { # elsif ($main::profile_type eq 'cpu')
$PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash
$profile_marker = $&;
}
print '--- ', $profile_marker, "\n";
if (defined($main::collected_profile)) {
# if used with remote fetch, simply dump the collected profile to output.
@@ -1171,6 +1191,12 @@ sub PrintSymbolizedProfile {
}
close(SRC);
} else {
# --raw/http: For everything to work correctly for non-remote profiles, we
# would need to extend PrintProfileData() to handle all possible profile
# types, re-enable the code that is currently disabled in ReadCPUProfile()
# and FixCallerAddresses(), and remove the remote profile dumping code in
# the block above.
die "--raw/http: jeprof can only dump remote profiles for --raw\n";
# dump a cpu-format profile to standard out
PrintProfileData($profile);
}
@@ -2821,6 +2847,43 @@ sub ExtractCalls {
return $calls;
}
sub FilterFrames {
my $symbols = shift;
my $profile = shift;
if ($main::opt_retain eq '' && $main::opt_exclude eq '') {
return $profile;
}
my $result = {};
foreach my $k (keys(%{$profile})) {
my $count = $profile->{$k};
my @addrs = split(/\n/, $k);
my @path = ();
foreach my $a (@addrs) {
my $sym;
if (exists($symbols->{$a})) {
$sym = $symbols->{$a}->[0];
} else {
$sym = $a;
}
if ($main::opt_retain ne '' && $sym !~ m/$main::opt_retain/) {
next;
}
if ($main::opt_exclude ne '' && $sym =~ m/$main::opt_exclude/) {
next;
}
push(@path, $a);
}
if (scalar(@path) > 0) {
my $reduced_path = join("\n", @path);
AddEntry($result, $reduced_path, $count);
}
}
return $result;
}
sub RemoveUninterestingFrames {
my $symbols = shift;
my $profile = shift;
@@ -2829,21 +2892,23 @@ sub RemoveUninterestingFrames {
my %skip = ();
my $skip_regexp = 'NOMATCH';
if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') {
foreach my $name ('calloc',
foreach my $name ('@JEMALLOC_PREFIX@calloc',
'cfree',
'malloc',
'free',
'memalign',
'posix_memalign',
'aligned_alloc',
'@JEMALLOC_PREFIX@malloc',
'newImpl',
'void* newImpl',
'@JEMALLOC_PREFIX@free',
'@JEMALLOC_PREFIX@memalign',
'@JEMALLOC_PREFIX@posix_memalign',
'@JEMALLOC_PREFIX@aligned_alloc',
'pvalloc',
'valloc',
'realloc',
'mallocx', # jemalloc
'rallocx', # jemalloc
'xallocx', # jemalloc
'dallocx', # jemalloc
'sdallocx', # jemalloc
'@JEMALLOC_PREFIX@valloc',
'@JEMALLOC_PREFIX@realloc',
'@JEMALLOC_PREFIX@mallocx',
'@JEMALLOC_PREFIX@rallocx',
'@JEMALLOC_PREFIX@xallocx',
'@JEMALLOC_PREFIX@dallocx',
'@JEMALLOC_PREFIX@sdallocx',
'tc_calloc',
'tc_cfree',
'tc_malloc',
@@ -2965,6 +3030,9 @@ sub RemoveUninterestingFrames {
my $reduced_path = join("\n", @path);
AddEntry($result, $reduced_path, $count);
}
$result = FilterFrames($symbols, $result);
return $result;
}
@@ -3274,7 +3342,7 @@ sub ResolveRedirectionForCurl {
# Add a timeout flat to URL_FETCHER. Returns a new list.
sub AddFetchTimeout {
my $timeout = shift;
my @fetcher = shift;
my @fetcher = @_;
if (defined($timeout)) {
if (join(" ", @fetcher) =~ m/\bcurl -s/) {
push(@fetcher, "--max-time", sprintf("%d", $timeout));
@@ -3320,6 +3388,27 @@ sub ReadSymbols {
return $map;
}
sub URLEncode {
my $str = shift;
$str =~ s/([^A-Za-z0-9\-_.!~*'()])/ sprintf "%%%02x", ord $1 /eg;
return $str;
}
sub AppendSymbolFilterParams {
my $url = shift;
my @params = ();
if ($main::opt_retain ne '') {
push(@params, sprintf("retain=%s", URLEncode($main::opt_retain)));
}
if ($main::opt_exclude ne '') {
push(@params, sprintf("exclude=%s", URLEncode($main::opt_exclude)));
}
if (scalar @params > 0) {
$url = sprintf("%s?%s", $url, join("&", @params));
}
return $url;
}
# Fetches and processes symbols to prepare them for use in the profile output
# code. If the optional 'symbol_map' arg is not given, fetches symbols from
# $SYMBOL_PAGE for all PC values found in profile. Otherwise, the raw symbols
@@ -3344,9 +3433,11 @@ sub FetchSymbols {
my $command_line;
if (join(" ", @URL_FETCHER) =~ m/\bcurl -s/) {
$url = ResolveRedirectionForCurl($url);
$url = AppendSymbolFilterParams($url);
$command_line = ShellEscape(@URL_FETCHER, "-d", "\@$main::tmpfile_sym",
$url);
} else {
$url = AppendSymbolFilterParams($url);
$command_line = (ShellEscape(@URL_FETCHER, "--post", $url)
. " < " . ShellEscape($main::tmpfile_sym));
}
@@ -3427,12 +3518,22 @@ sub FetchDynamicProfile {
}
$url .= sprintf("seconds=%d", $main::opt_seconds);
$fetch_timeout = $main::opt_seconds * 1.01 + 60;
# Set $profile_type for consumption by PrintSymbolizedProfile.
$main::profile_type = 'cpu';
} else {
# For non-CPU profiles, we add a type-extension to
# the target profile file name.
my $suffix = $path;
$suffix =~ s,/,.,g;
$profile_file .= $suffix;
# Set $profile_type for consumption by PrintSymbolizedProfile.
if ($path =~ m/$HEAP_PAGE/) {
$main::profile_type = 'heap';
} elsif ($path =~ m/$GROWTH_PAGE/) {
$main::profile_type = 'growth';
} elsif ($path =~ m/$CONTENTION_PAGE/) {
$main::profile_type = 'contention';
}
}
my $profile_dir = $ENV{"JEPROF_TMPDIR"} || ($ENV{HOME} . "/jeprof");
@@ -3730,6 +3831,8 @@ sub ReadProfile {
my $symbol_marker = $&;
$PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash
my $profile_marker = $&;
$HEAP_PAGE =~ m,[^/]+$,; # matches everything after the last slash
my $heap_marker = $&;
# Look at first line to see if it is a heap or a CPU profile.
# CPU profile may start with no header at all, and just binary data
@@ -3756,7 +3859,13 @@ sub ReadProfile {
$header = ReadProfileHeader(*PROFILE) || "";
}
if ($header =~ m/^--- *($heap_marker|$growth_marker)/o) {
# Skip "--- ..." line for profile types that have their own headers.
$header = ReadProfileHeader(*PROFILE) || "";
}
$main::profile_type = '';
if ($header =~ m/^heap profile:.*$growth_marker/o) {
$main::profile_type = 'growth';
$result = ReadHeapProfile($prog, *PROFILE, $header);
@@ -3808,9 +3917,9 @@ sub ReadProfile {
# independent implementation.
sub FixCallerAddresses {
my $stack = shift;
if ($main::use_symbolized_profile) {
return $stack;
} else {
# --raw/http: Always subtract one from pc's, because PrintSymbolizedProfile()
# dumps unadjusted profiles.
{
$stack =~ /(\s)/;
my $delimiter = $1;
my @addrs = split(' ', $stack);
@@ -3878,12 +3987,7 @@ sub ReadCPUProfile {
for (my $j = 0; $j < $d; $j++) {
my $pc = $slots->get($i+$j);
# Subtract one from caller pc so we map back to call instr.
# However, don't do this if we're reading a symbolized profile
# file, in which case the subtract-one was done when the file
# was written.
if ($j > 0 && !$main::use_symbolized_profile) {
$pc--;
}
$pc--;
$pc = sprintf("%0*x", $address_length, $pc);
$pcs->{$pc} = 1;
push @k, $pc;
@@ -4469,7 +4573,7 @@ sub ParseTextSectionHeader {
# Split /proc/pid/maps dump into a list of libraries
sub ParseLibraries {
return if $main::use_symbol_page; # We don't need libraries info.
my $prog = shift;
my $prog = Cwd::abs_path(shift);
my $map = shift;
my $pcs = shift;
@@ -4502,6 +4606,16 @@ sub ParseLibraries {
$finish = HexExtend($2);
$offset = $zero_offset;
$lib = $3;
} elsif (($l =~ /^($h)-($h)\s+..x.\s+($h)\s+\S+:\S+\s+\d+\s+(\S+)$/i) && ($4 eq $prog)) {
# PIEs and address space randomization do not play well with our
# default assumption that main executable is at lowest
# addresses. So we're detecting main executable in
# /proc/self/maps as well.
$start = HexExtend($1);
$finish = HexExtend($2);
$offset = HexExtend($3);
$lib = $4;
$lib =~ s|\\|/|g; # turn windows-style paths into unix-style paths
}
# FreeBSD 10.0 virtual memory map /proc/curproc/map as defined in
# function procfs_doprocmap (sys/fs/procfs/procfs_map.c)
deps/jemalloc/config.guess → deps/jemalloc/build-aux/config.guess
Vendored
+108 -66
View File
@@ -1,8 +1,8 @@
#! /bin/sh
# Attempt to guess a canonical system name.
# Copyright 1992-2014 Free Software Foundation, Inc.
# Copyright 1992-2016 Free Software Foundation, Inc.
timestamp='2014-03-23'
timestamp='2016-10-02'
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
@@ -24,12 +24,12 @@ timestamp='2014-03-23'
# program. This Exception is an additional permission under section 7
# of the GNU General Public License, version 3 ("GPLv3").
#
# Originally written by Per Bothner.
# Originally written by Per Bothner; maintained since 2000 by Ben Elliston.
#
# You can get the latest version of this script from:
# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
#
# Please send patches with a ChangeLog entry to config-patches@gnu.org.
# Please send patches to <config-patches@gnu.org>.
me=`echo "$0" | sed -e 's,.*/,,'`
@@ -50,7 +50,7 @@ version="\
GNU config.guess ($timestamp)
Originally written by Per Bothner.
Copyright 1992-2014 Free Software Foundation, Inc.
Copyright 1992-2016 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -168,19 +168,29 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
# Note: NetBSD doesn't particularly care about the vendor
# portion of the name. We always set it to "unknown".
sysctl="sysctl -n hw.machine_arch"
UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \
/usr/sbin/$sysctl 2>/dev/null || echo unknown)`
UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \
/sbin/$sysctl 2>/dev/null || \
/usr/sbin/$sysctl 2>/dev/null || \
echo unknown)`
case "${UNAME_MACHINE_ARCH}" in
armeb) machine=armeb-unknown ;;
arm*) machine=arm-unknown ;;
sh3el) machine=shl-unknown ;;
sh3eb) machine=sh-unknown ;;
sh5el) machine=sh5le-unknown ;;
earmv*)
arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'`
endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'`
machine=${arch}${endian}-unknown
;;
*) machine=${UNAME_MACHINE_ARCH}-unknown ;;
esac
# The Operating System including object format, if it has switched
# to ELF recently, or will in the future.
# to ELF recently (or will in the future) and ABI.
case "${UNAME_MACHINE_ARCH}" in
earm*)
os=netbsdelf
;;
arm*|i386|m68k|ns32k|sh3*|sparc|vax)
eval $set_cc_for_build
if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
@@ -197,6 +207,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
os=netbsd
;;
esac
# Determine ABI tags.
case "${UNAME_MACHINE_ARCH}" in
earm*)
expr='s/^earmv[0-9]/-eabi/;s/eb$//'
abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"`
;;
esac
# The OS release
# Debian GNU/NetBSD machines have a different userland, and
# thus, need a distinct triplet. However, they do not need
@@ -207,13 +224,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
release='-gnu'
;;
*)
release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2`
;;
esac
# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
# contains redundant information, the shorter form:
# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
echo "${machine}-${os}${release}"
echo "${machine}-${os}${release}${abi}"
exit ;;
*:Bitrig:*:*)
UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
@@ -223,6 +240,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE}
exit ;;
*:LibertyBSD:*:*)
UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'`
echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE}
exit ;;
*:ekkoBSD:*:*)
echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE}
exit ;;
@@ -235,6 +256,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
*:MirBSD:*:*)
echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE}
exit ;;
*:Sortix:*:*)
echo ${UNAME_MACHINE}-unknown-sortix
exit ;;
alpha:OSF1:*:*)
case $UNAME_RELEASE in
*4.0)
@@ -251,42 +275,42 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1`
case "$ALPHA_CPU_TYPE" in
"EV4 (21064)")
UNAME_MACHINE="alpha" ;;
UNAME_MACHINE=alpha ;;
"EV4.5 (21064)")
UNAME_MACHINE="alpha" ;;
UNAME_MACHINE=alpha ;;
"LCA4 (21066/21068)")
UNAME_MACHINE="alpha" ;;
UNAME_MACHINE=alpha ;;
"EV5 (21164)")
UNAME_MACHINE="alphaev5" ;;
UNAME_MACHINE=alphaev5 ;;
"EV5.6 (21164A)")
UNAME_MACHINE="alphaev56" ;;
UNAME_MACHINE=alphaev56 ;;
"EV5.6 (21164PC)")
UNAME_MACHINE="alphapca56" ;;
UNAME_MACHINE=alphapca56 ;;
"EV5.7 (21164PC)")
UNAME_MACHINE="alphapca57" ;;
UNAME_MACHINE=alphapca57 ;;
"EV6 (21264)")
UNAME_MACHINE="alphaev6" ;;
UNAME_MACHINE=alphaev6 ;;
"EV6.7 (21264A)")
UNAME_MACHINE="alphaev67" ;;
UNAME_MACHINE=alphaev67 ;;
"EV6.8CB (21264C)")
UNAME_MACHINE="alphaev68" ;;
UNAME_MACHINE=alphaev68 ;;
"EV6.8AL (21264B)")
UNAME_MACHINE="alphaev68" ;;
UNAME_MACHINE=alphaev68 ;;
"EV6.8CX (21264D)")
UNAME_MACHINE="alphaev68" ;;
UNAME_MACHINE=alphaev68 ;;
"EV6.9A (21264/EV69A)")
UNAME_MACHINE="alphaev69" ;;
UNAME_MACHINE=alphaev69 ;;
"EV7 (21364)")
UNAME_MACHINE="alphaev7" ;;
UNAME_MACHINE=alphaev7 ;;
"EV7.9 (21364A)")
UNAME_MACHINE="alphaev79" ;;
UNAME_MACHINE=alphaev79 ;;
esac
# A Pn.n version is a patched version.
# A Vn.n version is a released version.
# A Tn.n version is a released field test version.
# A Xn.n version is an unreleased experimental baselevel.
# 1.2 uses "1.2" for uname -r.
echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
# Reset EXIT trap before exiting to avoid spurious non-zero exit code.
exitcode=$?
trap '' 0
@@ -359,16 +383,16 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
exit ;;
i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*)
eval $set_cc_for_build
SUN_ARCH="i386"
SUN_ARCH=i386
# If there is a compiler, see if it is configured for 64-bit objects.
# Note that the Sun cc does not turn __LP64__ into 1 like gcc does.
# This test works for both compilers.
if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \
(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
grep IS_64BIT_ARCH >/dev/null
then
SUN_ARCH="x86_64"
SUN_ARCH=x86_64
fi
fi
echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
@@ -393,7 +417,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
exit ;;
sun*:*:4.2BSD:*)
UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null`
test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3
test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3
case "`/bin/arch`" in
sun3)
echo m68k-sun-sunos${UNAME_RELEASE}
@@ -579,8 +603,9 @@ EOF
else
IBM_ARCH=powerpc
fi
if [ -x /usr/bin/oslevel ] ; then
IBM_REV=`/usr/bin/oslevel`
if [ -x /usr/bin/lslpp ] ; then
IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc |
awk -F: '{ print $3 }' | sed s/[0-9]*$/0/`
else
IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
fi
@@ -617,13 +642,13 @@ EOF
sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
case "${sc_cpu_version}" in
523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0
528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1
532) # CPU_PA_RISC2_0
case "${sc_kernel_bits}" in
32) HP_ARCH="hppa2.0n" ;;
64) HP_ARCH="hppa2.0w" ;;
'') HP_ARCH="hppa2.0" ;; # HP-UX 10.20
32) HP_ARCH=hppa2.0n ;;
64) HP_ARCH=hppa2.0w ;;
'') HP_ARCH=hppa2.0 ;; # HP-UX 10.20
esac ;;
esac
fi
@@ -662,11 +687,11 @@ EOF
exit (0);
}
EOF
(CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
(CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
test -z "$HP_ARCH" && HP_ARCH=hppa
fi ;;
esac
if [ ${HP_ARCH} = "hppa2.0w" ]
if [ ${HP_ARCH} = hppa2.0w ]
then
eval $set_cc_for_build
@@ -679,12 +704,12 @@ EOF
# $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess
# => hppa64-hp-hpux11.23
if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) |
if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) |
grep -q __LP64__
then
HP_ARCH="hppa2.0w"
HP_ARCH=hppa2.0w
else
HP_ARCH="hppa64"
HP_ARCH=hppa64
fi
fi
echo ${HP_ARCH}-hp-hpux${HPUX_REV}
@@ -789,14 +814,14 @@ EOF
echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
exit ;;
F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
exit ;;
5000:UNIX_System_V:4.*:*)
FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'`
echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
exit ;;
i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
@@ -878,7 +903,7 @@ EOF
exit ;;
*:GNU/*:*:*)
# other systems with GNU libc and userland
echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC}
echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC}
exit ;;
i*86:Minix:*:*)
echo ${UNAME_MACHINE}-pc-minix
@@ -901,7 +926,7 @@ EOF
EV68*) UNAME_MACHINE=alphaev68 ;;
esac
objdump --private-headers /bin/sh | grep -q ld.so.1
if test "$?" = 0 ; then LIBC="gnulibc1" ; fi
if test "$?" = 0 ; then LIBC=gnulibc1 ; fi
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
arc:Linux:*:* | arceb:Linux:*:*)
@@ -932,6 +957,9 @@ EOF
crisv32:Linux:*:*)
echo ${UNAME_MACHINE}-axis-linux-${LIBC}
exit ;;
e2k:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
frv:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
@@ -944,6 +972,9 @@ EOF
ia64:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
k1om:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
m32r*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
@@ -969,6 +1000,9 @@ EOF
eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'`
test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; }
;;
mips64el:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
openrisc*:Linux:*:*)
echo or1k-unknown-linux-${LIBC}
exit ;;
@@ -1001,6 +1035,9 @@ EOF
ppcle:Linux:*:*)
echo powerpcle-unknown-linux-${LIBC}
exit ;;
riscv32:Linux:*:* | riscv64:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
s390:Linux:*:* | s390x:Linux:*:*)
echo ${UNAME_MACHINE}-ibm-linux-${LIBC}
exit ;;
@@ -1020,7 +1057,7 @@ EOF
echo ${UNAME_MACHINE}-dec-linux-${LIBC}
exit ;;
x86_64:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
echo ${UNAME_MACHINE}-pc-linux-${LIBC}
exit ;;
xtensa*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
@@ -1099,7 +1136,7 @@ EOF
# uname -m prints for DJGPP always 'pc', but it prints nothing about
# the processor, so we play safe by assuming i586.
# Note: whatever this is, it MUST be the same as what config.sub
# prints for the "djgpp" host, or else GDB configury will decide that
# prints for the "djgpp" host, or else GDB configure will decide that
# this is a cross-build.
echo i586-pc-msdosdjgpp
exit ;;
@@ -1248,6 +1285,9 @@ EOF
SX-8R:SUPER-UX:*:*)
echo sx8r-nec-superux${UNAME_RELEASE}
exit ;;
SX-ACE:SUPER-UX:*:*)
echo sxace-nec-superux${UNAME_RELEASE}
exit ;;
Power*:Rhapsody:*:*)
echo powerpc-apple-rhapsody${UNAME_RELEASE}
exit ;;
@@ -1261,9 +1301,9 @@ EOF
UNAME_PROCESSOR=powerpc
fi
if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then
if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
grep IS_64BIT_ARCH >/dev/null
then
case $UNAME_PROCESSOR in
@@ -1285,7 +1325,7 @@ EOF
exit ;;
*:procnto*:*:* | *:QNX:[0123456789]*:*)
UNAME_PROCESSOR=`uname -p`
if test "$UNAME_PROCESSOR" = "x86"; then
if test "$UNAME_PROCESSOR" = x86; then
UNAME_PROCESSOR=i386
UNAME_MACHINE=pc
fi
@@ -1316,7 +1356,7 @@ EOF
# "uname -m" is not consistent, so use $cputype instead. 386
# is converted to i386 for consistency with other x86
# operating systems.
if test "$cputype" = "386"; then
if test "$cputype" = 386; then
UNAME_MACHINE=i386
else
UNAME_MACHINE="$cputype"
@@ -1358,7 +1398,7 @@ EOF
echo i386-pc-xenix
exit ;;
i*86:skyos:*:*)
echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//'
echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'`
exit ;;
i*86:rdos:*:*)
echo ${UNAME_MACHINE}-pc-rdos
@@ -1369,23 +1409,25 @@ EOF
x86_64:VMkernel:*:*)
echo ${UNAME_MACHINE}-unknown-esx
exit ;;
amd64:Isilon\ OneFS:*:*)
echo x86_64-unknown-onefs
exit ;;
esac
cat >&2 <<EOF
$0: unable to guess system type
This script, last modified $timestamp, has failed to recognize
the operating system you are using. It is advised that you
download the most up to date version of the config scripts from
This script (version $timestamp), has failed to recognize the
operating system you are using. If your script is old, overwrite
config.guess and config.sub with the latest versions from:
http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
and
http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
If the version you run ($0) is already up to date, please
send the following data and any information you think might be
pertinent to <config-patches@gnu.org> in order to provide the needed
information to handle your system.
If $0 has already been updated, send the following data and any
information you think might be pertinent to config-patches@gnu.org to
provide the necessary information to handle your system.
config.guess timestamp = $timestamp
deps/jemalloc/config.sub → deps/jemalloc/build-aux/config.sub
Vendored
+52 -24
View File
@@ -1,8 +1,8 @@
#! /bin/sh
# Configuration validation subroutine script.
# Copyright 1992-2014 Free Software Foundation, Inc.
# Copyright 1992-2016 Free Software Foundation, Inc.
timestamp='2014-05-01'
timestamp='2016-11-04'
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
@@ -25,7 +25,7 @@ timestamp='2014-05-01'
# of the GNU General Public License, version 3 ("GPLv3").
# Please send patches with a ChangeLog entry to config-patches@gnu.org.
# Please send patches to <config-patches@gnu.org>.
#
# Configuration subroutine to validate and canonicalize a configuration type.
# Supply the specified configuration type as an argument.
@@ -33,7 +33,7 @@ timestamp='2014-05-01'
# Otherwise, we print the canonical config type on stdout and succeed.
# You can get the latest version of this script from:
# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
# This file is supposed to be the same for all GNU packages
# and recognize all the CPU types, system types and aliases
@@ -53,8 +53,7 @@ timestamp='2014-05-01'
me=`echo "$0" | sed -e 's,.*/,,'`
usage="\
Usage: $0 [OPTION] CPU-MFR-OPSYS
$0 [OPTION] ALIAS
Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS
Canonicalize a configuration name.
@@ -68,7 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>."
version="\
GNU config.sub ($timestamp)
Copyright 1992-2014 Free Software Foundation, Inc.
Copyright 1992-2016 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -117,8 +116,8 @@ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
case $maybe_os in
nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
knetbsd*-gnu* | netbsd*-gnu* | \
kopensolaris*-gnu* | \
knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \
kopensolaris*-gnu* | cloudabi*-eabi* | \
storm-chaos* | os2-emx* | rtmk-nova*)
os=-$maybe_os
basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
@@ -255,12 +254,13 @@ case $basic_machine in
| arc | arceb \
| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
| avr | avr32 \
| ba \
| be32 | be64 \
| bfin \
| c4x | c8051 | clipper \
| d10v | d30v | dlx | dsp16xx \
| epiphany \
| fido | fr30 | frv \
| e2k | epiphany \
| fido | fr30 | frv | ft32 \
| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
| hexagon \
| i370 | i860 | i960 | ia64 \
@@ -301,10 +301,12 @@ case $basic_machine in
| open8 | or1k | or1knd | or32 \
| pdp10 | pdp11 | pj | pjl \
| powerpc | powerpc64 | powerpc64le | powerpcle \
| pru \
| pyramid \
| riscv32 | riscv64 \
| rl78 | rx \
| score \
| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
| sh64 | sh64le \
| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
@@ -312,6 +314,7 @@ case $basic_machine in
| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
| ubicom32 \
| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
| visium \
| we32k \
| x86 | xc16x | xstormy16 | xtensa \
| z8k | z80)
@@ -326,6 +329,9 @@ case $basic_machine in
c6x)
basic_machine=tic6x-unknown
;;
leon|leon[3-9])
basic_machine=sparc-$basic_machine
;;
m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip)
basic_machine=$basic_machine-unknown
os=-none
@@ -371,12 +377,13 @@ case $basic_machine in
| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
| arm-* | armbe-* | armle-* | armeb-* | armv*-* \
| avr-* | avr32-* \
| ba-* \
| be32-* | be64-* \
| bfin-* | bs2000-* \
| c[123]* | c30-* | [cjt]90-* | c4x-* \
| c8051-* | clipper-* | craynv-* | cydra-* \
| d10v-* | d30v-* | dlx-* \
| elxsi-* \
| e2k-* | elxsi-* \
| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
| h8300-* | h8500-* \
| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
@@ -422,13 +429,15 @@ case $basic_machine in
| orion-* \
| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
| pru-* \
| pyramid-* \
| riscv32-* | riscv64-* \
| rl78-* | romp-* | rs6000-* | rx-* \
| sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \
| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
| sparclite-* \
| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \
| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \
| tahoe-* \
| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
| tile*-* \
@@ -436,6 +445,7 @@ case $basic_machine in
| ubicom32-* \
| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
| vax-* \
| visium-* \
| we32k-* \
| x86-* | x86_64-* | xc16x-* | xps100-* \
| xstormy16-* | xtensa*-* \
@@ -512,6 +522,9 @@ case $basic_machine in
basic_machine=i386-pc
os=-aros
;;
asmjs)
basic_machine=asmjs-unknown
;;
aux)
basic_machine=m68k-apple
os=-aux
@@ -632,6 +645,14 @@ case $basic_machine in
basic_machine=m68k-bull
os=-sysv3
;;
e500v[12])
basic_machine=powerpc-unknown
os=$os"spe"
;;
e500v[12]-*)
basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
os=$os"spe"
;;
ebmon29k)
basic_machine=a29k-amd
os=-ebmon
@@ -773,6 +794,9 @@ case $basic_machine in
basic_machine=m68k-isi
os=-sysv
;;
leon-*|leon[3-9]-*)
basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'`
;;
m68knommu)
basic_machine=m68k-unknown
os=-linux
@@ -828,6 +852,10 @@ case $basic_machine in
basic_machine=powerpc-unknown
os=-morphos
;;
moxiebox)
basic_machine=moxie-unknown
os=-moxiebox
;;
msdos)
basic_machine=i386-pc
os=-msdos
@@ -1004,7 +1032,7 @@ case $basic_machine in
ppc-* | ppcbe-*)
basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
;;
ppcle | powerpclittle | ppc-le | powerpc-little)
ppcle | powerpclittle)
basic_machine=powerpcle-unknown
;;
ppcle-* | powerpclittle-*)
@@ -1014,7 +1042,7 @@ case $basic_machine in
;;
ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
;;
ppc64le | powerpc64little | ppc64-le | powerpc64-little)
ppc64le | powerpc64little)
basic_machine=powerpc64le-unknown
;;
ppc64le-* | powerpc64little-*)
@@ -1360,27 +1388,28 @@ case $os in
| -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
| -sym* | -kopensolaris* | -plan9* \
| -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
| -aos* | -aros* \
| -aos* | -aros* | -cloudabi* | -sortix* \
| -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
| -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
| -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
| -bitrig* | -openbsd* | -solidbsd* \
| -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \
| -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
| -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
| -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
| -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
| -chorusos* | -chorusrdb* | -cegcc* \
| -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
| -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
| -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
| -linux-newlib* | -linux-musl* | -linux-uclibc* \
| -uxpv* | -beos* | -mpeix* | -udk* \
| -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \
| -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
| -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
| -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
| -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
| -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
| -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
| -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* | -tirtos*)
| -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \
| -onefs* | -tirtos* | -phoenix* | -fuchsia*)
# Remember, each alternative MUST END IN *, to match a version number.
;;
-qnx*)
@@ -1404,9 +1433,6 @@ case $os in
-mac*)
os=`echo $os | sed -e 's|mac|macos|'`
;;
# Apple iOS
-ios*)
;;
-linux-dietlibc)
os=-linux-dietlibc
;;
@@ -1515,6 +1541,8 @@ case $os in
;;
-nacl*)
;;
-ios)
;;
-none)
;;
*)
deps/jemalloc/install-sh → deps/jemalloc/build-aux/install-sh
Vendored
View File
deps/jemalloc/configure
Vendored
+4950 -932
View File
File diff suppressed because it is too large Load Diff
deps/jemalloc/configure.ac
Vendored
+959 -410
View File
File diff suppressed because it is too large Load Diff
deps/jemalloc/coverage.sh
Vendored
-16
View File
@@ -1,16 +0,0 @@
#!/bin/sh
set -e
objdir=$1
suffix=$2
shift 2
objs=$@
gcov -b -p -f -o "${objdir}" ${objs}
# Move gcov outputs so that subsequent gcov invocations won't clobber results
# for the same sources with different compilation flags.
for f in `find . -maxdepth 1 -type f -name '*.gcov'` ; do
mv "${f}" "${f}.${suffix}"
done
deps/jemalloc/doc/html.xsl.in
Vendored
+1
View File
@@ -1,4 +1,5 @@
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
<xsl:import href="@XSLROOT@/html/docbook.xsl"/>
<xsl:import href="@abs_srcroot@doc/stylesheet.xsl"/>
<xsl:output method="xml" encoding="utf-8"/>
</xsl:stylesheet>
deps/jemalloc/doc/jemalloc.3
Vendored
-1961
View File
File diff suppressed because it is too large Load Diff
deps/jemalloc/doc/jemalloc.html
Vendored
-1825
View File
File diff suppressed because one or more lines are too long
deps/jemalloc/doc/jemalloc.xml.in
Vendored
+1270 -653
View File
File diff suppressed because it is too large Load Diff
deps/jemalloc/doc/stylesheet.xsl
Vendored
+5 -2
View File
@@ -1,7 +1,10 @@
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
<xsl:param name="funcsynopsis.style">ansi</xsl:param>
<xsl:param name="function.parens" select="1"/>
<xsl:param name="function.parens" select="0"/>
<xsl:template match="function">
<xsl:call-template name="inline.monoseq"/>
</xsl:template>
<xsl:template match="mallctl">
"<xsl:call-template name="inline.monoseq"/>"
<quote><xsl:call-template name="inline.monoseq"/></quote>
</xsl:template>
</xsl:stylesheet>
deps/jemalloc/include/jemalloc/internal/arena.h
Vendored
-1347
View File
File diff suppressed because it is too large Load Diff
deps/jemalloc/include/jemalloc/internal/arena_externs.h
Vendored
+94
View File
@@ -0,0 +1,94 @@
#ifndef JEMALLOC_INTERNAL_ARENA_EXTERNS_H
#define JEMALLOC_INTERNAL_ARENA_EXTERNS_H
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/stats.h"
extern ssize_t opt_dirty_decay_ms;
extern ssize_t opt_muzzy_decay_ms;
extern percpu_arena_mode_t opt_percpu_arena;
extern const char *percpu_arena_mode_names[];
extern const uint64_t h_steps[SMOOTHSTEP_NSTEPS];
extern malloc_mutex_t arenas_lock;
void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena,
unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms,
ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
bin_stats_t *bstats, arena_stats_large_t *lstats);
void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent);
#ifdef JEMALLOC_JET
size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr);
#endif
extent_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena,
size_t usize, size_t alignment, bool *zero);
void arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena,
extent_t *extent);
void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t oldsize);
void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t oldsize);
ssize_t arena_dirty_decay_ms_get(arena_t *arena);
bool arena_dirty_decay_ms_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_ms);
ssize_t arena_muzzy_decay_ms_get(arena_t *arena);
bool arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_ms);
void arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread,
bool all);
void arena_reset(tsd_t *tsd, arena_t *arena);
void arena_destroy(tsd_t *tsd, arena_t *arena);
void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
cache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes);
void arena_alloc_junk_small(void *ptr, const bin_info_t *bin_info,
bool zero);
typedef void (arena_dalloc_junk_small_t)(void *, const bin_info_t *);
extern arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small;
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
szind_t ind, bool zero);
void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, tcache_t *tcache);
void arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize);
void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
bool slow_path);
void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, void *ptr);
void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero);
void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, tcache_t *tcache);
dss_prec_t arena_dss_prec_get(arena_t *arena);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
ssize_t arena_dirty_decay_ms_default_get(void);
bool arena_dirty_decay_ms_default_set(ssize_t decay_ms);
ssize_t arena_muzzy_decay_ms_default_get(void);
bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
bool arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena,
size_t *old_limit, size_t *new_limit);
unsigned arena_nthreads_get(arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal);
size_t arena_extent_sn_next(arena_t *arena);
arena_t *arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
void arena_boot(void);
void arena_prefork0(tsdn_t *tsdn, arena_t *arena);
void arena_prefork1(tsdn_t *tsdn, arena_t *arena);
void arena_prefork2(tsdn_t *tsdn, arena_t *arena);
void arena_prefork3(tsdn_t *tsdn, arena_t *arena);
void arena_prefork4(tsdn_t *tsdn, arena_t *arena);
void arena_prefork5(tsdn_t *tsdn, arena_t *arena);
void arena_prefork6(tsdn_t *tsdn, arena_t *arena);
void arena_prefork7(tsdn_t *tsdn, arena_t *arena);
void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
void arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
#endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */
deps/jemalloc/include/jemalloc/internal/arena_inlines_a.h
Vendored
+57
View File
@@ -0,0 +1,57 @@
#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_A_H
#define JEMALLOC_INTERNAL_ARENA_INLINES_A_H
static inline unsigned
arena_ind_get(const arena_t *arena) {
return base_ind_get(arena->base);
}
static inline void
arena_internal_add(arena_t *arena, size_t size) {
atomic_fetch_add_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
}
static inline void
arena_internal_sub(arena_t *arena, size_t size) {
atomic_fetch_sub_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
}
static inline size_t
arena_internal_get(arena_t *arena) {
return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
}
static inline bool
arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) {
cassert(config_prof);
if (likely(prof_interval == 0 || !prof_active_get_unlocked())) {
return false;
}
return prof_accum_add(tsdn, &arena->prof_accum, accumbytes);
}
static inline void
percpu_arena_update(tsd_t *tsd, unsigned cpu) {
assert(have_percpu_arena);
arena_t *oldarena = tsd_arena_get(tsd);
assert(oldarena != NULL);
unsigned oldind = arena_ind_get(oldarena);
if (oldind != cpu) {
unsigned newind = cpu;
arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
assert(newarena != NULL);
/* Set new arena/tcache associations. */
arena_migrate(tsd, oldind, newind);
tcache_t *tcache = tcache_get(tsd);
if (tcache != NULL) {
tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
newarena);
}
}
}
#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_A_H */
deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h
Vendored
+354
View File
@@ -0,0 +1,354 @@
#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H
#define JEMALLOC_INTERNAL_ARENA_INLINES_B_H
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/ticker.h"
JEMALLOC_ALWAYS_INLINE prof_tctx_t *
arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) {
cassert(config_prof);
assert(ptr != NULL);
/* Static check. */
if (alloc_ctx == NULL) {
const extent_t *extent = iealloc(tsdn, ptr);
if (unlikely(!extent_slab_get(extent))) {
return large_prof_tctx_get(tsdn, extent);
}
} else {
if (unlikely(!alloc_ctx->slab)) {
return large_prof_tctx_get(tsdn, iealloc(tsdn, ptr));
}
}
return (prof_tctx_t *)(uintptr_t)1U;
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize,
alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) {
cassert(config_prof);
assert(ptr != NULL);
/* Static check. */
if (alloc_ctx == NULL) {
extent_t *extent = iealloc(tsdn, ptr);
if (unlikely(!extent_slab_get(extent))) {
large_prof_tctx_set(tsdn, extent, tctx);
}
} else {
if (unlikely(!alloc_ctx->slab)) {
large_prof_tctx_set(tsdn, iealloc(tsdn, ptr), tctx);
}
}
}
static inline void
arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) {
cassert(config_prof);
assert(ptr != NULL);
extent_t *extent = iealloc(tsdn, ptr);
assert(!extent_slab_get(extent));
large_prof_tctx_reset(tsdn, extent);
}
JEMALLOC_ALWAYS_INLINE void
arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
tsd_t *tsd;
ticker_t *decay_ticker;
if (unlikely(tsdn_null(tsdn))) {
return;
}
tsd = tsdn_tsd(tsdn);
decay_ticker = decay_ticker_get(tsd, arena_ind_get(arena));
if (unlikely(decay_ticker == NULL)) {
return;
}
if (unlikely(ticker_ticks(decay_ticker, nticks))) {
arena_decay(tsdn, arena, false, false);
}
}
JEMALLOC_ALWAYS_INLINE void
arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_assert_not_owner(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_assert_not_owner(tsdn, &arena->decay_muzzy.mtx);
arena_decay_ticks(tsdn, arena, 1);
}
JEMALLOC_ALWAYS_INLINE void *
arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
tcache_t *tcache, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(size != 0);
if (likely(tcache != NULL)) {
if (likely(size <= SMALL_MAXCLASS)) {
return tcache_alloc_small(tsdn_tsd(tsdn), arena,
tcache, size, ind, zero, slow_path);
}
if (likely(size <= tcache_maxclass)) {
return tcache_alloc_large(tsdn_tsd(tsdn), arena,
tcache, size, ind, zero, slow_path);
}
/* (size > tcache_maxclass) case falls through. */
assert(size > tcache_maxclass);
}
return arena_malloc_hard(tsdn, arena, size, ind, zero);
}
JEMALLOC_ALWAYS_INLINE arena_t *
arena_aalloc(tsdn_t *tsdn, const void *ptr) {
return extent_arena_get(iealloc(tsdn, ptr));
}
JEMALLOC_ALWAYS_INLINE size_t
arena_salloc(tsdn_t *tsdn, const void *ptr) {
assert(ptr != NULL);
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true);
assert(szind != NSIZES);
return sz_index2size(szind);
}
JEMALLOC_ALWAYS_INLINE size_t
arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
/*
* Return 0 if ptr is not within an extent managed by jemalloc. This
* function has two extra costs relative to isalloc():
* - The rtree calls cannot claim to be dependent lookups, which induces
* rtree lookup load dependencies.
* - The lookup may fail, so there is an extra branch to check for
* failure.
*/
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
extent_t *extent;
szind_t szind;
if (rtree_extent_szind_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)ptr, false, &extent, &szind)) {
return 0;
}
if (extent == NULL) {
return 0;
}
assert(extent_state_get(extent) == extent_state_active);
/* Only slab members should be looked up via interior pointers. */
assert(extent_addr_get(extent) == ptr || extent_slab_get(extent));
assert(szind != NSIZES);
return sz_index2size(szind);
}
static inline void
arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
assert(ptr != NULL);
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
szind_t szind;
bool slab;
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
true, &szind, &slab);
if (config_debug) {
extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
rtree_ctx, (uintptr_t)ptr, true);
assert(szind == extent_szind_get(extent));
assert(szind < NSIZES);
assert(slab == extent_slab_get(extent));
}
if (likely(slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
extent_t *extent = iealloc(tsdn, ptr);
large_dalloc(tsdn, extent);
}
}
JEMALLOC_ALWAYS_INLINE void
arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
alloc_ctx_t *alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
if (unlikely(tcache == NULL)) {
arena_dalloc_no_tcache(tsdn, ptr);
return;
}
szind_t szind;
bool slab;
rtree_ctx_t *rtree_ctx;
if (alloc_ctx != NULL) {
szind = alloc_ctx->szind;
slab = alloc_ctx->slab;
assert(szind != NSIZES);
} else {
rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &szind, &slab);
}
if (config_debug) {
rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
rtree_ctx, (uintptr_t)ptr, true);
assert(szind == extent_szind_get(extent));
assert(szind < NSIZES);
assert(slab == extent_slab_get(extent));
}
if (likely(slab)) {
/* Small allocation. */
tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
slow_path);
} else {
if (szind < nhbins) {
if (config_prof && unlikely(szind < NBINS)) {
arena_dalloc_promoted(tsdn, ptr, tcache,
slow_path);
} else {
tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
szind, slow_path);
}
} else {
extent_t *extent = iealloc(tsdn, ptr);
large_dalloc(tsdn, extent);
}
}
}
static inline void
arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
assert(ptr != NULL);
assert(size <= LARGE_MAXCLASS);
szind_t szind;
bool slab;
if (!config_prof || !opt_prof) {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
szind = sz_size2index(size);
slab = (szind < NBINS);
}
if ((config_prof && opt_prof) || config_debug) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
&rtree_ctx_fallback);
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &szind, &slab);
assert(szind == sz_size2index(size));
assert((config_prof && opt_prof) || slab == (szind < NBINS));
if (config_debug) {
extent_t *extent = rtree_extent_read(tsdn,
&extents_rtree, rtree_ctx, (uintptr_t)ptr, true);
assert(szind == extent_szind_get(extent));
assert(slab == extent_slab_get(extent));
}
}
if (likely(slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
extent_t *extent = iealloc(tsdn, ptr);
large_dalloc(tsdn, extent);
}
}
JEMALLOC_ALWAYS_INLINE void
arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
alloc_ctx_t *alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
assert(size <= LARGE_MAXCLASS);
if (unlikely(tcache == NULL)) {
arena_sdalloc_no_tcache(tsdn, ptr, size);
return;
}
szind_t szind;
bool slab;
UNUSED alloc_ctx_t local_ctx;
if (config_prof && opt_prof) {
if (alloc_ctx == NULL) {
/* Uncommon case and should be a static check. */
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
&rtree_ctx_fallback);
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &local_ctx.szind,
&local_ctx.slab);
assert(local_ctx.szind == sz_size2index(size));
alloc_ctx = &local_ctx;
}
slab = alloc_ctx->slab;
szind = alloc_ctx->szind;
} else {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
szind = sz_size2index(size);
slab = (szind < NBINS);
}
if (config_debug) {
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &szind, &slab);
extent_t *extent = rtree_extent_read(tsdn,
&extents_rtree, rtree_ctx, (uintptr_t)ptr, true);
assert(szind == extent_szind_get(extent));
assert(slab == extent_slab_get(extent));
}
if (likely(slab)) {
/* Small allocation. */
tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
slow_path);
} else {
if (szind < nhbins) {
if (config_prof && unlikely(szind < NBINS)) {
arena_dalloc_promoted(tsdn, ptr, tcache,
slow_path);
} else {
tcache_dalloc_large(tsdn_tsd(tsdn),
tcache, ptr, szind, slow_path);
}
} else {
extent_t *extent = iealloc(tsdn, ptr);
large_dalloc(tsdn, extent);
}
}
}
#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */
deps/jemalloc/include/jemalloc/internal/arena_stats.h
Vendored
+237
View File
@@ -0,0 +1,237 @@
#ifndef JEMALLOC_INTERNAL_ARENA_STATS_H
#define JEMALLOC_INTERNAL_ARENA_STATS_H
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mutex_prof.h"
#include "jemalloc/internal/size_classes.h"
/*
* In those architectures that support 64-bit atomics, we use atomic updates for
* our 64-bit values. Otherwise, we use a plain uint64_t and synchronize
* externally.
*/
#ifdef JEMALLOC_ATOMIC_U64
typedef atomic_u64_t arena_stats_u64_t;
#else
/* Must hold the arena stats mutex while reading atomically. */
typedef uint64_t arena_stats_u64_t;
#endif
typedef struct arena_stats_large_s arena_stats_large_t;
struct arena_stats_large_s {
/*
* Total number of allocation/deallocation requests served directly by
* the arena.
*/
arena_stats_u64_t nmalloc;
arena_stats_u64_t ndalloc;
/*
* Number of allocation requests that correspond to this size class.
* This includes requests served by tcache, though tcache only
* periodically merges into this counter.
*/
arena_stats_u64_t nrequests; /* Partially derived. */
/* Current number of allocations of this size class. */
size_t curlextents; /* Derived. */
};
typedef struct arena_stats_decay_s arena_stats_decay_t;
struct arena_stats_decay_s {
/* Total number of purge sweeps. */
arena_stats_u64_t npurge;
/* Total number of madvise calls made. */
arena_stats_u64_t nmadvise;
/* Total number of pages purged. */
arena_stats_u64_t purged;
};
/*
* Arena stats. Note that fields marked "derived" are not directly maintained
* within the arena code; rather their values are derived during stats merge
* requests.
*/
typedef struct arena_stats_s arena_stats_t;
struct arena_stats_s {
#ifndef JEMALLOC_ATOMIC_U64
malloc_mutex_t mtx;
#endif
/* Number of bytes currently mapped, excluding retained memory. */
atomic_zu_t mapped; /* Partially derived. */
/*
* Number of unused virtual memory bytes currently retained. Retained
* bytes are technically mapped (though always decommitted or purged),
* but they are excluded from the mapped statistic (above).
*/
atomic_zu_t retained; /* Derived. */
arena_stats_decay_t decay_dirty;
arena_stats_decay_t decay_muzzy;
atomic_zu_t base; /* Derived. */
atomic_zu_t internal;
atomic_zu_t resident; /* Derived. */
atomic_zu_t metadata_thp;
atomic_zu_t allocated_large; /* Derived. */
arena_stats_u64_t nmalloc_large; /* Derived. */
arena_stats_u64_t ndalloc_large; /* Derived. */
arena_stats_u64_t nrequests_large; /* Derived. */
/* Number of bytes cached in tcache associated with this arena. */
atomic_zu_t tcache_bytes; /* Derived. */
mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes];
/* One element for each large size class. */
arena_stats_large_t lstats[NSIZES - NBINS];
/* Arena uptime. */
nstime_t uptime;
};
static inline bool
arena_stats_init(UNUSED tsdn_t *tsdn, arena_stats_t *arena_stats) {
if (config_debug) {
for (size_t i = 0; i < sizeof(arena_stats_t); i++) {
assert(((char *)arena_stats)[i] == 0);
}
}
#ifndef JEMALLOC_ATOMIC_U64
if (malloc_mutex_init(&arena_stats->mtx, "arena_stats",
WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) {
return true;
}
#endif
/* Memory is zeroed, so there is no need to clear stats. */
return false;
}
static inline void
arena_stats_lock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
#ifndef JEMALLOC_ATOMIC_U64
malloc_mutex_lock(tsdn, &arena_stats->mtx);
#endif
}
static inline void
arena_stats_unlock(tsdn_t *tsdn, arena_stats_t *arena_stats) {
#ifndef JEMALLOC_ATOMIC_U64
malloc_mutex_unlock(tsdn, &arena_stats->mtx);
#endif
}
static inline uint64_t
arena_stats_read_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
arena_stats_u64_t *p) {
#ifdef JEMALLOC_ATOMIC_U64
return atomic_load_u64(p, ATOMIC_RELAXED);
#else
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
return *p;
#endif
}
static inline void
arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
arena_stats_u64_t *p, uint64_t x) {
#ifdef JEMALLOC_ATOMIC_U64
atomic_fetch_add_u64(p, x, ATOMIC_RELAXED);
#else
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
*p += x;
#endif
}
UNUSED static inline void
arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
arena_stats_u64_t *p, uint64_t x) {
#ifdef JEMALLOC_ATOMIC_U64
UNUSED uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED);
assert(r - x <= r);
#else
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
*p -= x;
assert(*p + x >= *p);
#endif
}
/*
* Non-atomically sets *dst += src. *dst needs external synchronization.
* This lets us avoid the cost of a fetch_add when its unnecessary (note that
* the types here are atomic).
*/
static inline void
arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) {
#ifdef JEMALLOC_ATOMIC_U64
uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED);
atomic_store_u64(dst, src + cur_dst, ATOMIC_RELAXED);
#else
*dst += src;
#endif
}
static inline size_t
arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) {
#ifdef JEMALLOC_ATOMIC_U64
return atomic_load_zu(p, ATOMIC_RELAXED);
#else
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
return atomic_load_zu(p, ATOMIC_RELAXED);
#endif
}
static inline void
arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p,
size_t x) {
#ifdef JEMALLOC_ATOMIC_U64
atomic_fetch_add_zu(p, x, ATOMIC_RELAXED);
#else
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
atomic_store_zu(p, cur + x, ATOMIC_RELAXED);
#endif
}
static inline void
arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p,
size_t x) {
#ifdef JEMALLOC_ATOMIC_U64
UNUSED size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED);
assert(r - x <= r);
#else
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
size_t cur = atomic_load_zu(p, ATOMIC_RELAXED);
atomic_store_zu(p, cur - x, ATOMIC_RELAXED);
#endif
}
/* Like the _u64 variant, needs an externally synchronized *dst. */
static inline void
arena_stats_accum_zu(atomic_zu_t *dst, size_t src) {
size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED);
atomic_store_zu(dst, src + cur_dst, ATOMIC_RELAXED);
}
static inline void
arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
szind_t szind, uint64_t nrequests) {
arena_stats_lock(tsdn, arena_stats);
arena_stats_add_u64(tsdn, arena_stats, &arena_stats->lstats[szind -
NBINS].nrequests, nrequests);
arena_stats_unlock(tsdn, arena_stats);
}
static inline void
arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) {
arena_stats_lock(tsdn, arena_stats);
arena_stats_add_zu(tsdn, arena_stats, &arena_stats->mapped, size);
arena_stats_unlock(tsdn, arena_stats);
}
#endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */
deps/jemalloc/include/jemalloc/internal/arena_structs_a.h
Vendored
+11
View File
@@ -0,0 +1,11 @@
#ifndef JEMALLOC_INTERNAL_ARENA_STRUCTS_A_H
#define JEMALLOC_INTERNAL_ARENA_STRUCTS_A_H
#include "jemalloc/internal/bitmap.h"
struct arena_slab_data_s {
/* Per region allocated/deallocated bitmap. */
bitmap_t bitmap[BITMAP_GROUPS_MAX];
};
#endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_A_H */
deps/jemalloc/include/jemalloc/internal/arena_structs_b.h
Vendored
+229
View File
@@ -0,0 +1,229 @@
#ifndef JEMALLOC_INTERNAL_ARENA_STRUCTS_B_H
#define JEMALLOC_INTERNAL_ARENA_STRUCTS_B_H
#include "jemalloc/internal/arena_stats.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/smoothstep.h"
#include "jemalloc/internal/ticker.h"
struct arena_decay_s {
/* Synchronizes all non-atomic fields. */
malloc_mutex_t mtx;
/*
* True if a thread is currently purging the extents associated with
* this decay structure.
*/
bool purging;
/*
* Approximate time in milliseconds from the creation of a set of unused
* dirty pages until an equivalent set of unused dirty pages is purged
* and/or reused.
*/
atomic_zd_t time_ms;
/* time / SMOOTHSTEP_NSTEPS. */
nstime_t interval;
/*
* Time at which the current decay interval logically started. We do
* not actually advance to a new epoch until sometime after it starts
* because of scheduling and computation delays, and it is even possible
* to completely skip epochs. In all cases, during epoch advancement we
* merge all relevant activity into the most recently recorded epoch.
*/
nstime_t epoch;
/* Deadline randomness generator. */
uint64_t jitter_state;
/*
* Deadline for current epoch. This is the sum of interval and per
* epoch jitter which is a uniform random variable in [0..interval).
* Epochs always advance by precise multiples of interval, but we
* randomize the deadline to reduce the likelihood of arenas purging in
* lockstep.
*/
nstime_t deadline;
/*
* Number of unpurged pages at beginning of current epoch. During epoch
* advancement we use the delta between arena->decay_*.nunpurged and
* extents_npages_get(&arena->extents_*) to determine how many dirty
* pages, if any, were generated.
*/
size_t nunpurged;
/*
* Trailing log of how many unused dirty pages were generated during
* each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last
* element is the most recent epoch. Corresponding epoch times are
* relative to epoch.
*/
size_t backlog[SMOOTHSTEP_NSTEPS];
/*
* Pointer to associated stats. These stats are embedded directly in
* the arena's stats due to how stats structures are shared between the
* arena and ctl code.
*
* Synchronization: Same as associated arena's stats field. */
arena_stats_decay_t *stats;
/* Peak number of pages in associated extents. Used for debug only. */
uint64_t ceil_npages;
};
struct arena_s {
/*
* Number of threads currently assigned to this arena. Each thread has
* two distinct assignments, one for application-serving allocation, and
* the other for internal metadata allocation. Internal metadata must
* not be allocated from arenas explicitly created via the arenas.create
* mallctl, because the arena.<i>.reset mallctl indiscriminately
* discards all allocations for the affected arena.
*
* 0: Application allocation.
* 1: Internal metadata allocation.
*
* Synchronization: atomic.
*/
atomic_u_t nthreads[2];
/*
* When percpu_arena is enabled, to amortize the cost of reading /
* updating the current CPU id, track the most recent thread accessing
* this arena, and only read CPU if there is a mismatch.
*/
tsdn_t *last_thd;
/* Synchronization: internal. */
arena_stats_t stats;
/*
* Lists of tcaches and cache_bin_array_descriptors for extant threads
* associated with this arena. Stats from these are merged
* incrementally, and at exit if opt_stats_print is enabled.
*
* Synchronization: tcache_ql_mtx.
*/
ql_head(tcache_t) tcache_ql;
ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
malloc_mutex_t tcache_ql_mtx;
/* Synchronization: internal. */
prof_accum_t prof_accum;
uint64_t prof_accumbytes;
/*
* PRNG state for cache index randomization of large allocation base
* pointers.
*
* Synchronization: atomic.
*/
atomic_zu_t offset_state;
/*
* Extent serial number generator state.
*
* Synchronization: atomic.
*/
atomic_zu_t extent_sn_next;
/*
* Represents a dss_prec_t, but atomically.
*
* Synchronization: atomic.
*/
atomic_u_t dss_prec;
/*
* Number of pages in active extents.
*
* Synchronization: atomic.
*/
atomic_zu_t nactive;
/*
* Extant large allocations.
*
* Synchronization: large_mtx.
*/
extent_list_t large;
/* Synchronizes all large allocation/update/deallocation. */
malloc_mutex_t large_mtx;
/*
* Collections of extents that were previously allocated. These are
* used when allocating extents, in an attempt to re-use address space.
*
* Synchronization: internal.
*/
extents_t extents_dirty;
extents_t extents_muzzy;
extents_t extents_retained;
/*
* Decay-based purging state, responsible for scheduling extent state
* transitions.
*
* Synchronization: internal.
*/
arena_decay_t decay_dirty; /* dirty --> muzzy */
arena_decay_t decay_muzzy; /* muzzy --> retained */
/*
* Next extent size class in a growing series to use when satisfying a
* request via the extent hooks (only if opt_retain). This limits the
* number of disjoint virtual memory ranges so that extent merging can
* be effective even if multiple arenas' extent allocation requests are
* highly interleaved.
*
* retain_grow_limit is the max allowed size ind to expand (unless the
* required size is greater). Default is no limit, and controlled
* through mallctl only.
*
* Synchronization: extent_grow_mtx
*/
pszind_t extent_grow_next;
pszind_t retain_grow_limit;
malloc_mutex_t extent_grow_mtx;
/*
* Available extent structures that were allocated via
* base_alloc_extent().
*
* Synchronization: extent_avail_mtx.
*/
extent_tree_t extent_avail;
malloc_mutex_t extent_avail_mtx;
/*
* bins is used to store heaps of free regions.
*
* Synchronization: internal.
*/
bin_t bins[NBINS];
/*
* Base allocator, from which arena metadata are allocated.
*
* Synchronization: internal.
*/
base_t *base;
/* Used to determine uptime. Read-only after initialization. */
nstime_t create_time;
};
/* Used in conjunction with tsd for fast arena-related context lookup. */
struct arena_tdata_s {
ticker_t decay_ticker;
};
/* Used to pass rtree lookup context down the path. */
struct alloc_ctx_s {
szind_t szind;
bool slab;
};
#endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_B_H */
deps/jemalloc/include/jemalloc/internal/arena_types.h
Vendored
+43
View File
@@ -0,0 +1,43 @@
#ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H
#define JEMALLOC_INTERNAL_ARENA_TYPES_H
/* Maximum number of regions in one slab. */
#define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN)
#define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS)
/* Default decay times in milliseconds. */
#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000)
#define MUZZY_DECAY_MS_DEFAULT ZD(10 * 1000)
/* Number of event ticks between time checks. */
#define DECAY_NTICKS_PER_UPDATE 1000
typedef struct arena_slab_data_s arena_slab_data_t;
typedef struct arena_decay_s arena_decay_t;
typedef struct arena_s arena_t;
typedef struct arena_tdata_s arena_tdata_t;
typedef struct alloc_ctx_s alloc_ctx_t;
typedef enum {
percpu_arena_mode_names_base = 0, /* Used for options processing. */
/*
* *_uninit are used only during bootstrapping, and must correspond
* to initialized variant plus percpu_arena_mode_enabled_base.
*/
percpu_arena_uninit = 0,
per_phycpu_arena_uninit = 1,
/* All non-disabled modes must come after percpu_arena_disabled. */
percpu_arena_disabled = 2,
percpu_arena_mode_names_limit = 3, /* Used for options processing. */
percpu_arena_mode_enabled_base = 3,
percpu_arena = 3,
per_phycpu_arena = 4 /* Hyper threads share arena. */
} percpu_arena_mode_t;
#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base)
#define PERCPU_ARENA_DEFAULT percpu_arena_disabled
#endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */
deps/jemalloc/include/jemalloc/internal/assert.h
Vendored
+56
View File
@@ -0,0 +1,56 @@
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/util.h"
/*
* Define a custom assert() in order to reduce the chances of deadlock during
* assertion failure.
*/
#ifndef assert
#define assert(e) do { \
if (unlikely(config_debug && !(e))) { \
malloc_printf( \
"<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", \
__FILE__, __LINE__, #e); \
abort(); \
} \
} while (0)
#endif
#ifndef not_reached
#define not_reached() do { \
if (config_debug) { \
malloc_printf( \
"<jemalloc>: %s:%d: Unreachable code reached\n", \
__FILE__, __LINE__); \
abort(); \
} \
unreachable(); \
} while (0)
#endif
#ifndef not_implemented
#define not_implemented() do { \
if (config_debug) { \
malloc_printf("<jemalloc>: %s:%d: Not implemented\n", \
__FILE__, __LINE__); \
abort(); \
} \
} while (0)
#endif
#ifndef assert_not_implemented
#define assert_not_implemented(e) do { \
if (unlikely(config_debug && !(e))) { \
not_implemented(); \
} \
} while (0)
#endif
/* Use to assert a particular configuration, e.g., cassert(config_debug). */
#ifndef cassert
#define cassert(c) do { \
if (unlikely(!(c))) { \
not_reached(); \
} \
} while (0)
#endif
deps/jemalloc/include/jemalloc/internal/atomic.h
Vendored
+57 -631
View File
@@ -1,651 +1,77 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#ifndef JEMALLOC_INTERNAL_ATOMIC_H
#define JEMALLOC_INTERNAL_ATOMIC_H
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#define ATOMIC_INLINE static inline
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
#define atomic_read_uint64(p) atomic_add_uint64(p, 0)
#define atomic_read_uint32(p) atomic_add_uint32(p, 0)
#define atomic_read_p(p) atomic_add_p(p, NULL)
#define atomic_read_z(p) atomic_add_z(p, 0)
#define atomic_read_u(p) atomic_add_u(p, 0)
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS)
# include "jemalloc/internal/atomic_gcc_atomic.h"
#elif defined(JEMALLOC_GCC_SYNC_ATOMICS)
# include "jemalloc/internal/atomic_gcc_sync.h"
#elif defined(_MSC_VER)
# include "jemalloc/internal/atomic_msvc.h"
#elif defined(JEMALLOC_C11_ATOMICS)
# include "jemalloc/internal/atomic_c11.h"
#else
# error "Don't have atomics implemented on this platform."
#endif
/*
* All arithmetic functions return the arithmetic result of the atomic
* operation. Some atomic operation APIs return the value prior to mutation, in
* which case the following functions must redundantly compute the result so
* that it can be returned. These functions are normally inlined, so the extra
* operations can be optimized away if the return values aren't used by the
* callers.
* This header gives more or less a backport of C11 atomics. The user can write
* JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_sizeof_type); to generate
* counterparts of the C11 atomic functions for type, as so:
* JEMALLOC_GENERATE_ATOMICS(int *, pi, 3);
* and then write things like:
* int *some_ptr;
* atomic_pi_t atomic_ptr_to_int;
* atomic_store_pi(&atomic_ptr_to_int, some_ptr, ATOMIC_RELAXED);
* int *prev_value = atomic_exchange_pi(&ptr_to_int, NULL, ATOMIC_ACQ_REL);
* assert(some_ptr == prev_value);
* and expect things to work in the obvious way.
*
* <t> atomic_read_<t>(<t> *p) { return (*p); }
* <t> atomic_add_<t>(<t> *p, <t> x) { return (*p + x); }
* <t> atomic_sub_<t>(<t> *p, <t> x) { return (*p - x); }
* bool atomic_cas_<t>(<t> *p, <t> c, <t> s)
* {
* if (*p != c)
* return (true);
* *p = s;
* return (false);
* }
* void atomic_write_<t>(<t> *p, <t> x) { *p = x; }
* Also included (with naming differences to avoid conflicts with the standard
* library):
* atomic_fence(atomic_memory_order_t) (mimics C11's atomic_thread_fence).
* ATOMIC_INIT (mimics C11's ATOMIC_VAR_INIT).
*/
#ifndef JEMALLOC_ENABLE_INLINE
uint64_t atomic_add_uint64(uint64_t *p, uint64_t x);
uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x);
bool atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s);
void atomic_write_uint64(uint64_t *p, uint64_t x);
uint32_t atomic_add_uint32(uint32_t *p, uint32_t x);
uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x);
bool atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s);
void atomic_write_uint32(uint32_t *p, uint32_t x);
void *atomic_add_p(void **p, void *x);
void *atomic_sub_p(void **p, void *x);
bool atomic_cas_p(void **p, void *c, void *s);
void atomic_write_p(void **p, const void *x);
size_t atomic_add_z(size_t *p, size_t x);
size_t atomic_sub_z(size_t *p, size_t x);
bool atomic_cas_z(size_t *p, size_t c, size_t s);
void atomic_write_z(size_t *p, size_t x);
unsigned atomic_add_u(unsigned *p, unsigned x);
unsigned atomic_sub_u(unsigned *p, unsigned x);
bool atomic_cas_u(unsigned *p, unsigned c, unsigned s);
void atomic_write_u(unsigned *p, unsigned x);
#endif
/*
* Pure convenience, so that we don't have to type "atomic_memory_order_"
* quite so often.
*/
#define ATOMIC_RELAXED atomic_memory_order_relaxed
#define ATOMIC_ACQUIRE atomic_memory_order_acquire
#define ATOMIC_RELEASE atomic_memory_order_release
#define ATOMIC_ACQ_REL atomic_memory_order_acq_rel
#define ATOMIC_SEQ_CST atomic_memory_order_seq_cst
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
/******************************************************************************/
/* 64-bit operations. */
/*
* Not all platforms have 64-bit atomics. If we do, this #define exposes that
* fact.
*/
#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
# if (defined(__amd64__) || defined(__x86_64__))
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
uint64_t t = x;
asm volatile (
"lock; xaddq %0, %1;"
: "+r" (t), "=m" (*p) /* Outputs. */
: "m" (*p) /* Inputs. */
);
return (t + x);
}
JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x)
{
uint64_t t;
x = (uint64_t)(-(int64_t)x);
t = x;
asm volatile (
"lock; xaddq %0, %1;"
: "+r" (t), "=m" (*p) /* Outputs. */
: "m" (*p) /* Inputs. */
);
return (t + x);
}
JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
{
uint8_t success;
asm volatile (
"lock; cmpxchgq %4, %0;"
"sete %1;"
: "=m" (*p), "=a" (success) /* Outputs. */
: "m" (*p), "a" (c), "r" (s) /* Inputs. */
: "memory" /* Clobbers. */
);
return (!(bool)success);
}
JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x)
{
asm volatile (
"xchgq %1, %0;" /* Lock is implied by xchgq. */
: "=m" (*p), "+r" (x) /* Outputs. */
: "m" (*p) /* Inputs. */
: "memory" /* Clobbers. */
);
}
# elif (defined(JEMALLOC_C11ATOMICS))
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
return (atomic_fetch_add(a, x) + x);
}
JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x)
{
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
return (atomic_fetch_sub(a, x) - x);
}
JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
{
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
return (!atomic_compare_exchange_strong(a, &c, s));
}
JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x)
{
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
atomic_store(a, x);
}
# elif (defined(JEMALLOC_ATOMIC9))
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
/*
* atomic_fetchadd_64() doesn't exist, but we only ever use this
* function on LP64 systems, so atomic_fetchadd_long() will do.
*/
assert(sizeof(uint64_t) == sizeof(unsigned long));
return (atomic_fetchadd_long(p, (unsigned long)x) + x);
}
JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x)
{
assert(sizeof(uint64_t) == sizeof(unsigned long));
return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
}
JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
{
assert(sizeof(uint64_t) == sizeof(unsigned long));
return (!atomic_cmpset_long(p, (unsigned long)c, (unsigned long)s));
}
JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x)
{
assert(sizeof(uint64_t) == sizeof(unsigned long));
atomic_store_rel_long(p, x);
}
# elif (defined(JEMALLOC_OSATOMIC))
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
}
JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x)
{
return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
}
JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
{
return (!OSAtomicCompareAndSwap64(c, s, (int64_t *)p));
}
JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x)
{
uint64_t o;
/*The documented OSAtomic*() API does not expose an atomic exchange. */
do {
o = atomic_read_uint64(p);
} while (atomic_cas_uint64(p, o, x));
}
# elif (defined(_MSC_VER))
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
return (InterlockedExchangeAdd64(p, x) + x);
}
JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x)
{
return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x);
}
JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
{
uint64_t o;
o = InterlockedCompareExchange64(p, s, c);
return (o != c);
}
JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x)
{
InterlockedExchange64(p, x);
}
# elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || \
defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
return (__sync_add_and_fetch(p, x));
}
JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x)
{
return (__sync_sub_and_fetch(p, x));
}
JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
{
return (!__sync_bool_compare_and_swap(p, c, s));
}
JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x)
{
__sync_lock_test_and_set(p, x);
}
# else
# error "Missing implementation for 64-bit atomic operations"
# endif
# define JEMALLOC_ATOMIC_U64
#endif
/******************************************************************************/
/* 32-bit operations. */
#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x)
{
uint32_t t = x;
JEMALLOC_GENERATE_ATOMICS(void *, p, LG_SIZEOF_PTR)
asm volatile (
"lock; xaddl %0, %1;"
: "+r" (t), "=m" (*p) /* Outputs. */
: "m" (*p) /* Inputs. */
);
/*
* There's no actual guarantee that sizeof(bool) == 1, but it's true on the only
* platform that actually needs to know the size, MSVC.
*/
JEMALLOC_GENERATE_ATOMICS(bool, b, 0)
return (t + x);
}
JEMALLOC_GENERATE_INT_ATOMICS(unsigned, u, LG_SIZEOF_INT)
JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x)
{
uint32_t t;
JEMALLOC_GENERATE_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR)
x = (uint32_t)(-(int32_t)x);
t = x;
asm volatile (
"lock; xaddl %0, %1;"
: "+r" (t), "=m" (*p) /* Outputs. */
: "m" (*p) /* Inputs. */
);
JEMALLOC_GENERATE_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR)
return (t + x);
}
JEMALLOC_GENERATE_INT_ATOMICS(uint32_t, u32, 2)
JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
{
uint8_t success;
asm volatile (
"lock; cmpxchgl %4, %0;"
"sete %1;"
: "=m" (*p), "=a" (success) /* Outputs. */
: "m" (*p), "a" (c), "r" (s) /* Inputs. */
: "memory"
);
return (!(bool)success);
}
JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x)
{
asm volatile (
"xchgl %1, %0;" /* Lock is implied by xchgl. */
: "=m" (*p), "+r" (x) /* Outputs. */
: "m" (*p) /* Inputs. */
: "memory" /* Clobbers. */
);
}
# elif (defined(JEMALLOC_C11ATOMICS))
JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x)
{
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
return (atomic_fetch_add(a, x) + x);
}
JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x)
{
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
return (atomic_fetch_sub(a, x) - x);
}
JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
{
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
return (!atomic_compare_exchange_strong(a, &c, s));
}
JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x)
{
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
atomic_store(a, x);
}
#elif (defined(JEMALLOC_ATOMIC9))
JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x)
{
return (atomic_fetchadd_32(p, x) + x);
}
JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x)
{
return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
}
JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
{
return (!atomic_cmpset_32(p, c, s));
}
JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x)
{
atomic_store_rel_32(p, x);
}
#elif (defined(JEMALLOC_OSATOMIC))
JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x)
{
return (OSAtomicAdd32((int32_t)x, (int32_t *)p));
}
JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x)
{
return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
}
JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
{
return (!OSAtomicCompareAndSwap32(c, s, (int32_t *)p));
}
JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x)
{
uint32_t o;
/*The documented OSAtomic*() API does not expose an atomic exchange. */
do {
o = atomic_read_uint32(p);
} while (atomic_cas_uint32(p, o, x));
}
#elif (defined(_MSC_VER))
JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x)
{
return (InterlockedExchangeAdd(p, x) + x);
}
JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x)
{
return (InterlockedExchangeAdd(p, -((int32_t)x)) - x);
}
JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
{
uint32_t o;
o = InterlockedCompareExchange(p, s, c);
return (o != c);
}
JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x)
{
InterlockedExchange(p, x);
}
#elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || \
defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x)
{
return (__sync_add_and_fetch(p, x));
}
JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x)
{
return (__sync_sub_and_fetch(p, x));
}
JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
{
return (!__sync_bool_compare_and_swap(p, c, s));
}
JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x)
{
__sync_lock_test_and_set(p, x);
}
#else
# error "Missing implementation for 32-bit atomic operations"
#ifdef JEMALLOC_ATOMIC_U64
JEMALLOC_GENERATE_INT_ATOMICS(uint64_t, u64, 3)
#endif
/******************************************************************************/
/* Pointer operations. */
JEMALLOC_INLINE void *
atomic_add_p(void **p, void *x)
{
#undef ATOMIC_INLINE
#if (LG_SIZEOF_PTR == 3)
return ((void *)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
#elif (LG_SIZEOF_PTR == 2)
return ((void *)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
#endif
}
JEMALLOC_INLINE void *
atomic_sub_p(void **p, void *x)
{
#if (LG_SIZEOF_PTR == 3)
return ((void *)atomic_add_uint64((uint64_t *)p,
(uint64_t)-((int64_t)x)));
#elif (LG_SIZEOF_PTR == 2)
return ((void *)atomic_add_uint32((uint32_t *)p,
(uint32_t)-((int32_t)x)));
#endif
}
JEMALLOC_INLINE bool
atomic_cas_p(void **p, void *c, void *s)
{
#if (LG_SIZEOF_PTR == 3)
return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
#elif (LG_SIZEOF_PTR == 2)
return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
#endif
}
JEMALLOC_INLINE void
atomic_write_p(void **p, const void *x)
{
#if (LG_SIZEOF_PTR == 3)
atomic_write_uint64((uint64_t *)p, (uint64_t)x);
#elif (LG_SIZEOF_PTR == 2)
atomic_write_uint32((uint32_t *)p, (uint32_t)x);
#endif
}
/******************************************************************************/
/* size_t operations. */
JEMALLOC_INLINE size_t
atomic_add_z(size_t *p, size_t x)
{
#if (LG_SIZEOF_PTR == 3)
return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
#elif (LG_SIZEOF_PTR == 2)
return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
#endif
}
JEMALLOC_INLINE size_t
atomic_sub_z(size_t *p, size_t x)
{
#if (LG_SIZEOF_PTR == 3)
return ((size_t)atomic_add_uint64((uint64_t *)p,
(uint64_t)-((int64_t)x)));
#elif (LG_SIZEOF_PTR == 2)
return ((size_t)atomic_add_uint32((uint32_t *)p,
(uint32_t)-((int32_t)x)));
#endif
}
JEMALLOC_INLINE bool
atomic_cas_z(size_t *p, size_t c, size_t s)
{
#if (LG_SIZEOF_PTR == 3)
return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
#elif (LG_SIZEOF_PTR == 2)
return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
#endif
}
JEMALLOC_INLINE void
atomic_write_z(size_t *p, size_t x)
{
#if (LG_SIZEOF_PTR == 3)
atomic_write_uint64((uint64_t *)p, (uint64_t)x);
#elif (LG_SIZEOF_PTR == 2)
atomic_write_uint32((uint32_t *)p, (uint32_t)x);
#endif
}
/******************************************************************************/
/* unsigned operations. */
JEMALLOC_INLINE unsigned
atomic_add_u(unsigned *p, unsigned x)
{
#if (LG_SIZEOF_INT == 3)
return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x));
#elif (LG_SIZEOF_INT == 2)
return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x));
#endif
}
JEMALLOC_INLINE unsigned
atomic_sub_u(unsigned *p, unsigned x)
{
#if (LG_SIZEOF_INT == 3)
return ((unsigned)atomic_add_uint64((uint64_t *)p,
(uint64_t)-((int64_t)x)));
#elif (LG_SIZEOF_INT == 2)
return ((unsigned)atomic_add_uint32((uint32_t *)p,
(uint32_t)-((int32_t)x)));
#endif
}
JEMALLOC_INLINE bool
atomic_cas_u(unsigned *p, unsigned c, unsigned s)
{
#if (LG_SIZEOF_INT == 3)
return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
#elif (LG_SIZEOF_INT == 2)
return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
#endif
}
JEMALLOC_INLINE void
atomic_write_u(unsigned *p, unsigned x)
{
#if (LG_SIZEOF_INT == 3)
atomic_write_uint64((uint64_t *)p, (uint64_t)x);
#elif (LG_SIZEOF_INT == 2)
atomic_write_uint32((uint32_t *)p, (uint32_t)x);
#endif
}
/******************************************************************************/
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#endif /* JEMALLOC_INTERNAL_ATOMIC_H */
deps/jemalloc/include/jemalloc/internal/atomic_c11.h
Vendored
+97
View File
@@ -0,0 +1,97 @@
#ifndef JEMALLOC_INTERNAL_ATOMIC_C11_H
#define JEMALLOC_INTERNAL_ATOMIC_C11_H
#include <stdatomic.h>
#define ATOMIC_INIT(...) ATOMIC_VAR_INIT(__VA_ARGS__)
#define atomic_memory_order_t memory_order
#define atomic_memory_order_relaxed memory_order_relaxed
#define atomic_memory_order_acquire memory_order_acquire
#define atomic_memory_order_release memory_order_release
#define atomic_memory_order_acq_rel memory_order_acq_rel
#define atomic_memory_order_seq_cst memory_order_seq_cst
#define atomic_fence atomic_thread_fence
#define JEMALLOC_GENERATE_ATOMICS(type, short_type, \
/* unused */ lg_size) \
typedef _Atomic(type) atomic_##short_type##_t; \
\
ATOMIC_INLINE type \
atomic_load_##short_type(const atomic_##short_type##_t *a, \
atomic_memory_order_t mo) { \
/* \
* A strict interpretation of the C standard prevents \
* atomic_load from taking a const argument, but it's \
* convenient for our purposes. This cast is a workaround. \
*/ \
atomic_##short_type##_t* a_nonconst = \
(atomic_##short_type##_t*)a; \
return atomic_load_explicit(a_nonconst, mo); \
} \
\
ATOMIC_INLINE void \
atomic_store_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
atomic_store_explicit(a, val, mo); \
} \
\
ATOMIC_INLINE type \
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return atomic_exchange_explicit(a, val, mo); \
} \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
return atomic_compare_exchange_weak_explicit(a, expected, \
desired, success_mo, failure_mo); \
} \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
return atomic_compare_exchange_strong_explicit(a, expected, \
desired, success_mo, failure_mo); \
}
/*
* Integral types have some special operations available that non-integral ones
* lack.
*/
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \
/* unused */ lg_size) \
JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
\
ATOMIC_INLINE type \
atomic_fetch_add_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_add_explicit(a, val, mo); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_sub_explicit(a, val, mo); \
} \
ATOMIC_INLINE type \
atomic_fetch_and_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_and_explicit(a, val, mo); \
} \
ATOMIC_INLINE type \
atomic_fetch_or_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_or_explicit(a, val, mo); \
} \
ATOMIC_INLINE type \
atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_xor_explicit(a, val, mo); \
}
#endif /* JEMALLOC_INTERNAL_ATOMIC_C11_H */
deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h
Vendored
+127
View File
@@ -0,0 +1,127 @@
#ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H
#define JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H
#include "jemalloc/internal/assert.h"
#define ATOMIC_INIT(...) {__VA_ARGS__}
typedef enum {
atomic_memory_order_relaxed,
atomic_memory_order_acquire,
atomic_memory_order_release,
atomic_memory_order_acq_rel,
atomic_memory_order_seq_cst
} atomic_memory_order_t;
ATOMIC_INLINE int
atomic_enum_to_builtin(atomic_memory_order_t mo) {
switch (mo) {
case atomic_memory_order_relaxed:
return __ATOMIC_RELAXED;
case atomic_memory_order_acquire:
return __ATOMIC_ACQUIRE;
case atomic_memory_order_release:
return __ATOMIC_RELEASE;
case atomic_memory_order_acq_rel:
return __ATOMIC_ACQ_REL;
case atomic_memory_order_seq_cst:
return __ATOMIC_SEQ_CST;
}
/* Can't happen; the switch is exhaustive. */
not_reached();
}
ATOMIC_INLINE void
atomic_fence(atomic_memory_order_t mo) {
__atomic_thread_fence(atomic_enum_to_builtin(mo));
}
#define JEMALLOC_GENERATE_ATOMICS(type, short_type, \
/* unused */ lg_size) \
typedef struct { \
type repr; \
} atomic_##short_type##_t; \
\
ATOMIC_INLINE type \
atomic_load_##short_type(const atomic_##short_type##_t *a, \
atomic_memory_order_t mo) { \
type result; \
__atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo)); \
return result; \
} \
\
ATOMIC_INLINE void \
atomic_store_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
__atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo)); \
} \
\
ATOMIC_INLINE type \
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
type result; \
__atomic_exchange(&a->repr, &val, &result, \
atomic_enum_to_builtin(mo)); \
return result; \
} \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
return __atomic_compare_exchange(&a->repr, expected, &desired, \
true, atomic_enum_to_builtin(success_mo), \
atomic_enum_to_builtin(failure_mo)); \
} \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
return __atomic_compare_exchange(&a->repr, expected, &desired, \
false, \
atomic_enum_to_builtin(success_mo), \
atomic_enum_to_builtin(failure_mo)); \
}
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \
/* unused */ lg_size) \
JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
\
ATOMIC_INLINE type \
atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __atomic_fetch_add(&a->repr, val, \
atomic_enum_to_builtin(mo)); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __atomic_fetch_sub(&a->repr, val, \
atomic_enum_to_builtin(mo)); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __atomic_fetch_and(&a->repr, val, \
atomic_enum_to_builtin(mo)); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __atomic_fetch_or(&a->repr, val, \
atomic_enum_to_builtin(mo)); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __atomic_fetch_xor(&a->repr, val, \
atomic_enum_to_builtin(mo)); \
}
#endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H */
deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h
Vendored
+191
View File
@@ -0,0 +1,191 @@
#ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H
#define JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H
#define ATOMIC_INIT(...) {__VA_ARGS__}
typedef enum {
atomic_memory_order_relaxed,
atomic_memory_order_acquire,
atomic_memory_order_release,
atomic_memory_order_acq_rel,
atomic_memory_order_seq_cst
} atomic_memory_order_t;
ATOMIC_INLINE void
atomic_fence(atomic_memory_order_t mo) {
/* Easy cases first: no barrier, and full barrier. */
if (mo == atomic_memory_order_relaxed) {
asm volatile("" ::: "memory");
return;
}
if (mo == atomic_memory_order_seq_cst) {
asm volatile("" ::: "memory");
__sync_synchronize();
asm volatile("" ::: "memory");
return;
}
asm volatile("" ::: "memory");
# if defined(__i386__) || defined(__x86_64__)
/* This is implicit on x86. */
# elif defined(__ppc__)
asm volatile("lwsync");
# elif defined(__sparc__) && defined(__arch64__)
if (mo == atomic_memory_order_acquire) {
asm volatile("membar #LoadLoad | #LoadStore");
} else if (mo == atomic_memory_order_release) {
asm volatile("membar #LoadStore | #StoreStore");
} else {
asm volatile("membar #LoadLoad | #LoadStore | #StoreStore");
}
# else
__sync_synchronize();
# endif
asm volatile("" ::: "memory");
}
/*
* A correct implementation of seq_cst loads and stores on weakly ordered
* architectures could do either of the following:
* 1. store() is weak-fence -> store -> strong fence, load() is load ->
* strong-fence.
* 2. store() is strong-fence -> store, load() is strong-fence -> load ->
* weak-fence.
* The tricky thing is, load() and store() above can be the load or store
* portions of a gcc __sync builtin, so we have to follow GCC's lead, which
* means going with strategy 2.
* On strongly ordered architectures, the natural strategy is to stick a strong
* fence after seq_cst stores, and have naked loads. So we want the strong
* fences in different places on different architectures.
* atomic_pre_sc_load_fence and atomic_post_sc_store_fence allow us to
* accomplish this.
*/
ATOMIC_INLINE void
atomic_pre_sc_load_fence() {
# if defined(__i386__) || defined(__x86_64__) || \
(defined(__sparc__) && defined(__arch64__))
atomic_fence(atomic_memory_order_relaxed);
# else
atomic_fence(atomic_memory_order_seq_cst);
# endif
}
ATOMIC_INLINE void
atomic_post_sc_store_fence() {
# if defined(__i386__) || defined(__x86_64__) || \
(defined(__sparc__) && defined(__arch64__))
atomic_fence(atomic_memory_order_seq_cst);
# else
atomic_fence(atomic_memory_order_relaxed);
# endif
}
#define JEMALLOC_GENERATE_ATOMICS(type, short_type, \
/* unused */ lg_size) \
typedef struct { \
type volatile repr; \
} atomic_##short_type##_t; \
\
ATOMIC_INLINE type \
atomic_load_##short_type(const atomic_##short_type##_t *a, \
atomic_memory_order_t mo) { \
if (mo == atomic_memory_order_seq_cst) { \
atomic_pre_sc_load_fence(); \
} \
type result = a->repr; \
if (mo != atomic_memory_order_relaxed) { \
atomic_fence(atomic_memory_order_acquire); \
} \
return result; \
} \
\
ATOMIC_INLINE void \
atomic_store_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
if (mo != atomic_memory_order_relaxed) { \
atomic_fence(atomic_memory_order_release); \
} \
a->repr = val; \
if (mo == atomic_memory_order_seq_cst) { \
atomic_post_sc_store_fence(); \
} \
} \
\
ATOMIC_INLINE type \
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
/* \
* Because of FreeBSD, we care about gcc 4.2, which doesn't have\
* an atomic exchange builtin. We fake it with a CAS loop. \
*/ \
while (true) { \
type old = a->repr; \
if (__sync_bool_compare_and_swap(&a->repr, old, val)) { \
return old; \
} \
} \
} \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
type prev = __sync_val_compare_and_swap(&a->repr, *expected, \
desired); \
if (prev == *expected) { \
return true; \
} else { \
*expected = prev; \
return false; \
} \
} \
ATOMIC_INLINE bool \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
type prev = __sync_val_compare_and_swap(&a->repr, *expected, \
desired); \
if (prev == *expected) { \
return true; \
} else { \
*expected = prev; \
return false; \
} \
}
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \
/* unused */ lg_size) \
JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
\
ATOMIC_INLINE type \
atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __sync_fetch_and_add(&a->repr, val); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __sync_fetch_and_sub(&a->repr, val); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __sync_fetch_and_and(&a->repr, val); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __sync_fetch_and_or(&a->repr, val); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return __sync_fetch_and_xor(&a->repr, val); \
}
#endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_SYNC_H */
deps/jemalloc/include/jemalloc/internal/atomic_msvc.h
Vendored
+158
View File
@@ -0,0 +1,158 @@
#ifndef JEMALLOC_INTERNAL_ATOMIC_MSVC_H
#define JEMALLOC_INTERNAL_ATOMIC_MSVC_H
#define ATOMIC_INIT(...) {__VA_ARGS__}
typedef enum {
atomic_memory_order_relaxed,
atomic_memory_order_acquire,
atomic_memory_order_release,
atomic_memory_order_acq_rel,
atomic_memory_order_seq_cst
} atomic_memory_order_t;
typedef char atomic_repr_0_t;
typedef short atomic_repr_1_t;
typedef long atomic_repr_2_t;
typedef __int64 atomic_repr_3_t;
ATOMIC_INLINE void
atomic_fence(atomic_memory_order_t mo) {
_ReadWriteBarrier();
# if defined(_M_ARM) || defined(_M_ARM64)
/* ARM needs a barrier for everything but relaxed. */
if (mo != atomic_memory_order_relaxed) {
MemoryBarrier();
}
# elif defined(_M_IX86) || defined (_M_X64)
/* x86 needs a barrier only for seq_cst. */
if (mo == atomic_memory_order_seq_cst) {
MemoryBarrier();
}
# else
# error "Don't know how to create atomics for this platform for MSVC."
# endif
_ReadWriteBarrier();
}
#define ATOMIC_INTERLOCKED_REPR(lg_size) atomic_repr_ ## lg_size ## _t
#define ATOMIC_CONCAT(a, b) ATOMIC_RAW_CONCAT(a, b)
#define ATOMIC_RAW_CONCAT(a, b) a ## b
#define ATOMIC_INTERLOCKED_NAME(base_name, lg_size) ATOMIC_CONCAT( \
base_name, ATOMIC_INTERLOCKED_SUFFIX(lg_size))
#define ATOMIC_INTERLOCKED_SUFFIX(lg_size) \
ATOMIC_CONCAT(ATOMIC_INTERLOCKED_SUFFIX_, lg_size)
#define ATOMIC_INTERLOCKED_SUFFIX_0 8
#define ATOMIC_INTERLOCKED_SUFFIX_1 16
#define ATOMIC_INTERLOCKED_SUFFIX_2
#define ATOMIC_INTERLOCKED_SUFFIX_3 64
#define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \
typedef struct { \
ATOMIC_INTERLOCKED_REPR(lg_size) repr; \
} atomic_##short_type##_t; \
\
ATOMIC_INLINE type \
atomic_load_##short_type(const atomic_##short_type##_t *a, \
atomic_memory_order_t mo) { \
ATOMIC_INTERLOCKED_REPR(lg_size) ret = a->repr; \
if (mo != atomic_memory_order_relaxed) { \
atomic_fence(atomic_memory_order_acquire); \
} \
return (type) ret; \
} \
\
ATOMIC_INLINE void \
atomic_store_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
if (mo != atomic_memory_order_relaxed) { \
atomic_fence(atomic_memory_order_release); \
} \
a->repr = (ATOMIC_INTERLOCKED_REPR(lg_size)) val; \
if (mo == atomic_memory_order_seq_cst) { \
atomic_fence(atomic_memory_order_seq_cst); \
} \
} \
\
ATOMIC_INLINE type \
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchange, \
lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
} \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
ATOMIC_INTERLOCKED_REPR(lg_size) e = \
(ATOMIC_INTERLOCKED_REPR(lg_size))*expected; \
ATOMIC_INTERLOCKED_REPR(lg_size) d = \
(ATOMIC_INTERLOCKED_REPR(lg_size))desired; \
ATOMIC_INTERLOCKED_REPR(lg_size) old = \
ATOMIC_INTERLOCKED_NAME(_InterlockedCompareExchange, \
lg_size)(&a->repr, d, e); \
if (old == e) { \
return true; \
} else { \
*expected = (type)old; \
return false; \
} \
} \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
/* We implement the weak version with strong semantics. */ \
return atomic_compare_exchange_weak_##short_type(a, expected, \
desired, success_mo, failure_mo); \
}
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \
JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \
\
ATOMIC_INLINE type \
atomic_fetch_add_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchangeAdd, \
lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
} \
\
ATOMIC_INLINE type \
atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
/* \
* MSVC warns on negation of unsigned operands, but for us it \
* gives exactly the right semantics (MAX_TYPE + 1 - operand). \
*/ \
__pragma(warning(push)) \
__pragma(warning(disable: 4146)) \
return atomic_fetch_add_##short_type(a, -val, mo); \
__pragma(warning(pop)) \
} \
ATOMIC_INLINE type \
atomic_fetch_and_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedAnd, lg_size)( \
&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
} \
ATOMIC_INLINE type \
atomic_fetch_or_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedOr, lg_size)( \
&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
} \
ATOMIC_INLINE type \
atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, \
type val, atomic_memory_order_t mo) { \
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedXor, lg_size)( \
&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
}
#endif /* JEMALLOC_INTERNAL_ATOMIC_MSVC_H */
deps/jemalloc/include/jemalloc/internal/background_thread_externs.h
Vendored
+33
View File
@@ -0,0 +1,33 @@
#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
extern bool opt_background_thread;
extern size_t opt_max_background_threads;
extern malloc_mutex_t background_thread_lock;
extern atomic_b_t background_thread_enabled_state;
extern size_t n_background_threads;
extern size_t max_background_threads;
extern background_thread_info_t *background_thread_info;
extern bool can_enable_background_thread;
bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
bool background_threads_enable(tsd_t *tsd);
bool background_threads_disable(tsd_t *tsd);
void background_thread_interval_check(tsdn_t *tsdn, arena_t *arena,
arena_decay_t *decay, size_t npages_new);
void background_thread_prefork0(tsdn_t *tsdn);
void background_thread_prefork1(tsdn_t *tsdn);
void background_thread_postfork_parent(tsdn_t *tsdn);
void background_thread_postfork_child(tsdn_t *tsdn);
bool background_thread_stats_read(tsdn_t *tsdn,
background_thread_stats_t *stats);
void background_thread_ctl_init(tsdn_t *tsdn);
#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *,
void *(*)(void *), void *__restrict);
#endif
bool background_thread_boot0(void);
bool background_thread_boot1(tsdn_t *tsdn);
#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H */
deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h
Vendored
+57
View File
@@ -0,0 +1,57 @@
#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H
#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H
JEMALLOC_ALWAYS_INLINE bool
background_thread_enabled(void) {
return atomic_load_b(&background_thread_enabled_state, ATOMIC_RELAXED);
}
JEMALLOC_ALWAYS_INLINE void
background_thread_enabled_set(tsdn_t *tsdn, bool state) {
malloc_mutex_assert_owner(tsdn, &background_thread_lock);
atomic_store_b(&background_thread_enabled_state, state, ATOMIC_RELAXED);
}
JEMALLOC_ALWAYS_INLINE background_thread_info_t *
arena_background_thread_info_get(arena_t *arena) {
unsigned arena_ind = arena_ind_get(arena);
return &background_thread_info[arena_ind % ncpus];
}
JEMALLOC_ALWAYS_INLINE uint64_t
background_thread_wakeup_time_get(background_thread_info_t *info) {
uint64_t next_wakeup = nstime_ns(&info->next_wakeup);
assert(atomic_load_b(&info->indefinite_sleep, ATOMIC_ACQUIRE) ==
(next_wakeup == BACKGROUND_THREAD_INDEFINITE_SLEEP));
return next_wakeup;
}
JEMALLOC_ALWAYS_INLINE void
background_thread_wakeup_time_set(tsdn_t *tsdn, background_thread_info_t *info,
uint64_t wakeup_time) {
malloc_mutex_assert_owner(tsdn, &info->mtx);
atomic_store_b(&info->indefinite_sleep,
wakeup_time == BACKGROUND_THREAD_INDEFINITE_SLEEP, ATOMIC_RELEASE);
nstime_init(&info->next_wakeup, wakeup_time);
}
JEMALLOC_ALWAYS_INLINE bool
background_thread_indefinite_sleep(background_thread_info_t *info) {
return atomic_load_b(&info->indefinite_sleep, ATOMIC_ACQUIRE);
}
JEMALLOC_ALWAYS_INLINE void
arena_background_thread_inactivity_check(tsdn_t *tsdn, arena_t *arena,
bool is_background_thread) {
if (!background_thread_enabled() || is_background_thread) {
return;
}
background_thread_info_t *info =
arena_background_thread_info_get(arena);
if (background_thread_indefinite_sleep(info)) {
background_thread_interval_check(tsdn, arena,
&arena->decay_dirty, 0);
}
}
#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H */
deps/jemalloc/include/jemalloc/internal/background_thread_structs.h
Vendored
+53
View File
@@ -0,0 +1,53 @@
#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
/* This file really combines "structs" and "types", but only transitionally. */
#if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK)
# define JEMALLOC_PTHREAD_CREATE_WRAPPER
#endif
#define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX
#define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT
typedef enum {
background_thread_stopped,
background_thread_started,
/* Thread waits on the global lock when paused (for arena_reset). */
background_thread_paused,
} background_thread_state_t;
struct background_thread_info_s {
#ifdef JEMALLOC_BACKGROUND_THREAD
/* Background thread is pthread specific. */
pthread_t thread;
pthread_cond_t cond;
#endif
malloc_mutex_t mtx;
background_thread_state_t state;
/* When true, it means no wakeup scheduled. */
atomic_b_t indefinite_sleep;
/* Next scheduled wakeup time (absolute time in ns). */
nstime_t next_wakeup;
/*
* Since the last background thread run, newly added number of pages
* that need to be purged by the next wakeup. This is adjusted on
* epoch advance, and is used to determine whether we should signal the
* background thread to wake up earlier.
*/
size_t npages_to_purge_new;
/* Stats: total number of runs since started. */
uint64_t tot_n_runs;
/* Stats: total sleep time since started. */
nstime_t tot_sleep_time;
};
typedef struct background_thread_info_s background_thread_info_t;
struct background_thread_stats_s {
size_t num_threads;
uint64_t num_runs;
nstime_t run_interval;
};
typedef struct background_thread_stats_s background_thread_stats_t;
#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H */
deps/jemalloc/include/jemalloc/internal/base.h
Vendored
-24
View File
@@ -1,24 +0,0 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void *base_alloc(size_t size);
void base_stats_get(size_t *allocated, size_t *resident, size_t *mapped);
bool base_boot(void);
void base_prefork(void);
void base_postfork_parent(void);
void base_postfork_child(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
deps/jemalloc/include/jemalloc/internal/base_externs.h
Vendored
+22
View File
@@ -0,0 +1,22 @@
#ifndef JEMALLOC_INTERNAL_BASE_EXTERNS_H
#define JEMALLOC_INTERNAL_BASE_EXTERNS_H
extern metadata_thp_mode_t opt_metadata_thp;
extern const char *metadata_thp_mode_names[];
base_t *b0get(void);
base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
void base_delete(tsdn_t *tsdn, base_t *base);
extent_hooks_t *base_extent_hooks_get(base_t *base);
extent_hooks_t *base_extent_hooks_set(base_t *base,
extent_hooks_t *extent_hooks);
void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);
extent_t *base_alloc_extent(tsdn_t *tsdn, base_t *base);
void base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated,
size_t *resident, size_t *mapped, size_t *n_thp);
void base_prefork(tsdn_t *tsdn, base_t *base);
void base_postfork_parent(tsdn_t *tsdn, base_t *base);
void base_postfork_child(tsdn_t *tsdn, base_t *base);
bool base_boot(tsdn_t *tsdn);
#endif /* JEMALLOC_INTERNAL_BASE_EXTERNS_H */
deps/jemalloc/include/jemalloc/internal/base_inlines.h
Vendored
+13
View File
@@ -0,0 +1,13 @@
#ifndef JEMALLOC_INTERNAL_BASE_INLINES_H
#define JEMALLOC_INTERNAL_BASE_INLINES_H
static inline unsigned
base_ind_get(const base_t *base) {
return base->ind;
}
static inline bool
metadata_thp_enabled(void) {
return (opt_metadata_thp != metadata_thp_disabled);
}
#endif /* JEMALLOC_INTERNAL_BASE_INLINES_H */
deps/jemalloc/include/jemalloc/internal/base_structs.h
Vendored
+59
View File
@@ -0,0 +1,59 @@
#ifndef JEMALLOC_INTERNAL_BASE_STRUCTS_H
#define JEMALLOC_INTERNAL_BASE_STRUCTS_H
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/size_classes.h"
/* Embedded at the beginning of every block of base-managed virtual memory. */
struct base_block_s {
/* Total size of block's virtual memory mapping. */
size_t size;
/* Next block in list of base's blocks. */
base_block_t *next;
/* Tracks unused trailing space. */
extent_t extent;
};
struct base_s {
/* Associated arena's index within the arenas array. */
unsigned ind;
/*
* User-configurable extent hook functions. Points to an
* extent_hooks_t.
*/
atomic_p_t extent_hooks;
/* Protects base_alloc() and base_stats_get() operations. */
malloc_mutex_t mtx;
/* Using THP when true (metadata_thp auto mode). */
bool auto_thp_switched;
/*
* Most recent size class in the series of increasingly large base
* extents. Logarithmic spacing between subsequent allocations ensures
* that the total number of distinct mappings remains small.
*/
pszind_t pind_last;
/* Serial number generation state. */
size_t extent_sn_next;
/* Chain of all blocks associated with base. */
base_block_t *blocks;
/* Heap of extents that track unused trailing space within blocks. */
extent_heap_t avail[NSIZES];
/* Stats, only maintained if config_stats. */
size_t allocated;
size_t resident;
size_t mapped;
/* Number of THP regions touched. */
size_t n_thp;
};
#endif /* JEMALLOC_INTERNAL_BASE_STRUCTS_H */
deps/jemalloc/include/jemalloc/internal/base_types.h
Vendored
+33
View File
@@ -0,0 +1,33 @@
#ifndef JEMALLOC_INTERNAL_BASE_TYPES_H
#define JEMALLOC_INTERNAL_BASE_TYPES_H
typedef struct base_block_s base_block_t;
typedef struct base_s base_t;
#define METADATA_THP_DEFAULT metadata_thp_disabled
/*
* In auto mode, arenas switch to huge pages for the base allocator on the
* second base block. a0 switches to thp on the 5th block (after 20 megabytes
* of metadata), since more metadata (e.g. rtree nodes) come from a0's base.
*/
#define BASE_AUTO_THP_THRESHOLD 2
#define BASE_AUTO_THP_THRESHOLD_A0 5
typedef enum {
metadata_thp_disabled = 0,
/*
* Lazily enable hugepage for metadata. To avoid high RSS caused by THP
* + low usage arena (i.e. THP becomes a significant percentage), the
* "auto" option only starts using THP after a base allocator used up
* the first THP region. Starting from the second hugepage (in a single
* arena), "auto" behaves the same as "always", i.e. madvise hugepage
* right away.
*/
metadata_thp_auto = 1,
metadata_thp_always = 2,
metadata_thp_mode_limit = 3
} metadata_thp_mode_t;
#endif /* JEMALLOC_INTERNAL_BASE_TYPES_H */
deps/jemalloc/include/jemalloc/internal/bin.h
Vendored
+106
View File
@@ -0,0 +1,106 @@
#ifndef JEMALLOC_INTERNAL_BIN_H
#define JEMALLOC_INTERNAL_BIN_H
#include "jemalloc/internal/extent_types.h"
#include "jemalloc/internal/extent_structs.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/bin_stats.h"
/*
* A bin contains a set of extents that are currently being used for slab
* allocations.
*/
/*
* Read-only information associated with each element of arena_t's bins array
* is stored separately, partly to reduce memory usage (only one copy, rather
* than one per arena), but mainly to avoid false cacheline sharing.
*
* Each slab has the following layout:
*
* /--------------------\
* | region 0 |
* |--------------------|
* | region 1 |
* |--------------------|
* | ... |
* | ... |
* | ... |
* |--------------------|
* | region nregs-1 |
* \--------------------/
*/
typedef struct bin_info_s bin_info_t;
struct bin_info_s {
/* Size of regions in a slab for this bin's size class. */
size_t reg_size;
/* Total size of a slab for this bin's size class. */
size_t slab_size;
/* Total number of regions in a slab for this bin's size class. */
uint32_t nregs;
/*
* Metadata used to manipulate bitmaps for slabs associated with this
* bin.
*/
bitmap_info_t bitmap_info;
};
extern const bin_info_t bin_infos[NBINS];
typedef struct bin_s bin_t;
struct bin_s {
/* All operations on bin_t fields require lock ownership. */
malloc_mutex_t lock;
/*
* Current slab being used to service allocations of this bin's size
* class. slabcur is independent of slabs_{nonfull,full}; whenever
* slabcur is reassigned, the previous slab must be deallocated or
* inserted into slabs_{nonfull,full}.
*/
extent_t *slabcur;
/*
* Heap of non-full slabs. This heap is used to assure that new
* allocations come from the non-full slab that is oldest/lowest in
* memory.
*/
extent_heap_t slabs_nonfull;
/* List used to track full slabs. */
extent_list_t slabs_full;
/* Bin statistics. */
bin_stats_t stats;
};
/* Initializes a bin to empty. Returns true on error. */
bool bin_init(bin_t *bin);
/* Forking. */
void bin_prefork(tsdn_t *tsdn, bin_t *bin);
void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin);
void bin_postfork_child(tsdn_t *tsdn, bin_t *bin);
/* Stats. */
static inline void
bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) {
malloc_mutex_lock(tsdn, &bin->lock);
malloc_mutex_prof_read(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
dst_bin_stats->nmalloc += bin->stats.nmalloc;
dst_bin_stats->ndalloc += bin->stats.ndalloc;
dst_bin_stats->nrequests += bin->stats.nrequests;
dst_bin_stats->curregs += bin->stats.curregs;
dst_bin_stats->nfills += bin->stats.nfills;
dst_bin_stats->nflushes += bin->stats.nflushes;
dst_bin_stats->nslabs += bin->stats.nslabs;
dst_bin_stats->reslabs += bin->stats.reslabs;
dst_bin_stats->curslabs += bin->stats.curslabs;
malloc_mutex_unlock(tsdn, &bin->lock);
}
#endif /* JEMALLOC_INTERNAL_BIN_H */
deps/jemalloc/include/jemalloc/internal/bin_stats.h
Vendored
+51
View File
@@ -0,0 +1,51 @@
#ifndef JEMALLOC_INTERNAL_BIN_STATS_H
#define JEMALLOC_INTERNAL_BIN_STATS_H
#include "jemalloc/internal/mutex_prof.h"
typedef struct bin_stats_s bin_stats_t;
struct bin_stats_s {
/*
* Total number of allocation/deallocation requests served directly by
* the bin. Note that tcache may allocate an object, then recycle it
* many times, resulting many increments to nrequests, but only one
* each to nmalloc and ndalloc.
*/
uint64_t nmalloc;
uint64_t ndalloc;
/*
* Number of allocation requests that correspond to the size of this
* bin. This includes requests served by tcache, though tcache only
* periodically merges into this counter.
*/
uint64_t nrequests;
/*
* Current number of regions of this size class, including regions
* currently cached by tcache.
*/
size_t curregs;
/* Number of tcache fills from this bin. */
uint64_t nfills;
/* Number of tcache flushes to this bin. */
uint64_t nflushes;
/* Total number of slabs created for this bin's size class. */
uint64_t nslabs;
/*
* Total number of slabs reused by extracting them from the slabs heap
* for this bin's size class.
*/
uint64_t reslabs;
/* Current number of slabs in this bin. */
size_t curslabs;
mutex_prof_data_t mutex_data;
};
#endif /* JEMALLOC_INTERNAL_BIN_STATS_H */
deps/jemalloc/include/jemalloc/internal/bit_util.h
Vendored
+165
View File
@@ -0,0 +1,165 @@
#ifndef JEMALLOC_INTERNAL_BIT_UTIL_H
#define JEMALLOC_INTERNAL_BIT_UTIL_H
#include "jemalloc/internal/assert.h"
#define BIT_UTIL_INLINE static inline
/* Sanity check. */
#if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \
|| !defined(JEMALLOC_INTERNAL_FFS)
# error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure
#endif
BIT_UTIL_INLINE unsigned
ffs_llu(unsigned long long bitmap) {
return JEMALLOC_INTERNAL_FFSLL(bitmap);
}
BIT_UTIL_INLINE unsigned
ffs_lu(unsigned long bitmap) {
return JEMALLOC_INTERNAL_FFSL(bitmap);
}
BIT_UTIL_INLINE unsigned
ffs_u(unsigned bitmap) {
return JEMALLOC_INTERNAL_FFS(bitmap);
}
BIT_UTIL_INLINE unsigned
ffs_zu(size_t bitmap) {
#if LG_SIZEOF_PTR == LG_SIZEOF_INT
return ffs_u(bitmap);
#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG
return ffs_lu(bitmap);
#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
return ffs_llu(bitmap);
#else
#error No implementation for size_t ffs()
#endif
}
BIT_UTIL_INLINE unsigned
ffs_u64(uint64_t bitmap) {
#if LG_SIZEOF_LONG == 3
return ffs_lu(bitmap);
#elif LG_SIZEOF_LONG_LONG == 3
return ffs_llu(bitmap);
#else
#error No implementation for 64-bit ffs()
#endif
}
BIT_UTIL_INLINE unsigned
ffs_u32(uint32_t bitmap) {
#if LG_SIZEOF_INT == 2
return ffs_u(bitmap);
#else
#error No implementation for 32-bit ffs()
#endif
return ffs_u(bitmap);
}
BIT_UTIL_INLINE uint64_t
pow2_ceil_u64(uint64_t x) {
x--;
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
x |= x >> 32;
x++;
return x;
}
BIT_UTIL_INLINE uint32_t
pow2_ceil_u32(uint32_t x) {
x--;
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
x++;
return x;
}
/* Compute the smallest power of 2 that is >= x. */
BIT_UTIL_INLINE size_t
pow2_ceil_zu(size_t x) {
#if (LG_SIZEOF_PTR == 3)
return pow2_ceil_u64(x);
#else
return pow2_ceil_u32(x);
#endif
}
#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
BIT_UTIL_INLINE unsigned
lg_floor(size_t x) {
size_t ret;
assert(x != 0);
asm ("bsr %1, %0"
: "=r"(ret) // Outputs.
: "r"(x) // Inputs.
);
assert(ret < UINT_MAX);
return (unsigned)ret;
}
#elif (defined(_MSC_VER))
BIT_UTIL_INLINE unsigned
lg_floor(size_t x) {
unsigned long ret;
assert(x != 0);
#if (LG_SIZEOF_PTR == 3)
_BitScanReverse64(&ret, x);
#elif (LG_SIZEOF_PTR == 2)
_BitScanReverse(&ret, x);
#else
# error "Unsupported type size for lg_floor()"
#endif
assert(ret < UINT_MAX);
return (unsigned)ret;
}
#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
BIT_UTIL_INLINE unsigned
lg_floor(size_t x) {
assert(x != 0);
#if (LG_SIZEOF_PTR == LG_SIZEOF_INT)
return ((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x);
#elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG)
return ((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x);
#else
# error "Unsupported type size for lg_floor()"
#endif
}
#else
BIT_UTIL_INLINE unsigned
lg_floor(size_t x) {
assert(x != 0);
x |= (x >> 1);
x |= (x >> 2);
x |= (x >> 4);
x |= (x >> 8);
x |= (x >> 16);
#if (LG_SIZEOF_PTR == 3)
x |= (x >> 32);
#endif
if (x == SIZE_T_MAX) {
return (8 << LG_SIZEOF_PTR) - 1;
}
x++;
return ffs_zu(x) - 2;
}
#endif
#undef BIT_UTIL_INLINE
#endif /* JEMALLOC_INTERNAL_BIT_UTIL_H */
deps/jemalloc/include/jemalloc/internal/bitmap.h
Vendored
+229 -90
View File
@@ -1,83 +1,159 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#ifndef JEMALLOC_INTERNAL_BITMAP_H
#define JEMALLOC_INTERNAL_BITMAP_H
#include "jemalloc/internal/arena_types.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/size_classes.h"
typedef unsigned long bitmap_t;
#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG
/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
#define LG_BITMAP_MAXBITS LG_RUN_MAXREGS
#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS)
typedef struct bitmap_level_s bitmap_level_t;
typedef struct bitmap_info_s bitmap_info_t;
typedef unsigned long bitmap_t;
#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG
#if LG_SLAB_MAXREGS > LG_CEIL_NSIZES
/* Maximum bitmap bit count is determined by maximum regions per slab. */
# define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS
#else
/* Maximum bitmap bit count is determined by number of extent size classes. */
# define LG_BITMAP_MAXBITS LG_CEIL_NSIZES
#endif
#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS)
/* Number of bits per group. */
#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3)
#define BITMAP_GROUP_NBITS (ZU(1) << LG_BITMAP_GROUP_NBITS)
#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS-1)
#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3)
#define BITMAP_GROUP_NBITS (1U << LG_BITMAP_GROUP_NBITS)
#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS-1)
/*
* Do some analysis on how big the bitmap is before we use a tree. For a brute
* force linear search, if we would have to call ffs_lu() more than 2^3 times,
* use a tree instead.
*/
#if LG_BITMAP_MAXBITS - LG_BITMAP_GROUP_NBITS > 3
# define BITMAP_USE_TREE
#endif
/* Number of groups required to store a given number of bits. */
#define BITMAP_BITS2GROUPS(nbits) \
((nbits + BITMAP_GROUP_NBITS_MASK) >> LG_BITMAP_GROUP_NBITS)
#define BITMAP_BITS2GROUPS(nbits) \
(((nbits) + BITMAP_GROUP_NBITS_MASK) >> LG_BITMAP_GROUP_NBITS)
/*
* Number of groups required at a particular level for a given number of bits.
*/
#define BITMAP_GROUPS_L0(nbits) \
#define BITMAP_GROUPS_L0(nbits) \
BITMAP_BITS2GROUPS(nbits)
#define BITMAP_GROUPS_L1(nbits) \
#define BITMAP_GROUPS_L1(nbits) \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(nbits))
#define BITMAP_GROUPS_L2(nbits) \
#define BITMAP_GROUPS_L2(nbits) \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))
#define BITMAP_GROUPS_L3(nbits) \
#define BITMAP_GROUPS_L3(nbits) \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \
BITMAP_BITS2GROUPS((nbits)))))
#define BITMAP_GROUPS_L4(nbits) \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))))
/*
* Assuming the number of levels, number of groups required for a given number
* of bits.
*/
#define BITMAP_GROUPS_1_LEVEL(nbits) \
#define BITMAP_GROUPS_1_LEVEL(nbits) \
BITMAP_GROUPS_L0(nbits)
#define BITMAP_GROUPS_2_LEVEL(nbits) \
#define BITMAP_GROUPS_2_LEVEL(nbits) \
(BITMAP_GROUPS_1_LEVEL(nbits) + BITMAP_GROUPS_L1(nbits))
#define BITMAP_GROUPS_3_LEVEL(nbits) \
#define BITMAP_GROUPS_3_LEVEL(nbits) \
(BITMAP_GROUPS_2_LEVEL(nbits) + BITMAP_GROUPS_L2(nbits))
#define BITMAP_GROUPS_4_LEVEL(nbits) \
#define BITMAP_GROUPS_4_LEVEL(nbits) \
(BITMAP_GROUPS_3_LEVEL(nbits) + BITMAP_GROUPS_L3(nbits))
#define BITMAP_GROUPS_5_LEVEL(nbits) \
(BITMAP_GROUPS_4_LEVEL(nbits) + BITMAP_GROUPS_L4(nbits))
/*
* Maximum number of groups required to support LG_BITMAP_MAXBITS.
*/
#ifdef BITMAP_USE_TREE
#if LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_1_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_1_LEVEL(BITMAP_MAXBITS)
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 2
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_2_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_2_LEVEL(BITMAP_MAXBITS)
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 3
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_3_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_3_LEVEL(BITMAP_MAXBITS)
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 4
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_4_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_4_LEVEL(BITMAP_MAXBITS)
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 5
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_5_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_5_LEVEL(BITMAP_MAXBITS)
#else
# error "Unsupported bitmap size"
#endif
/* Maximum number of levels possible. */
#define BITMAP_MAX_LEVELS \
(LG_BITMAP_MAXBITS / LG_SIZEOF_BITMAP) \
+ !!(LG_BITMAP_MAXBITS % LG_SIZEOF_BITMAP)
/*
* Maximum number of levels possible. This could be statically computed based
* on LG_BITMAP_MAXBITS:
*
* #define BITMAP_MAX_LEVELS \
* (LG_BITMAP_MAXBITS / LG_SIZEOF_BITMAP) \
* + !!(LG_BITMAP_MAXBITS % LG_SIZEOF_BITMAP)
*
* However, that would not allow the generic BITMAP_INFO_INITIALIZER() macro, so
* instead hardcode BITMAP_MAX_LEVELS to the largest number supported by the
* various cascading macros. The only additional cost this incurs is some
* unused trailing entries in bitmap_info_t structures; the bitmaps themselves
* are not impacted.
*/
#define BITMAP_MAX_LEVELS 5
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#define BITMAP_INFO_INITIALIZER(nbits) { \
/* nbits. */ \
nbits, \
/* nlevels. */ \
(BITMAP_GROUPS_L0(nbits) > BITMAP_GROUPS_L1(nbits)) + \
(BITMAP_GROUPS_L1(nbits) > BITMAP_GROUPS_L2(nbits)) + \
(BITMAP_GROUPS_L2(nbits) > BITMAP_GROUPS_L3(nbits)) + \
(BITMAP_GROUPS_L3(nbits) > BITMAP_GROUPS_L4(nbits)) + 1, \
/* levels. */ \
{ \
{0}, \
{BITMAP_GROUPS_L0(nbits)}, \
{BITMAP_GROUPS_L1(nbits) + BITMAP_GROUPS_L0(nbits)}, \
{BITMAP_GROUPS_L2(nbits) + BITMAP_GROUPS_L1(nbits) + \
BITMAP_GROUPS_L0(nbits)}, \
{BITMAP_GROUPS_L3(nbits) + BITMAP_GROUPS_L2(nbits) + \
BITMAP_GROUPS_L1(nbits) + BITMAP_GROUPS_L0(nbits)}, \
{BITMAP_GROUPS_L4(nbits) + BITMAP_GROUPS_L3(nbits) + \
BITMAP_GROUPS_L2(nbits) + BITMAP_GROUPS_L1(nbits) \
+ BITMAP_GROUPS_L0(nbits)} \
} \
}
struct bitmap_level_s {
#else /* BITMAP_USE_TREE */
#define BITMAP_GROUPS(nbits) BITMAP_BITS2GROUPS(nbits)
#define BITMAP_GROUPS_MAX BITMAP_BITS2GROUPS(BITMAP_MAXBITS)
#define BITMAP_INFO_INITIALIZER(nbits) { \
/* nbits. */ \
nbits, \
/* ngroups. */ \
BITMAP_BITS2GROUPS(nbits) \
}
#endif /* BITMAP_USE_TREE */
typedef struct bitmap_level_s {
/* Offset of this level's groups within the array of groups. */
size_t group_offset;
};
} bitmap_level_t;
struct bitmap_info_s {
typedef struct bitmap_info_s {
/* Logical number of bits in bitmap (stored at bottom level). */
size_t nbits;
#ifdef BITMAP_USE_TREE
/* Number of levels necessary for nbits. */
unsigned nlevels;
@@ -86,54 +162,48 @@ struct bitmap_info_s {
* bottom to top (e.g. the bottom level is stored in levels[0]).
*/
bitmap_level_t levels[BITMAP_MAX_LEVELS+1];
};
#else /* BITMAP_USE_TREE */
/* Number of groups necessary for nbits. */
size_t ngroups;
#endif /* BITMAP_USE_TREE */
} bitmap_info_t;
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
size_t bitmap_size(const bitmap_info_t *binfo);
void bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
size_t bitmap_info_ngroups(const bitmap_info_t *binfo);
size_t bitmap_size(size_t nbits);
void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
bool bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo);
bool bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
void bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
size_t bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo);
void bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_BITMAP_C_))
JEMALLOC_INLINE bool
bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo)
{
unsigned rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
static inline bool
bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
#ifdef BITMAP_USE_TREE
size_t rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
bitmap_t rg = bitmap[rgoff];
/* The bitmap is full iff the root group is 0. */
return (rg == 0);
#else
size_t i;
for (i = 0; i < binfo->ngroups; i++) {
if (bitmap[i] != 0) {
return false;
}
}
return true;
#endif
}
JEMALLOC_INLINE bool
bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
{
static inline bool
bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
size_t goff;
bitmap_t g;
assert(bit < binfo->nbits);
goff = bit >> LG_BITMAP_GROUP_NBITS;
g = bitmap[goff];
return (!(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))));
return !(g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK)));
}
JEMALLOC_INLINE void
bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
{
static inline void
bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
size_t goff;
bitmap_t *gp;
bitmap_t g;
@@ -143,10 +213,11 @@ bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
goff = bit >> LG_BITMAP_GROUP_NBITS;
gp = &bitmap[goff];
g = *gp;
assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)));
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
assert(g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK)));
g ^= ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK);
*gp = g;
assert(bitmap_get(bitmap, binfo, bit));
#ifdef BITMAP_USE_TREE
/* Propagate group state transitions up the tree. */
if (g == 0) {
unsigned i;
@@ -155,45 +226,113 @@ bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
goff = bit >> LG_BITMAP_GROUP_NBITS;
gp = &bitmap[binfo->levels[i].group_offset + goff];
g = *gp;
assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)));
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
assert(g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK)));
g ^= ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK);
*gp = g;
if (g != 0)
if (g != 0) {
break;
}
}
}
#endif
}
/* ffu: find first unset >= bit. */
static inline size_t
bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) {
assert(min_bit < binfo->nbits);
#ifdef BITMAP_USE_TREE
size_t bit = 0;
for (unsigned level = binfo->nlevels; level--;) {
size_t lg_bits_per_group = (LG_BITMAP_GROUP_NBITS * (level +
1));
bitmap_t group = bitmap[binfo->levels[level].group_offset + (bit
>> lg_bits_per_group)];
unsigned group_nmask = (unsigned)(((min_bit > bit) ? (min_bit -
bit) : 0) >> (lg_bits_per_group - LG_BITMAP_GROUP_NBITS));
assert(group_nmask <= BITMAP_GROUP_NBITS);
bitmap_t group_mask = ~((1LU << group_nmask) - 1);
bitmap_t group_masked = group & group_mask;
if (group_masked == 0LU) {
if (group == 0LU) {
return binfo->nbits;
}
/*
* min_bit was preceded by one or more unset bits in
* this group, but there are no other unset bits in this
* group. Try again starting at the first bit of the
* next sibling. This will recurse at most once per
* non-root level.
*/
size_t sib_base = bit + (ZU(1) << lg_bits_per_group);
assert(sib_base > min_bit);
assert(sib_base > bit);
if (sib_base >= binfo->nbits) {
return binfo->nbits;
}
return bitmap_ffu(bitmap, binfo, sib_base);
}
bit += ((size_t)(ffs_lu(group_masked) - 1)) <<
(lg_bits_per_group - LG_BITMAP_GROUP_NBITS);
}
assert(bit >= min_bit);
assert(bit < binfo->nbits);
return bit;
#else
size_t i = min_bit >> LG_BITMAP_GROUP_NBITS;
bitmap_t g = bitmap[i] & ~((1LU << (min_bit & BITMAP_GROUP_NBITS_MASK))
- 1);
size_t bit;
do {
bit = ffs_lu(g);
if (bit != 0) {
return (i << LG_BITMAP_GROUP_NBITS) + (bit - 1);
}
i++;
g = bitmap[i];
} while (i < binfo->ngroups);
return binfo->nbits;
#endif
}
/* sfu: set first unset. */
JEMALLOC_INLINE size_t
bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo)
{
static inline size_t
bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) {
size_t bit;
bitmap_t g;
unsigned i;
assert(!bitmap_full(bitmap, binfo));
#ifdef BITMAP_USE_TREE
i = binfo->nlevels - 1;
g = bitmap[binfo->levels[i].group_offset];
bit = jemalloc_ffsl(g) - 1;
bit = ffs_lu(g) - 1;
while (i > 0) {
i--;
g = bitmap[binfo->levels[i].group_offset + bit];
bit = (bit << LG_BITMAP_GROUP_NBITS) + (jemalloc_ffsl(g) - 1);
bit = (bit << LG_BITMAP_GROUP_NBITS) + (ffs_lu(g) - 1);
}
#else
i = 0;
g = bitmap[0];
while ((bit = ffs_lu(g)) == 0) {
i++;
g = bitmap[i];
}
bit = (i << LG_BITMAP_GROUP_NBITS) + (bit - 1);
#endif
bitmap_set(bitmap, binfo, bit);
return (bit);
return bit;
}
JEMALLOC_INLINE void
bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
{
static inline void
bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
size_t goff;
bitmap_t *gp;
bitmap_t g;
bool propagate;
UNUSED bool propagate;
assert(bit < binfo->nbits);
assert(bitmap_get(bitmap, binfo, bit));
@@ -201,10 +340,11 @@ bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
gp = &bitmap[goff];
g = *gp;
propagate = (g == 0);
assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))) == 0);
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
assert((g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK))) == 0);
g ^= ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK);
*gp = g;
assert(!bitmap_get(bitmap, binfo, bit));
#ifdef BITMAP_USE_TREE
/* Propagate group state transitions up the tree. */
if (propagate) {
unsigned i;
@@ -214,17 +354,16 @@ bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit)
gp = &bitmap[binfo->levels[i].group_offset + goff];
g = *gp;
propagate = (g == 0);
assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)))
assert((g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK)))
== 0);
g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK);
g ^= ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK);
*gp = g;
if (!propagate)
if (!propagate) {
break;
}
}
}
#endif /* BITMAP_USE_TREE */
}
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#endif /* JEMALLOC_INTERNAL_BITMAP_H */
deps/jemalloc/include/jemalloc/internal/cache_bin.h
Vendored
+114
View File
@@ -0,0 +1,114 @@
#ifndef JEMALLOC_INTERNAL_CACHE_BIN_H
#define JEMALLOC_INTERNAL_CACHE_BIN_H
#include "jemalloc/internal/ql.h"
/*
* The cache_bins are the mechanism that the tcache and the arena use to
* communicate. The tcache fills from and flushes to the arena by passing a
* cache_bin_t to fill/flush. When the arena needs to pull stats from the
* tcaches associated with it, it does so by iterating over its
* cache_bin_array_descriptor_t objects and reading out per-bin stats it
* contains. This makes it so that the arena need not know about the existence
* of the tcache at all.
*/
/*
* The count of the number of cached allocations in a bin. We make this signed
* so that negative numbers can encode "invalid" states (e.g. a low water mark
* of -1 for a cache that has been depleted).
*/
typedef int32_t cache_bin_sz_t;
typedef struct cache_bin_stats_s cache_bin_stats_t;
struct cache_bin_stats_s {
/*
* Number of allocation requests that corresponded to the size of this
* bin.
*/
uint64_t nrequests;
};
/*
* Read-only information associated with each element of tcache_t's tbins array
* is stored separately, mainly to reduce memory usage.
*/
typedef struct cache_bin_info_s cache_bin_info_t;
struct cache_bin_info_s {
/* Upper limit on ncached. */
cache_bin_sz_t ncached_max;
};
typedef struct cache_bin_s cache_bin_t;
struct cache_bin_s {
/* Min # cached since last GC. */
cache_bin_sz_t low_water;
/* # of cached objects. */
cache_bin_sz_t ncached;
/*
* ncached and stats are both modified frequently. Let's keep them
* close so that they have a higher chance of being on the same
* cacheline, thus less write-backs.
*/
cache_bin_stats_t tstats;
/*
* Stack of available objects.
*
* To make use of adjacent cacheline prefetch, the items in the avail
* stack goes to higher address for newer allocations. avail points
* just above the available space, which means that
* avail[-ncached, ... -1] are available items and the lowest item will
* be allocated first.
*/
void **avail;
};
typedef struct cache_bin_array_descriptor_s cache_bin_array_descriptor_t;
struct cache_bin_array_descriptor_s {
/*
* The arena keeps a list of the cache bins associated with it, for
* stats collection.
*/
ql_elm(cache_bin_array_descriptor_t) link;
/* Pointers to the tcache bins. */
cache_bin_t *bins_small;
cache_bin_t *bins_large;
};
static inline void
cache_bin_array_descriptor_init(cache_bin_array_descriptor_t *descriptor,
cache_bin_t *bins_small, cache_bin_t *bins_large) {
ql_elm_new(descriptor, link);
descriptor->bins_small = bins_small;
descriptor->bins_large = bins_large;
}
JEMALLOC_ALWAYS_INLINE void *
cache_bin_alloc_easy(cache_bin_t *bin, bool *success) {
void *ret;
if (unlikely(bin->ncached == 0)) {
bin->low_water = -1;
*success = false;
return NULL;
}
/*
* success (instead of ret) should be checked upon the return of this
* function. We avoid checking (ret == NULL) because there is never a
* null stored on the avail stack (which is unknown to the compiler),
* and eagerly checking ret would cause pipeline stall (waiting for the
* cacheline).
*/
*success = true;
ret = *(bin->avail - bin->ncached);
bin->ncached--;
if (unlikely(bin->ncached < bin->low_water)) {
bin->low_water = bin->ncached;
}
return ret;
}
#endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */
deps/jemalloc/include/jemalloc/internal/chunk.h
Vendored
-99
View File
@@ -1,99 +0,0 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
/*
* Size and alignment of memory chunks that are allocated by the OS's virtual
* memory system.
*/
#define LG_CHUNK_DEFAULT 21
/* Return the chunk address for allocation address a. */
#define CHUNK_ADDR2BASE(a) \
((void *)((uintptr_t)(a) & ~chunksize_mask))
/* Return the chunk offset of address a. */
#define CHUNK_ADDR2OFFSET(a) \
((size_t)((uintptr_t)(a) & chunksize_mask))
/* Return the smallest chunk multiple that is >= s. */
#define CHUNK_CEILING(s) \
(((s) + chunksize_mask) & ~chunksize_mask)
#define CHUNK_HOOKS_INITIALIZER { \
NULL, \
NULL, \
NULL, \
NULL, \
NULL, \
NULL, \
NULL \
}
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
extern size_t opt_lg_chunk;
extern const char *opt_dss;
extern rtree_t chunks_rtree;
extern size_t chunksize;
extern size_t chunksize_mask; /* (chunksize - 1). */
extern size_t chunk_npages;
extern const chunk_hooks_t chunk_hooks_default;
chunk_hooks_t chunk_hooks_get(arena_t *arena);
chunk_hooks_t chunk_hooks_set(arena_t *arena,
const chunk_hooks_t *chunk_hooks);
bool chunk_register(const void *chunk, const extent_node_t *node);
void chunk_deregister(const void *chunk, const extent_node_t *node);
void *chunk_alloc_base(size_t size);
void *chunk_alloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks,
void *new_addr, size_t size, size_t alignment, bool *zero,
bool dalloc_node);
void *chunk_alloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks,
void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit);
void chunk_dalloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks,
void *chunk, size_t size, bool committed);
void chunk_dalloc_arena(arena_t *arena, chunk_hooks_t *chunk_hooks,
void *chunk, size_t size, bool zeroed, bool committed);
void chunk_dalloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks,
void *chunk, size_t size, bool committed);
bool chunk_purge_arena(arena_t *arena, void *chunk, size_t offset,
size_t length);
bool chunk_purge_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks,
void *chunk, size_t size, size_t offset, size_t length);
bool chunk_boot(void);
void chunk_prefork(void);
void chunk_postfork_parent(void);
void chunk_postfork_child(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
extent_node_t *chunk_lookup(const void *chunk, bool dependent);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_CHUNK_C_))
JEMALLOC_INLINE extent_node_t *
chunk_lookup(const void *ptr, bool dependent)
{
return (rtree_get(&chunks_rtree, (uintptr_t)ptr, dependent));
}
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#include "jemalloc/internal/chunk_dss.h"
#include "jemalloc/internal/chunk_mmap.h"
deps/jemalloc/include/jemalloc/internal/chunk_dss.h
Vendored
-39
View File
@@ -1,39 +0,0 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
typedef enum {
dss_prec_disabled = 0,
dss_prec_primary = 1,
dss_prec_secondary = 2,
dss_prec_limit = 3
} dss_prec_t;
#define DSS_PREC_DEFAULT dss_prec_secondary
#define DSS_DEFAULT "secondary"
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
extern const char *dss_prec_names[];
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
dss_prec_t chunk_dss_prec_get(void);
bool chunk_dss_prec_set(dss_prec_t dss_prec);
void *chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit);
bool chunk_in_dss(void *chunk);
bool chunk_dss_boot(void);
void chunk_dss_prefork(void);
void chunk_dss_postfork_parent(void);
void chunk_dss_postfork_child(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
deps/jemalloc/include/jemalloc/internal/chunk_mmap.h
Vendored
-21
View File
@@ -1,21 +0,0 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void *chunk_alloc_mmap(size_t size, size_t alignment, bool *zero,
bool *commit);
bool chunk_dalloc_mmap(void *chunk, size_t size);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
deps/jemalloc/include/jemalloc/internal/ckh.h
Vendored
+65 -52
View File
@@ -1,88 +1,101 @@
#ifndef JEMALLOC_INTERNAL_CKH_H
#define JEMALLOC_INTERNAL_CKH_H
#include "jemalloc/internal/tsd.h"
/* Cuckoo hashing implementation. Skip to the end for the interface. */
/******************************************************************************/
/* INTERNAL DEFINITIONS -- IGNORE */
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
typedef struct ckh_s ckh_t;
typedef struct ckhc_s ckhc_t;
/* Typedefs to allow easy function pointer passing. */
typedef void ckh_hash_t (const void *, size_t[2]);
typedef bool ckh_keycomp_t (const void *, const void *);
/* Maintain counters used to get an idea of performance. */
/* #define CKH_COUNT */
/* #define CKH_COUNT */
/* Print counter values in ckh_delete() (requires CKH_COUNT). */
/* #define CKH_VERBOSE */
/* #define CKH_VERBOSE */
/*
* There are 2^LG_CKH_BUCKET_CELLS cells in each hash table bucket. Try to fit
* one bucket per L1 cache line.
*/
#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1)
#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1)
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
/* Typedefs to allow easy function pointer passing. */
typedef void ckh_hash_t (const void *, size_t[2]);
typedef bool ckh_keycomp_t (const void *, const void *);
/* Hash table cell. */
struct ckhc_s {
const void *key;
const void *data;
};
typedef struct {
const void *key;
const void *data;
} ckhc_t;
struct ckh_s {
/* The hash table itself. */
typedef struct {
#ifdef CKH_COUNT
/* Counters used to get an idea of performance. */
uint64_t ngrows;
uint64_t nshrinks;
uint64_t nshrinkfails;
uint64_t ninserts;
uint64_t nrelocs;
uint64_t ngrows;
uint64_t nshrinks;
uint64_t nshrinkfails;
uint64_t ninserts;
uint64_t nrelocs;
#endif
/* Used for pseudo-random number generation. */
#define CKH_A 1103515241
#define CKH_C 12347
uint32_t prng_state;
uint64_t prng_state;
/* Total number of items. */
size_t count;
size_t count;
/*
* Minimum and current number of hash table buckets. There are
* 2^LG_CKH_BUCKET_CELLS cells per bucket.
*/
unsigned lg_minbuckets;
unsigned lg_curbuckets;
unsigned lg_minbuckets;
unsigned lg_curbuckets;
/* Hash and comparison functions. */
ckh_hash_t *hash;
ckh_keycomp_t *keycomp;
ckh_hash_t *hash;
ckh_keycomp_t *keycomp;
/* Hash table with 2^lg_curbuckets buckets. */
ckhc_t *tab;
};
ckhc_t *tab;
} ckh_t;
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
/* BEGIN PUBLIC API */
/******************************************************************************/
bool ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
/* Lifetime management. Minitems is the initial capacity. */
bool ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh_keycomp_t *keycomp);
void ckh_delete(tsd_t *tsd, ckh_t *ckh);
size_t ckh_count(ckh_t *ckh);
bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data);
bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data);
bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
void ckh_delete(tsd_t *tsd, ckh_t *ckh);
/* Get the number of elements in the set. */
size_t ckh_count(ckh_t *ckh);
/*
* To iterate over the elements in the table, initialize *tabind to 0 and call
* this function until it returns true. Each call that returns false will
* update *key and *data to the next element in the table, assuming the pointers
* are non-NULL.
*/
bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data);
/*
* Basic hash table operations -- insert, removal, lookup. For ckh_remove and
* ckh_search, key or data can be NULL. The hash-table only stores pointers to
* the key and value, and doesn't do any lifetime management.
*/
bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data);
bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
void **data);
bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data);
void ckh_string_hash(const void *key, size_t r_hash[2]);
bool ckh_string_keycomp(const void *k1, const void *k2);
void ckh_pointer_hash(const void *key, size_t r_hash[2]);
bool ckh_pointer_keycomp(const void *k1, const void *k2);
bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
/* Some useful hash and comparison functions for strings and pointers. */
void ckh_string_hash(const void *key, size_t r_hash[2]);
bool ckh_string_keycomp(const void *k1, const void *k2);
void ckh_pointer_hash(const void *key, size_t r_hash[2]);
bool ckh_pointer_keycomp(const void *k1, const void *k2);
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#endif /* JEMALLOC_INTERNAL_CKH_H */
deps/jemalloc/include/jemalloc/internal/ctl.h
Vendored
+91 -71
View File
@@ -1,81 +1,107 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#ifndef JEMALLOC_INTERNAL_CTL_H
#define JEMALLOC_INTERNAL_CTL_H
typedef struct ctl_node_s ctl_node_t;
typedef struct ctl_named_node_s ctl_named_node_t;
typedef struct ctl_indexed_node_s ctl_indexed_node_t;
typedef struct ctl_arena_stats_s ctl_arena_stats_t;
typedef struct ctl_stats_s ctl_stats_t;
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex_prof.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/stats.h"
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
/* Maximum ctl tree depth. */
#define CTL_MAX_DEPTH 7
struct ctl_node_s {
bool named;
};
typedef struct ctl_node_s {
bool named;
} ctl_node_t;
struct ctl_named_node_s {
struct ctl_node_s node;
const char *name;
typedef struct ctl_named_node_s {
ctl_node_t node;
const char *name;
/* If (nchildren == 0), this is a terminal node. */
unsigned nchildren;
const ctl_node_t *children;
int (*ctl)(const size_t *, size_t, void *, size_t *,
void *, size_t);
};
size_t nchildren;
const ctl_node_t *children;
int (*ctl)(tsd_t *, const size_t *, size_t, void *, size_t *, void *,
size_t);
} ctl_named_node_t;
struct ctl_indexed_node_s {
struct ctl_node_s node;
const ctl_named_node_t *(*index)(const size_t *, size_t, size_t);
};
typedef struct ctl_indexed_node_s {
struct ctl_node_s node;
const ctl_named_node_t *(*index)(tsdn_t *, const size_t *, size_t,
size_t);
} ctl_indexed_node_t;
struct ctl_arena_stats_s {
bool initialized;
unsigned nthreads;
const char *dss;
ssize_t lg_dirty_mult;
size_t pactive;
size_t pdirty;
arena_stats_t astats;
typedef struct ctl_arena_stats_s {
arena_stats_t astats;
/* Aggregate stats for small size classes, based on bin stats. */
size_t allocated_small;
uint64_t nmalloc_small;
uint64_t ndalloc_small;
uint64_t nrequests_small;
size_t allocated_small;
uint64_t nmalloc_small;
uint64_t ndalloc_small;
uint64_t nrequests_small;
malloc_bin_stats_t bstats[NBINS];
malloc_large_stats_t *lstats; /* nlclasses elements. */
malloc_huge_stats_t *hstats; /* nhclasses elements. */
bin_stats_t bstats[NBINS];
arena_stats_large_t lstats[NSIZES - NBINS];
} ctl_arena_stats_t;
typedef struct ctl_stats_s {
size_t allocated;
size_t active;
size_t metadata;
size_t metadata_thp;
size_t resident;
size_t mapped;
size_t retained;
background_thread_stats_t background_thread;
mutex_prof_data_t mutex_prof_data[mutex_prof_num_global_mutexes];
} ctl_stats_t;
typedef struct ctl_arena_s ctl_arena_t;
struct ctl_arena_s {
unsigned arena_ind;
bool initialized;
ql_elm(ctl_arena_t) destroyed_link;
/* Basic stats, supported even if !config_stats. */
unsigned nthreads;
const char *dss;
ssize_t dirty_decay_ms;
ssize_t muzzy_decay_ms;
size_t pactive;
size_t pdirty;
size_t pmuzzy;
/* NULL if !config_stats. */
ctl_arena_stats_t *astats;
};
struct ctl_stats_s {
size_t allocated;
size_t active;
size_t metadata;
size_t resident;
size_t mapped;
unsigned narenas;
ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */
};
typedef struct ctl_arenas_s {
uint64_t epoch;
unsigned narenas;
ql_head(ctl_arena_t) destroyed;
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
/*
* Element 0 corresponds to merged stats for extant arenas (accessed via
* MALLCTL_ARENAS_ALL), element 1 corresponds to merged stats for
* destroyed arenas (accessed via MALLCTL_ARENAS_DESTROYED), and the
* remaining MALLOCX_ARENA_LIMIT elements correspond to arenas.
*/
ctl_arena_t *arenas[2 + MALLOCX_ARENA_LIMIT];
} ctl_arenas_t;
int ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp,
size_t newlen);
int ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp);
int ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
void *newp, size_t newlen);
bool ctl_boot(void);
void ctl_prefork(void);
void ctl_postfork_parent(void);
void ctl_postfork_child(void);
int ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp);
#define xmallctl(name, oldp, oldlenp, newp, newlen) do { \
int ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen);
bool ctl_boot(void);
void ctl_prefork(tsdn_t *tsdn);
void ctl_postfork_parent(tsdn_t *tsdn);
void ctl_postfork_child(tsdn_t *tsdn);
#define xmallctl(name, oldp, oldlenp, newp, newlen) do { \
if (je_mallctl(name, oldp, oldlenp, newp, newlen) \
!= 0) { \
malloc_printf( \
@@ -85,7 +111,7 @@ void ctl_postfork_child(void);
} \
} while (0)
#define xmallctlnametomib(name, mibp, miblenp) do { \
#define xmallctlnametomib(name, mibp, miblenp) do { \
if (je_mallctlnametomib(name, mibp, miblenp) != 0) { \
malloc_printf("<jemalloc>: Failure in " \
"xmallctlnametomib(\"%s\", ...)\n", name); \
@@ -93,7 +119,7 @@ void ctl_postfork_child(void);
} \
} while (0)
#define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do { \
#define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do { \
if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp, \
newlen) != 0) { \
malloc_write( \
@@ -102,10 +128,4 @@ void ctl_postfork_child(void);
} \
} while (0)
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#endif /* JEMALLOC_INTERNAL_CTL_H */
deps/jemalloc/include/jemalloc/internal/div.h
Vendored
+41
View File
@@ -0,0 +1,41 @@
#ifndef JEMALLOC_INTERNAL_DIV_H
#define JEMALLOC_INTERNAL_DIV_H
#include "jemalloc/internal/assert.h"
/*
* This module does the division that computes the index of a region in a slab,
* given its offset relative to the base.
* That is, given a divisor d, an n = i * d (all integers), we'll return i.
* We do some pre-computation to do this more quickly than a CPU division
* instruction.
* We bound n < 2^32, and don't support dividing by one.
*/
typedef struct div_info_s div_info_t;
struct div_info_s {
uint32_t magic;
#ifdef JEMALLOC_DEBUG
size_t d;
#endif
};
void div_init(div_info_t *div_info, size_t divisor);
static inline size_t
div_compute(div_info_t *div_info, size_t n) {
assert(n <= (uint32_t)-1);
/*
* This generates, e.g. mov; imul; shr on x86-64. On a 32-bit machine,
* the compilers I tried were all smart enough to turn this into the
* appropriate "get the high 32 bits of the result of a multiply" (e.g.
* mul; mov edx eax; on x86, umull on arm, etc.).
*/
size_t i = ((uint64_t)n * (uint64_t)div_info->magic) >> 32;
#ifdef JEMALLOC_DEBUG
assert(i * div_info->d == n);
#endif
return i;
}
#endif /* JEMALLOC_INTERNAL_DIV_H */
deps/jemalloc/include/jemalloc/internal/emitter.h
Vendored
+435
View File
@@ -0,0 +1,435 @@
#ifndef JEMALLOC_INTERNAL_EMITTER_H
#define JEMALLOC_INTERNAL_EMITTER_H
#include "jemalloc/internal/ql.h"
typedef enum emitter_output_e emitter_output_t;
enum emitter_output_e {
emitter_output_json,
emitter_output_table
};
typedef enum emitter_justify_e emitter_justify_t;
enum emitter_justify_e {
emitter_justify_left,
emitter_justify_right,
/* Not for users; just to pass to internal functions. */
emitter_justify_none
};
typedef enum emitter_type_e emitter_type_t;
enum emitter_type_e {
emitter_type_bool,
emitter_type_int,
emitter_type_unsigned,
emitter_type_uint32,
emitter_type_uint64,
emitter_type_size,
emitter_type_ssize,
emitter_type_string,
/*
* A title is a column title in a table; it's just a string, but it's
* not quoted.
*/
emitter_type_title,
};
typedef struct emitter_col_s emitter_col_t;
struct emitter_col_s {
/* Filled in by the user. */
emitter_justify_t justify;
int width;
emitter_type_t type;
union {
bool bool_val;
int int_val;
unsigned unsigned_val;
uint32_t uint32_val;
uint64_t uint64_val;
size_t size_val;
ssize_t ssize_val;
const char *str_val;
};
/* Filled in by initialization. */
ql_elm(emitter_col_t) link;
};
typedef struct emitter_row_s emitter_row_t;
struct emitter_row_s {
ql_head(emitter_col_t) cols;
};
static inline void
emitter_row_init(emitter_row_t *row) {
ql_new(&row->cols);
}
static inline void
emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
ql_elm_new(col, link);
ql_tail_insert(&row->cols, col, link);
}
typedef struct emitter_s emitter_t;
struct emitter_s {
emitter_output_t output;
/* The output information. */
void (*write_cb)(void *, const char *);
void *cbopaque;
int nesting_depth;
/* True if we've already emitted a value at the given depth. */
bool item_at_depth;
};
static inline void
emitter_init(emitter_t *emitter, emitter_output_t emitter_output,
void (*write_cb)(void *, const char *), void *cbopaque) {
emitter->output = emitter_output;
emitter->write_cb = write_cb;
emitter->cbopaque = cbopaque;
emitter->item_at_depth = false;
emitter->nesting_depth = 0;
}
/* Internal convenience function. Write to the emitter the given string. */
JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void
emitter_printf(emitter_t *emitter, const char *format, ...) {
va_list ap;
va_start(ap, format);
malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap);
va_end(ap);
}
/* Write to the emitter the given string, but only in table mode. */
JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void
emitter_table_printf(emitter_t *emitter, const char *format, ...) {
if (emitter->output == emitter_output_table) {
va_list ap;
va_start(ap, format);
malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap);
va_end(ap);
}
}
static inline void
emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier,
emitter_justify_t justify, int width) {
size_t written;
if (justify == emitter_justify_none) {
written = malloc_snprintf(out_fmt, out_size,
"%%%s", fmt_specifier);
} else if (justify == emitter_justify_left) {
written = malloc_snprintf(out_fmt, out_size,
"%%-%d%s", width, fmt_specifier);
} else {
written = malloc_snprintf(out_fmt, out_size,
"%%%d%s", width, fmt_specifier);
}
/* Only happens in case of bad format string, which *we* choose. */
assert(written < out_size);
}
/*
* Internal. Emit the given value type in the relevant encoding (so that the
* bool true gets mapped to json "true", but the string "true" gets mapped to
* json "\"true\"", for instance.
*
* Width is ignored if justify is emitter_justify_none.
*/
static inline void
emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
emitter_type_t value_type, const void *value) {
size_t str_written;
#define BUF_SIZE 256
#define FMT_SIZE 10
/*
* We dynamically generate a format string to emit, to let us use the
* snprintf machinery. This is kinda hacky, but gets the job done
* quickly without having to think about the various snprintf edge
* cases.
*/
char fmt[FMT_SIZE];
char buf[BUF_SIZE];
#define EMIT_SIMPLE(type, format) \
emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width); \
emitter_printf(emitter, fmt, *(const type *)value); \
switch (value_type) {
case emitter_type_bool:
emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width);
emitter_printf(emitter, fmt, *(const bool *)value ?
"true" : "false");
break;
case emitter_type_int:
EMIT_SIMPLE(int, "d")
break;
case emitter_type_unsigned:
EMIT_SIMPLE(unsigned, "u")
break;
case emitter_type_ssize:
EMIT_SIMPLE(ssize_t, "zd")
break;
case emitter_type_size:
EMIT_SIMPLE(size_t, "zu")
break;
case emitter_type_string:
str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"",
*(const char *const *)value);
/*
* We control the strings we output; we shouldn't get anything
* anywhere near the fmt size.
*/
assert(str_written < BUF_SIZE);
emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width);
emitter_printf(emitter, fmt, buf);
break;
case emitter_type_uint32:
EMIT_SIMPLE(uint32_t, FMTu32)
break;
case emitter_type_uint64:
EMIT_SIMPLE(uint64_t, FMTu64)
break;
case emitter_type_title:
EMIT_SIMPLE(char *const, "s");
break;
default:
unreachable();
}
#undef BUF_SIZE
#undef FMT_SIZE
}
/* Internal functions. In json mode, tracks nesting state. */
static inline void
emitter_nest_inc(emitter_t *emitter) {
emitter->nesting_depth++;
emitter->item_at_depth = false;
}
static inline void
emitter_nest_dec(emitter_t *emitter) {
emitter->nesting_depth--;
emitter->item_at_depth = true;
}
static inline void
emitter_indent(emitter_t *emitter) {
int amount = emitter->nesting_depth;
const char *indent_str;
if (emitter->output == emitter_output_json) {
indent_str = "\t";
} else {
amount *= 2;
indent_str = " ";
}
for (int i = 0; i < amount; i++) {
emitter_printf(emitter, "%s", indent_str);
}
}
static inline void
emitter_json_key_prefix(emitter_t *emitter) {
emitter_printf(emitter, "%s\n", emitter->item_at_depth ? "," : "");
emitter_indent(emitter);
}
static inline void
emitter_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth == 0);
emitter_printf(emitter, "{");
emitter_nest_inc(emitter);
} else {
// tabular init
emitter_printf(emitter, "%s", "");
}
}
static inline void
emitter_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth == 1);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n}\n");
}
}
/*
* Note emits a different kv pair as well, but only in table mode. Omits the
* note if table_note_key is NULL.
*/
static inline void
emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_type_t value_type, const void *value,
const char *table_note_key, emitter_type_t table_note_value_type,
const void *table_note_value) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": ", json_key);
emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value);
} else {
emitter_indent(emitter);
emitter_printf(emitter, "%s: ", table_key);
emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value);
if (table_note_key != NULL) {
emitter_printf(emitter, " (%s: ", table_note_key);
emitter_print_value(emitter, emitter_justify_none, -1,
table_note_value_type, table_note_value);
emitter_printf(emitter, ")");
}
emitter_printf(emitter, "\n");
}
emitter->item_at_depth = true;
}
static inline void
emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_type_t value_type, const void *value) {
emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL,
emitter_type_bool, NULL);
}
static inline void
emitter_json_kv(emitter_t *emitter, const char *json_key,
emitter_type_t value_type, const void *value) {
if (emitter->output == emitter_output_json) {
emitter_kv(emitter, json_key, NULL, value_type, value);
}
}
static inline void
emitter_table_kv(emitter_t *emitter, const char *table_key,
emitter_type_t value_type, const void *value) {
if (emitter->output == emitter_output_table) {
emitter_kv(emitter, NULL, table_key, value_type, value);
}
}
static inline void
emitter_dict_begin(emitter_t *emitter, const char *json_key,
const char *table_header) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": {", json_key);
emitter_nest_inc(emitter);
} else {
emitter_indent(emitter);
emitter_printf(emitter, "%s\n", table_header);
emitter_nest_inc(emitter);
}
}
static inline void
emitter_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n");
emitter_indent(emitter);
emitter_printf(emitter, "}");
} else {
emitter_nest_dec(emitter);
}
}
static inline void
emitter_json_dict_begin(emitter_t *emitter, const char *json_key) {
if (emitter->output == emitter_output_json) {
emitter_dict_begin(emitter, json_key, NULL);
}
}
static inline void
emitter_json_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
emitter_dict_end(emitter);
}
}
static inline void
emitter_table_dict_begin(emitter_t *emitter, const char *table_key) {
if (emitter->output == emitter_output_table) {
emitter_dict_begin(emitter, NULL, table_key);
}
}
static inline void
emitter_table_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_table) {
emitter_dict_end(emitter);
}
}
static inline void
emitter_json_arr_begin(emitter_t *emitter, const char *json_key) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": [", json_key);
emitter_nest_inc(emitter);
}
}
static inline void
emitter_json_arr_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n");
emitter_indent(emitter);
emitter_printf(emitter, "]");
}
}
static inline void
emitter_json_arr_obj_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "{");
emitter_nest_inc(emitter);
}
}
static inline void
emitter_json_arr_obj_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n");
emitter_indent(emitter);
emitter_printf(emitter, "}");
}
}
static inline void
emitter_json_arr_value(emitter_t *emitter, emitter_type_t value_type,
const void *value) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value);
}
}
static inline void
emitter_table_row(emitter_t *emitter, emitter_row_t *row) {
if (emitter->output != emitter_output_table) {
return;
}
emitter_col_t *col;
ql_foreach(col, &row->cols, link) {
emitter_print_value(emitter, col->justify, col->width,
col->type, (const void *)&col->bool_val);
}
emitter_table_printf(emitter, "\n");
}
#endif /* JEMALLOC_INTERNAL_EMITTER_H */
deps/jemalloc/include/jemalloc/internal/extent.h
Vendored
-239
View File
@@ -1,239 +0,0 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
typedef struct extent_node_s extent_node_t;
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
/* Tree of extents. Use accessor functions for en_* fields. */
struct extent_node_s {
/* Arena from which this extent came, if any. */
arena_t *en_arena;
/* Pointer to the extent that this tree node is responsible for. */
void *en_addr;
/* Total region size. */
size_t en_size;
/*
* The zeroed flag is used by chunk recycling code to track whether
* memory is zero-filled.
*/
bool en_zeroed;
/*
* True if physical memory is committed to the extent, whether
* explicitly or implicitly as on a system that overcommits and
* satisfies physical memory needs on demand via soft page faults.
*/
bool en_committed;
/*
* The achunk flag is used to validate that huge allocation lookups
* don't return arena chunks.
*/
bool en_achunk;
/* Profile counters, used for huge objects. */
prof_tctx_t *en_prof_tctx;
/* Linkage for arena's runs_dirty and chunks_cache rings. */
arena_runs_dirty_link_t rd;
qr(extent_node_t) cc_link;
union {
/* Linkage for the size/address-ordered tree. */
rb_node(extent_node_t) szad_link;
/* Linkage for arena's huge and node_cache lists. */
ql_elm(extent_node_t) ql_link;
};
/* Linkage for the address-ordered tree. */
rb_node(extent_node_t) ad_link;
};
typedef rb_tree(extent_node_t) extent_tree_t;
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t)
rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t)
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
arena_t *extent_node_arena_get(const extent_node_t *node);
void *extent_node_addr_get(const extent_node_t *node);
size_t extent_node_size_get(const extent_node_t *node);
bool extent_node_zeroed_get(const extent_node_t *node);
bool extent_node_committed_get(const extent_node_t *node);
bool extent_node_achunk_get(const extent_node_t *node);
prof_tctx_t *extent_node_prof_tctx_get(const extent_node_t *node);
void extent_node_arena_set(extent_node_t *node, arena_t *arena);
void extent_node_addr_set(extent_node_t *node, void *addr);
void extent_node_size_set(extent_node_t *node, size_t size);
void extent_node_zeroed_set(extent_node_t *node, bool zeroed);
void extent_node_committed_set(extent_node_t *node, bool committed);
void extent_node_achunk_set(extent_node_t *node, bool achunk);
void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx);
void extent_node_init(extent_node_t *node, arena_t *arena, void *addr,
size_t size, bool zeroed, bool committed);
void extent_node_dirty_linkage_init(extent_node_t *node);
void extent_node_dirty_insert(extent_node_t *node,
arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty);
void extent_node_dirty_remove(extent_node_t *node);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
JEMALLOC_INLINE arena_t *
extent_node_arena_get(const extent_node_t *node)
{
return (node->en_arena);
}
JEMALLOC_INLINE void *
extent_node_addr_get(const extent_node_t *node)
{
return (node->en_addr);
}
JEMALLOC_INLINE size_t
extent_node_size_get(const extent_node_t *node)
{
return (node->en_size);
}
JEMALLOC_INLINE bool
extent_node_zeroed_get(const extent_node_t *node)
{
return (node->en_zeroed);
}
JEMALLOC_INLINE bool
extent_node_committed_get(const extent_node_t *node)
{
assert(!node->en_achunk);
return (node->en_committed);
}
JEMALLOC_INLINE bool
extent_node_achunk_get(const extent_node_t *node)
{
return (node->en_achunk);
}
JEMALLOC_INLINE prof_tctx_t *
extent_node_prof_tctx_get(const extent_node_t *node)
{
return (node->en_prof_tctx);
}
JEMALLOC_INLINE void
extent_node_arena_set(extent_node_t *node, arena_t *arena)
{
node->en_arena = arena;
}
JEMALLOC_INLINE void
extent_node_addr_set(extent_node_t *node, void *addr)
{
node->en_addr = addr;
}
JEMALLOC_INLINE void
extent_node_size_set(extent_node_t *node, size_t size)
{
node->en_size = size;
}
JEMALLOC_INLINE void
extent_node_zeroed_set(extent_node_t *node, bool zeroed)
{
node->en_zeroed = zeroed;
}
JEMALLOC_INLINE void
extent_node_committed_set(extent_node_t *node, bool committed)
{
node->en_committed = committed;
}
JEMALLOC_INLINE void
extent_node_achunk_set(extent_node_t *node, bool achunk)
{
node->en_achunk = achunk;
}
JEMALLOC_INLINE void
extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx)
{
node->en_prof_tctx = tctx;
}
JEMALLOC_INLINE void
extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size,
bool zeroed, bool committed)
{
extent_node_arena_set(node, arena);
extent_node_addr_set(node, addr);
extent_node_size_set(node, size);
extent_node_zeroed_set(node, zeroed);
extent_node_committed_set(node, committed);
extent_node_achunk_set(node, false);
if (config_prof)
extent_node_prof_tctx_set(node, NULL);
}
JEMALLOC_INLINE void
extent_node_dirty_linkage_init(extent_node_t *node)
{
qr_new(&node->rd, rd_link);
qr_new(node, cc_link);
}
JEMALLOC_INLINE void
extent_node_dirty_insert(extent_node_t *node,
arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty)
{
qr_meld(runs_dirty, &node->rd, rd_link);
qr_meld(chunks_dirty, node, cc_link);
}
JEMALLOC_INLINE void
extent_node_dirty_remove(extent_node_t *node)
{
qr_remove(&node->rd, rd_link);
qr_remove(node, cc_link);
}
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
deps/jemalloc/include/jemalloc/internal/extent_dss.h
Vendored
+26
View File
@@ -0,0 +1,26 @@
#ifndef JEMALLOC_INTERNAL_EXTENT_DSS_H
#define JEMALLOC_INTERNAL_EXTENT_DSS_H
typedef enum {
dss_prec_disabled = 0,
dss_prec_primary = 1,
dss_prec_secondary = 2,
dss_prec_limit = 3
} dss_prec_t;
#define DSS_PREC_DEFAULT dss_prec_secondary
#define DSS_DEFAULT "secondary"
extern const char *dss_prec_names[];
extern const char *opt_dss;
dss_prec_t extent_dss_prec_get(void);
bool extent_dss_prec_set(dss_prec_t dss_prec);
void *extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit);
bool extent_in_dss(void *addr);
bool extent_dss_mergeable(void *addr_a, void *addr_b);
void extent_dss_boot(void);
#endif /* JEMALLOC_INTERNAL_EXTENT_DSS_H */
deps/jemalloc/include/jemalloc/internal/extent_externs.h
Vendored
+73
View File
@@ -0,0 +1,73 @@
#ifndef JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
#define JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mutex_pool.h"
#include "jemalloc/internal/ph.h"
#include "jemalloc/internal/rtree.h"
extern size_t opt_lg_extent_max_active_fit;
extern rtree_t extents_rtree;
extern const extent_hooks_t extent_hooks_default;
extern mutex_pool_t extent_mutex_pool;
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
extent_hooks_t *extent_hooks_get(arena_t *arena);
extent_hooks_t *extent_hooks_set(tsd_t *tsd, arena_t *arena,
extent_hooks_t *extent_hooks);
#ifdef JEMALLOC_JET
size_t extent_size_quantize_floor(size_t size);
size_t extent_size_quantize_ceil(size_t size);
#endif
rb_proto(, extent_avail_, extent_tree_t, extent_t)
ph_proto(, extent_heap_, extent_heap_t, extent_t)
bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,
bool delay_coalesce);
extent_state_t extents_state_get(const extents_t *extents);
size_t extents_npages_get(extents_t *extents);
extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr,
size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
bool *zero, bool *commit);
void extents_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent);
extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_min);
void extents_prefork(tsdn_t *tsdn, extents_t *extents);
void extents_postfork_parent(tsdn_t *tsdn, extents_t *extents);
void extents_postfork_child(tsdn_t *tsdn, extents_t *extents);
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit);
void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent);
void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent);
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b);
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b);
bool extent_boot(void);
#endif /* JEMALLOC_INTERNAL_EXTENT_EXTERNS_H */
deps/jemalloc/include/jemalloc/internal/extent_inlines.h
Vendored
+433
View File
@@ -0,0 +1,433 @@
#ifndef JEMALLOC_INTERNAL_EXTENT_INLINES_H
#define JEMALLOC_INTERNAL_EXTENT_INLINES_H
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mutex_pool.h"
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/sz.h"
static inline void
extent_lock(tsdn_t *tsdn, extent_t *extent) {
assert(extent != NULL);
mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent);
}
static inline void
extent_unlock(tsdn_t *tsdn, extent_t *extent) {
assert(extent != NULL);
mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)extent);
}
static inline void
extent_lock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
assert(extent1 != NULL && extent2 != NULL);
mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1,
(uintptr_t)extent2);
}
static inline void
extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
assert(extent1 != NULL && extent2 != NULL);
mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1,
(uintptr_t)extent2);
}
static inline arena_t *
extent_arena_get(const extent_t *extent) {
unsigned arena_ind = (unsigned)((extent->e_bits &
EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT);
/*
* The following check is omitted because we should never actually read
* a NULL arena pointer.
*/
if (false && arena_ind >= MALLOCX_ARENA_LIMIT) {
return NULL;
}
assert(arena_ind < MALLOCX_ARENA_LIMIT);
return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE);
}
static inline szind_t
extent_szind_get_maybe_invalid(const extent_t *extent) {
szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >>
EXTENT_BITS_SZIND_SHIFT);
assert(szind <= NSIZES);
return szind;
}
static inline szind_t
extent_szind_get(const extent_t *extent) {
szind_t szind = extent_szind_get_maybe_invalid(extent);
assert(szind < NSIZES); /* Never call when "invalid". */
return szind;
}
static inline size_t
extent_usize_get(const extent_t *extent) {
return sz_index2size(extent_szind_get(extent));
}
static inline size_t
extent_sn_get(const extent_t *extent) {
return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >>
EXTENT_BITS_SN_SHIFT);
}
static inline extent_state_t
extent_state_get(const extent_t *extent) {
return (extent_state_t)((extent->e_bits & EXTENT_BITS_STATE_MASK) >>
EXTENT_BITS_STATE_SHIFT);
}
static inline bool
extent_zeroed_get(const extent_t *extent) {
return (bool)((extent->e_bits & EXTENT_BITS_ZEROED_MASK) >>
EXTENT_BITS_ZEROED_SHIFT);
}
static inline bool
extent_committed_get(const extent_t *extent) {
return (bool)((extent->e_bits & EXTENT_BITS_COMMITTED_MASK) >>
EXTENT_BITS_COMMITTED_SHIFT);
}
static inline bool
extent_dumpable_get(const extent_t *extent) {
return (bool)((extent->e_bits & EXTENT_BITS_DUMPABLE_MASK) >>
EXTENT_BITS_DUMPABLE_SHIFT);
}
static inline bool
extent_slab_get(const extent_t *extent) {
return (bool)((extent->e_bits & EXTENT_BITS_SLAB_MASK) >>
EXTENT_BITS_SLAB_SHIFT);
}
static inline unsigned
extent_nfree_get(const extent_t *extent) {
assert(extent_slab_get(extent));
return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK) >>
EXTENT_BITS_NFREE_SHIFT);
}
static inline void *
extent_base_get(const extent_t *extent) {
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
!extent_slab_get(extent));
return PAGE_ADDR2BASE(extent->e_addr);
}
static inline void *
extent_addr_get(const extent_t *extent) {
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
!extent_slab_get(extent));
return extent->e_addr;
}
static inline size_t
extent_size_get(const extent_t *extent) {
return (extent->e_size_esn & EXTENT_SIZE_MASK);
}
static inline size_t
extent_esn_get(const extent_t *extent) {
return (extent->e_size_esn & EXTENT_ESN_MASK);
}
static inline size_t
extent_bsize_get(const extent_t *extent) {
return extent->e_bsize;
}
static inline void *
extent_before_get(const extent_t *extent) {
return (void *)((uintptr_t)extent_base_get(extent) - PAGE);
}
static inline void *
extent_last_get(const extent_t *extent) {
return (void *)((uintptr_t)extent_base_get(extent) +
extent_size_get(extent) - PAGE);
}
static inline void *
extent_past_get(const extent_t *extent) {
return (void *)((uintptr_t)extent_base_get(extent) +
extent_size_get(extent));
}
static inline arena_slab_data_t *
extent_slab_data_get(extent_t *extent) {
assert(extent_slab_get(extent));
return &extent->e_slab_data;
}
static inline const arena_slab_data_t *
extent_slab_data_get_const(const extent_t *extent) {
assert(extent_slab_get(extent));
return &extent->e_slab_data;
}
static inline prof_tctx_t *
extent_prof_tctx_get(const extent_t *extent) {
return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx,
ATOMIC_ACQUIRE);
}
static inline void
extent_arena_set(extent_t *extent, arena_t *arena) {
unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U <<
MALLOCX_ARENA_BITS) - 1);
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ARENA_MASK) |
((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT);
}
static inline void
extent_addr_set(extent_t *extent, void *addr) {
extent->e_addr = addr;
}
static inline void
extent_addr_randomize(UNUSED tsdn_t *tsdn, extent_t *extent, size_t alignment) {
assert(extent_base_get(extent) == extent_addr_get(extent));
if (alignment < PAGE) {
unsigned lg_range = LG_PAGE -
lg_floor(CACHELINE_CEILING(alignment));
size_t r;
if (!tsdn_null(tsdn)) {
tsd_t *tsd = tsdn_tsd(tsdn);
r = (size_t)prng_lg_range_u64(
tsd_offset_statep_get(tsd), lg_range);
} else {
r = prng_lg_range_zu(
&extent_arena_get(extent)->offset_state,
lg_range, true);
}
uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE -
lg_range);
extent->e_addr = (void *)((uintptr_t)extent->e_addr +
random_offset);
assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) ==
extent->e_addr);
}
}
static inline void
extent_size_set(extent_t *extent, size_t size) {
assert((size & ~EXTENT_SIZE_MASK) == 0);
extent->e_size_esn = size | (extent->e_size_esn & ~EXTENT_SIZE_MASK);
}
static inline void
extent_esn_set(extent_t *extent, size_t esn) {
extent->e_size_esn = (extent->e_size_esn & ~EXTENT_ESN_MASK) | (esn &
EXTENT_ESN_MASK);
}
static inline void
extent_bsize_set(extent_t *extent, size_t bsize) {
extent->e_bsize = bsize;
}
static inline void
extent_szind_set(extent_t *extent, szind_t szind) {
assert(szind <= NSIZES); /* NSIZES means "invalid". */
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) |
((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT);
}
static inline void
extent_nfree_set(extent_t *extent, unsigned nfree) {
assert(extent_slab_get(extent));
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) |
((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT);
}
static inline void
extent_nfree_inc(extent_t *extent) {
assert(extent_slab_get(extent));
extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
}
static inline void
extent_nfree_dec(extent_t *extent) {
assert(extent_slab_get(extent));
extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
}
static inline void
extent_sn_set(extent_t *extent, size_t sn) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) |
((uint64_t)sn << EXTENT_BITS_SN_SHIFT);
}
static inline void
extent_state_set(extent_t *extent, extent_state_t state) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK) |
((uint64_t)state << EXTENT_BITS_STATE_SHIFT);
}
static inline void
extent_zeroed_set(extent_t *extent, bool zeroed) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ZEROED_MASK) |
((uint64_t)zeroed << EXTENT_BITS_ZEROED_SHIFT);
}
static inline void
extent_committed_set(extent_t *extent, bool committed) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_COMMITTED_MASK) |
((uint64_t)committed << EXTENT_BITS_COMMITTED_SHIFT);
}
static inline void
extent_dumpable_set(extent_t *extent, bool dumpable) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_DUMPABLE_MASK) |
((uint64_t)dumpable << EXTENT_BITS_DUMPABLE_SHIFT);
}
static inline void
extent_slab_set(extent_t *extent, bool slab) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SLAB_MASK) |
((uint64_t)slab << EXTENT_BITS_SLAB_SHIFT);
}
static inline void
extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) {
atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE);
}
static inline void
extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed,
bool committed, bool dumpable) {
assert(addr == PAGE_ADDR2BASE(addr) || !slab);
extent_arena_set(extent, arena);
extent_addr_set(extent, addr);
extent_size_set(extent, size);
extent_slab_set(extent, slab);
extent_szind_set(extent, szind);
extent_sn_set(extent, sn);
extent_state_set(extent, state);
extent_zeroed_set(extent, zeroed);
extent_committed_set(extent, committed);
extent_dumpable_set(extent, dumpable);
ql_elm_new(extent, ql_link);
if (config_prof) {
extent_prof_tctx_set(extent, NULL);
}
}
static inline void
extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) {
extent_arena_set(extent, NULL);
extent_addr_set(extent, addr);
extent_bsize_set(extent, bsize);
extent_slab_set(extent, false);
extent_szind_set(extent, NSIZES);
extent_sn_set(extent, sn);
extent_state_set(extent, extent_state_active);
extent_zeroed_set(extent, true);
extent_committed_set(extent, true);
extent_dumpable_set(extent, true);
}
static inline void
extent_list_init(extent_list_t *list) {
ql_new(list);
}
static inline extent_t *
extent_list_first(const extent_list_t *list) {
return ql_first(list);
}
static inline extent_t *
extent_list_last(const extent_list_t *list) {
return ql_last(list, ql_link);
}
static inline void
extent_list_append(extent_list_t *list, extent_t *extent) {
ql_tail_insert(list, extent, ql_link);
}
static inline void
extent_list_prepend(extent_list_t *list, extent_t *extent) {
ql_head_insert(list, extent, ql_link);
}
static inline void
extent_list_replace(extent_list_t *list, extent_t *to_remove,
extent_t *to_insert) {
ql_after_insert(to_remove, to_insert, ql_link);
ql_remove(list, to_remove, ql_link);
}
static inline void
extent_list_remove(extent_list_t *list, extent_t *extent) {
ql_remove(list, extent, ql_link);
}
static inline int
extent_sn_comp(const extent_t *a, const extent_t *b) {
size_t a_sn = extent_sn_get(a);
size_t b_sn = extent_sn_get(b);
return (a_sn > b_sn) - (a_sn < b_sn);
}
static inline int
extent_esn_comp(const extent_t *a, const extent_t *b) {
size_t a_esn = extent_esn_get(a);
size_t b_esn = extent_esn_get(b);
return (a_esn > b_esn) - (a_esn < b_esn);
}
static inline int
extent_ad_comp(const extent_t *a, const extent_t *b) {
uintptr_t a_addr = (uintptr_t)extent_addr_get(a);
uintptr_t b_addr = (uintptr_t)extent_addr_get(b);
return (a_addr > b_addr) - (a_addr < b_addr);
}
static inline int
extent_ead_comp(const extent_t *a, const extent_t *b) {
uintptr_t a_eaddr = (uintptr_t)a;
uintptr_t b_eaddr = (uintptr_t)b;
return (a_eaddr > b_eaddr) - (a_eaddr < b_eaddr);
}
static inline int
extent_snad_comp(const extent_t *a, const extent_t *b) {
int ret;
ret = extent_sn_comp(a, b);
if (ret != 0) {
return ret;
}
ret = extent_ad_comp(a, b);
return ret;
}
static inline int
extent_esnead_comp(const extent_t *a, const extent_t *b) {
int ret;
ret = extent_esn_comp(a, b);
if (ret != 0) {
return ret;
}
ret = extent_ead_comp(a, b);
return ret;
}
#endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */

Some files were not shown because too many files have changed in this diff Show More