Redis/redis
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Compare commits

base: Redis/redis:6.2.19
Branches Tags
Redis/redis:6.2 Redis/redis:unstable Redis/redis:7.2 Redis/redis:7.4 Redis/redis:8.0
Redis/redis:7.2.10 Redis/redis:6.2.19 Redis/redis:7.4.5 Redis/redis:8.0.3 Redis/redis:8.2-rc1 Redis/redis:8.2-rc1-int Redis/redis:8.2-m01 Redis/redis:8.2-m01-int2 Redis/redis:8.2-m01-int Redis/redis:8.0.2 Redis/redis:7.2.9 Redis/redis:7.4.4 Redis/redis:8.0.1 Redis/redis:8.0.1-int Redis/redis:8.0.0 Redis/redis:8.0-rc2-int Redis/redis:7.4.3 Redis/redis:7.2.8 Redis/redis:6.2.18 Redis/redis:8.0-rc1 Redis/redis:8.0-rc1-int2 Redis/redis:8.0-rc1-int Redis/redis:8.0-m04 Redis/redis:8.0-m04-int Redis/redis:8.0-m03 Redis/redis:7.2.7 Redis/redis:7.4.2 Redis/redis:6.2.17 Redis/redis:8.0-m02 Redis/redis:6.2.16 Redis/redis:7.2.6 Redis/redis:7.4.1 Redis/redis:8.0-m01 Redis/redis:7.4.0 Redis/redis:7.4-rc2 Redis/redis:7.4-rc1 Redis/redis:7.2.5 Redis/redis:7.2.4 Redis/redis:7.0.15 Redis/redis:7.2.3 Redis/redis:7.2.2 Redis/redis:7.0.14 Redis/redis:6.2.15 Redis/redis:6.2.14 Redis/redis:7.2.1 Redis/redis:7.0.13 Redis/redis:7.2.0 Redis/redis:7.2-rc3 Redis/redis:7.0.12 Redis/redis:6.2.13 Redis/redis:6.0.20 Redis/redis:7.2-rc2 Redis/redis:6.0.19 Redis/redis:6.2.12 Redis/redis:7.0.11 Redis/redis:7.2-rc1 Redis/redis:7.0.10 Redis/redis:7.0.9 Redis/redis:6.2.11 Redis/redis:6.0.18 Redis/redis:6.2.10 Redis/redis:6.0.17 Redis/redis:6.2.9 Redis/redis:7.0.8 Redis/redis:7.0.7 Redis/redis:7.0.6 Redis/redis:6.2.8 Redis/redis:7.0.5 Redis/redis:7.0.4 Redis/redis:7.0.3 Redis/redis:7.0.2 Redis/redis:7.0.1 Redis/redis:7.0.0 Redis/redis:6.2.7 Redis/redis:7.0-rc3 Redis/redis:7.0-rc2 Redis/redis:7.0-rc1 Redis/redis:6.2.6 Redis/redis:6.0.16 Redis/redis:5.0.14 Redis/redis:6.2.5 Redis/redis:5.0.13 Redis/redis:6.0.15 Redis/redis:6.0.14 Redis/redis:6.2.4 Redis/redis:6.2.3 Redis/redis:6.0.13 Redis/redis:6.2.2 Redis/redis:6.2.1 Redis/redis:6.0.12 Redis/redis:5.0.12 Redis/redis:6.0.11 Redis/redis:6.2.0 Redis/redis:5.0.11 Redis/redis:6.2-rc3 Redis/redis:6.0.10 Redis/redis:6.2-rc2 Redis/redis:6.2-rc1 Redis/redis:6.0.9 Redis/redis:5.0.10 Redis/redis:6.0.8 Redis/redis:6.0.7 Redis/redis:6.0.6 Redis/redis:6.0.5 Redis/redis:6.0.4 Redis/redis:6.0.3 Redis/redis:6.0.2 Redis/redis:6.0.1 Redis/redis:6.0.0 Redis/redis:5.0.9 Redis/redis:6.0-rc4 Redis/redis:6.0-rc3 Redis/redis:5.0.8 Redis/redis:6.0-rc2 Redis/redis:6.0-rc1 Redis/redis:5.0.7 Redis/redis:5.0.6 Redis/redis:5.0.5 Redis/redis:3.2.13 Redis/redis:4.0.14 Redis/redis:5.0.4 Redis/redis:4.0.13 Redis/redis:5.0.3 Redis/redis:4.0.12 Redis/redis:5.0.2 Redis/redis:5.0.1 Redis/redis:5.0.0 Redis/redis:5.0-rc6 Redis/redis:5.0-rc5 Redis/redis:4.0.11 Redis/redis:5.0-rc4 Redis/redis:5.0-rc3 Redis/redis:5.0-rc2 Redis/redis:4.0.10 Redis/redis:3.2.12 Redis/redis:5.0-rc1 Redis/redis:4.0.9 Redis/redis:4.0.8 Redis/redis:4.0.7 Redis/redis:4.0.6 Redis/redis:4.0.5 Redis/redis:4.0.4 Redis/redis:4.0.3 Redis/redis:3.2.11 Redis/redis:4.0.2 Redis/redis:3.2.10 Redis/redis:4.0.1 Redis/redis:4.0.0 Redis/redis:3.2.9 Redis/redis:4.0-rc3 Redis/redis:3.2.8 Redis/redis:3.2.7 Redis/redis:3.2.6 Redis/redis:4.0-rc2 Redis/redis:4.0-rc1 Redis/redis:3.2.5 Redis/redis:3.2.4 Redis/redis:3.2.3 Redis/redis:3.2.2 Redis/redis:3.2.1 Redis/redis:3.2.0 Redis/redis:3.2.0-rc3 Redis/redis:3.0.7 Redis/redis:3.2.0-rc2 Redis/redis:3.2-rc1 Redis/redis:3.0.6 Redis/redis:2.8.24 Redis/redis:3.0.5 Redis/redis:2.8.23 Redis/redis:2.8.22 Redis/redis:3.0.4 Redis/redis:3.0.3 Redis/redis:3.0.2 Redis/redis:2.8.21 Redis/redis:2.8.20 Redis/redis:3.0.1 Redis/redis:3.0.0 Redis/redis:3.0.0-rc6 Redis/redis:3.0.0-rc5 Redis/redis:3.0.0-rc4 Redis/redis:3.0.0-rc3 Redis/redis:3.0.0-rc2 Redis/redis:2.8.19 Redis/redis:2.8.18 Redis/redis:3.0.0-rc1 Redis/redis:2.8.17 Redis/redis:2.8.16 Redis/redis:2.8.15 Redis/redis:2.8.14 Redis/redis:3.0.0-beta8 Redis/redis:2.8.13 Redis/redis:3.0.0-beta7 Redis/redis:2.8.12 Redis/redis:2.8.11 Redis/redis:3.0.0-beta6 Redis/redis:2.8.10 Redis/redis:3.0.0-beta5 Redis/redis:3.0.0-beta4 Redis/redis:3.0.0-beta3 Redis/redis:2.8.9 Redis/redis:2.8.8 Redis/redis:3.0.0-beta2 Redis/redis:2.8.7 Redis/redis:2.8.6 Redis/redis:3.0.0-beta1 Redis/redis:2.8.5 Redis/redis:2.8.4 Redis/redis:2.6.17 Redis/redis:2.8.3 Redis/redis:2.8.2 Redis/redis:2.8.1 Redis/redis:2.8.0 Redis/redis:2.8.0-rc6 Redis/redis:2.8.0-rc5 Redis/redis:2.8.0-rc4 Redis/redis:2.6.16 Redis/redis:2.6.15 Redis/redis:2.8.0-rc3 Redis/redis:2.8.0-rc2 Redis/redis:2.8.0-rc1 Redis/redis:2.6.14-2 Redis/redis:2.6.14-1 Redis/redis:2.6.14 Redis/redis:2.6.13 Redis/redis:2.6.12 Redis/redis:2.6.10-3 Redis/redis:2.6.10-2 Redis/redis:2.6.11 Redis/redis:2.6.10-1 Redis/redis:2.6.10 Redis/redis:2.6.9-1 Redis/redis:2.6.9 Redis/redis:2.6.8-1 Redis/redis:2.6.8 Redis/redis:2.6.7-1 Redis/redis:2.6.7 Redis/redis:2.6.6 Redis/redis:2.4.18 Redis/redis:2.6.5 Redis/redis:2.6.4 Redis/redis:2.6.3 Redis/redis:2.6.2 Redis/redis:2.6.1 Redis/redis:2.6.0 Redis/redis:2.6.0-rc8 Redis/redis:2.6.0-rc7 Redis/redis:2.4.17 Redis/redis:2.6.0-rc6 Redis/redis:2.4.16 Redis/redis:2.4.15 Redis/redis:2.6.0-rc5 Redis/redis:2.6.0-rc4 Redis/redis:2.4.14 Redis/redis:2.6.0-rc3 Redis/redis:2.6.0-rc2 Redis/redis:2.4.13 Redis/redis:2.4.12 Redis/redis:2.6.0-rc1 Redis/redis:2.4.11 Redis/redis:2.4.10 Redis/redis:2.4.9 Redis/redis:2.4.8 Redis/redis:2.4.7 Redis/redis:2.4.6 Redis/redis:2.4.5 Redis/redis:2.4.4 Redis/redis:2.4.3 Redis/redis:2.4.2 Redis/redis:2.2.15 Redis/redis:2.4.1 Redis/redis:2.4.0 Redis/redis:2.2.14 Redis/redis:2.4.0-rc8 Redis/redis:2.2.13 Redis/redis:2.4.0-rc7 Redis/redis:2.4.0-rc6 Redis/redis:2.4.0-rc5 Redis/redis:2.4.0-rc4 Redis/redis:2.2.12 Redis/redis:2.4.0-rc3 Redis/redis:with-deprecated-diskstore Redis/redis:2.2.111-scripting Redis/redis:2.2.11 Redis/redis:2.2.110-scripting Redis/redis:2.2.10 Redis/redis:2.2.9 Redis/redis:2.4.0-rc2 Redis/redis:2.4.0-rc1 Redis/redis:2.2.8 Redis/redis:2.2.107-scripting Redis/redis:2.2.7 Redis/redis:2.2.106-scripting Redis/redis:2.2.6 Redis/redis:2.2.105-scripting Redis/redis:2.2.5 Redis/redis:2.2.4 Redis/redis:2.2.3 Redis/redis:3.0-alpha0 Redis/redis:2.0.5 Redis/redis:2.2.2 Redis/redis:2.2.1 Redis/redis:2.2.0 Redis/redis:2.2.0-rc4 Redis/redis:2.3-alpha0 Redis/redis:2.2.0-rc3 Redis/redis:2.2.0-rc2 Redis/redis:2.2.0-rc1 Redis/redis:2.2-alpha6 Redis/redis:2.2-alpha5 Redis/redis:v2.0.4-stable Redis/redis:2.0.4 Redis/redis:2.2-alpha4 Redis/redis:2.2-alpha3 Redis/redis:v2.0.3-stable Redis/redis:2.0.3 Redis/redis:v2.0.2-stable Redis/redis:2.0.2 Redis/redis:2.2-alpha2 Redis/redis:v2.0.1-stable Redis/redis:2.0.1 Redis/redis:v2.0.0-stable Redis/redis:2.0.0 Redis/redis:2.2-alpha1 Redis/redis:twitter-20100825 Redis/redis:twitter-20100804 Redis/redis:2.2-alpha0 Redis/redis:v2.0.0-rc4 Redis/redis:v2.0.0-rc3 Redis/redis:v2.0.0-rc2 Redis/redis:v2.0.0-rc1b Redis/redis:v2.1.1-watch Redis/redis:v2.0.0-rc1 Redis/redis:v1.3.12 Redis/redis:v1.3.11 Redis/redis:v1.3.10 Redis/redis:v1.3.9 Redis/redis:v1.3.8 Redis/redis:v1.3.7 Redis/redis:1.3.6 Redis/redis:vm-playpen
...
compare: Redis/redis:7.0.6
Branches Tags
Redis/redis:unstable Redis/redis:7.2 Redis/redis:6.2 Redis/redis:7.4 Redis/redis:8.0
Redis/redis:7.2.10 Redis/redis:6.2.19 Redis/redis:7.4.5 Redis/redis:8.0.3 Redis/redis:8.2-rc1 Redis/redis:8.2-rc1-int Redis/redis:8.2-m01 Redis/redis:8.2-m01-int2 Redis/redis:8.2-m01-int Redis/redis:8.0.2 Redis/redis:7.2.9 Redis/redis:7.4.4 Redis/redis:8.0.1 Redis/redis:8.0.1-int Redis/redis:8.0.0 Redis/redis:8.0-rc2-int Redis/redis:7.4.3 Redis/redis:7.2.8 Redis/redis:6.2.18 Redis/redis:8.0-rc1 Redis/redis:8.0-rc1-int2 Redis/redis:8.0-rc1-int Redis/redis:8.0-m04 Redis/redis:8.0-m04-int Redis/redis:8.0-m03 Redis/redis:7.2.7 Redis/redis:7.4.2 Redis/redis:6.2.17 Redis/redis:8.0-m02 Redis/redis:6.2.16 Redis/redis:7.2.6 Redis/redis:7.4.1 Redis/redis:8.0-m01 Redis/redis:7.4.0 Redis/redis:7.4-rc2 Redis/redis:7.4-rc1 Redis/redis:7.2.5 Redis/redis:7.2.4 Redis/redis:7.0.15 Redis/redis:7.2.3 Redis/redis:7.2.2 Redis/redis:7.0.14 Redis/redis:6.2.15 Redis/redis:6.2.14 Redis/redis:7.2.1 Redis/redis:7.0.13 Redis/redis:7.2.0 Redis/redis:7.2-rc3 Redis/redis:7.0.12 Redis/redis:6.2.13 Redis/redis:6.0.20 Redis/redis:7.2-rc2 Redis/redis:6.0.19 Redis/redis:6.2.12 Redis/redis:7.0.11 Redis/redis:7.2-rc1 Redis/redis:7.0.10 Redis/redis:7.0.9 Redis/redis:6.2.11 Redis/redis:6.0.18 Redis/redis:6.2.10 Redis/redis:6.0.17 Redis/redis:6.2.9 Redis/redis:7.0.8 Redis/redis:7.0.7 Redis/redis:7.0.6 Redis/redis:6.2.8 Redis/redis:7.0.5 Redis/redis:7.0.4 Redis/redis:7.0.3 Redis/redis:7.0.2 Redis/redis:7.0.1 Redis/redis:7.0.0 Redis/redis:6.2.7 Redis/redis:7.0-rc3 Redis/redis:7.0-rc2 Redis/redis:7.0-rc1 Redis/redis:6.2.6 Redis/redis:6.0.16 Redis/redis:5.0.14 Redis/redis:6.2.5 Redis/redis:5.0.13 Redis/redis:6.0.15 Redis/redis:6.0.14 Redis/redis:6.2.4 Redis/redis:6.2.3 Redis/redis:6.0.13 Redis/redis:6.2.2 Redis/redis:6.2.1 Redis/redis:6.0.12 Redis/redis:5.0.12 Redis/redis:6.0.11 Redis/redis:6.2.0 Redis/redis:5.0.11 Redis/redis:6.2-rc3 Redis/redis:6.0.10 Redis/redis:6.2-rc2 Redis/redis:6.2-rc1 Redis/redis:6.0.9 Redis/redis:5.0.10 Redis/redis:6.0.8 Redis/redis:6.0.7 Redis/redis:6.0.6 Redis/redis:6.0.5 Redis/redis:6.0.4 Redis/redis:6.0.3 Redis/redis:6.0.2 Redis/redis:6.0.1 Redis/redis:6.0.0 Redis/redis:5.0.9 Redis/redis:6.0-rc4 Redis/redis:6.0-rc3 Redis/redis:5.0.8 Redis/redis:6.0-rc2 Redis/redis:6.0-rc1 Redis/redis:5.0.7 Redis/redis:5.0.6 Redis/redis:5.0.5 Redis/redis:3.2.13 Redis/redis:4.0.14 Redis/redis:5.0.4 Redis/redis:4.0.13 Redis/redis:5.0.3 Redis/redis:4.0.12 Redis/redis:5.0.2 Redis/redis:5.0.1 Redis/redis:5.0.0 Redis/redis:5.0-rc6 Redis/redis:5.0-rc5 Redis/redis:4.0.11 Redis/redis:5.0-rc4 Redis/redis:5.0-rc3 Redis/redis:5.0-rc2 Redis/redis:4.0.10 Redis/redis:3.2.12 Redis/redis:5.0-rc1 Redis/redis:4.0.9 Redis/redis:4.0.8 Redis/redis:4.0.7 Redis/redis:4.0.6 Redis/redis:4.0.5 Redis/redis:4.0.4 Redis/redis:4.0.3 Redis/redis:3.2.11 Redis/redis:4.0.2 Redis/redis:3.2.10 Redis/redis:4.0.1 Redis/redis:4.0.0 Redis/redis:3.2.9 Redis/redis:4.0-rc3 Redis/redis:3.2.8 Redis/redis:3.2.7 Redis/redis:3.2.6 Redis/redis:4.0-rc2 Redis/redis:4.0-rc1 Redis/redis:3.2.5 Redis/redis:3.2.4 Redis/redis:3.2.3 Redis/redis:3.2.2 Redis/redis:3.2.1 Redis/redis:3.2.0 Redis/redis:3.2.0-rc3 Redis/redis:3.0.7 Redis/redis:3.2.0-rc2 Redis/redis:3.2-rc1 Redis/redis:3.0.6 Redis/redis:2.8.24 Redis/redis:3.0.5 Redis/redis:2.8.23 Redis/redis:2.8.22 Redis/redis:3.0.4 Redis/redis:3.0.3 Redis/redis:3.0.2 Redis/redis:2.8.21 Redis/redis:2.8.20 Redis/redis:3.0.1 Redis/redis:3.0.0 Redis/redis:3.0.0-rc6 Redis/redis:3.0.0-rc5 Redis/redis:3.0.0-rc4 Redis/redis:3.0.0-rc3 Redis/redis:3.0.0-rc2 Redis/redis:2.8.19 Redis/redis:2.8.18 Redis/redis:3.0.0-rc1 Redis/redis:2.8.17 Redis/redis:2.8.16 Redis/redis:2.8.15 Redis/redis:2.8.14 Redis/redis:3.0.0-beta8 Redis/redis:2.8.13 Redis/redis:3.0.0-beta7 Redis/redis:2.8.12 Redis/redis:2.8.11 Redis/redis:3.0.0-beta6 Redis/redis:2.8.10 Redis/redis:3.0.0-beta5 Redis/redis:3.0.0-beta4 Redis/redis:3.0.0-beta3 Redis/redis:2.8.9 Redis/redis:2.8.8 Redis/redis:3.0.0-beta2 Redis/redis:2.8.7 Redis/redis:2.8.6 Redis/redis:3.0.0-beta1 Redis/redis:2.8.5 Redis/redis:2.8.4 Redis/redis:2.6.17 Redis/redis:2.8.3 Redis/redis:2.8.2 Redis/redis:2.8.1 Redis/redis:2.8.0 Redis/redis:2.8.0-rc6 Redis/redis:2.8.0-rc5 Redis/redis:2.8.0-rc4 Redis/redis:2.6.16 Redis/redis:2.6.15 Redis/redis:2.8.0-rc3 Redis/redis:2.8.0-rc2 Redis/redis:2.8.0-rc1 Redis/redis:2.6.14-2 Redis/redis:2.6.14-1 Redis/redis:2.6.14 Redis/redis:2.6.13 Redis/redis:2.6.12 Redis/redis:2.6.10-3 Redis/redis:2.6.10-2 Redis/redis:2.6.11 Redis/redis:2.6.10-1 Redis/redis:2.6.10 Redis/redis:2.6.9-1 Redis/redis:2.6.9 Redis/redis:2.6.8-1 Redis/redis:2.6.8 Redis/redis:2.6.7-1 Redis/redis:2.6.7 Redis/redis:2.6.6 Redis/redis:2.4.18 Redis/redis:2.6.5 Redis/redis:2.6.4 Redis/redis:2.6.3 Redis/redis:2.6.2 Redis/redis:2.6.1 Redis/redis:2.6.0 Redis/redis:2.6.0-rc8 Redis/redis:2.6.0-rc7 Redis/redis:2.4.17 Redis/redis:2.6.0-rc6 Redis/redis:2.4.16 Redis/redis:2.4.15 Redis/redis:2.6.0-rc5 Redis/redis:2.6.0-rc4 Redis/redis:2.4.14 Redis/redis:2.6.0-rc3 Redis/redis:2.6.0-rc2 Redis/redis:2.4.13 Redis/redis:2.4.12 Redis/redis:2.6.0-rc1 Redis/redis:2.4.11 Redis/redis:2.4.10 Redis/redis:2.4.9 Redis/redis:2.4.8 Redis/redis:2.4.7 Redis/redis:2.4.6 Redis/redis:2.4.5 Redis/redis:2.4.4 Redis/redis:2.4.3 Redis/redis:2.4.2 Redis/redis:2.2.15 Redis/redis:2.4.1 Redis/redis:2.4.0 Redis/redis:2.2.14 Redis/redis:2.4.0-rc8 Redis/redis:2.2.13 Redis/redis:2.4.0-rc7 Redis/redis:2.4.0-rc6 Redis/redis:2.4.0-rc5 Redis/redis:2.4.0-rc4 Redis/redis:2.2.12 Redis/redis:2.4.0-rc3 Redis/redis:with-deprecated-diskstore Redis/redis:2.2.111-scripting Redis/redis:2.2.11 Redis/redis:2.2.110-scripting Redis/redis:2.2.10 Redis/redis:2.2.9 Redis/redis:2.4.0-rc2 Redis/redis:2.4.0-rc1 Redis/redis:2.2.8 Redis/redis:2.2.107-scripting Redis/redis:2.2.7 Redis/redis:2.2.106-scripting Redis/redis:2.2.6 Redis/redis:2.2.105-scripting Redis/redis:2.2.5 Redis/redis:2.2.4 Redis/redis:2.2.3 Redis/redis:3.0-alpha0 Redis/redis:2.0.5 Redis/redis:2.2.2 Redis/redis:2.2.1 Redis/redis:2.2.0 Redis/redis:2.2.0-rc4 Redis/redis:2.3-alpha0 Redis/redis:2.2.0-rc3 Redis/redis:2.2.0-rc2 Redis/redis:2.2.0-rc1 Redis/redis:2.2-alpha6 Redis/redis:2.2-alpha5 Redis/redis:v2.0.4-stable Redis/redis:2.0.4 Redis/redis:2.2-alpha4 Redis/redis:2.2-alpha3 Redis/redis:v2.0.3-stable Redis/redis:2.0.3 Redis/redis:v2.0.2-stable Redis/redis:2.0.2 Redis/redis:2.2-alpha2 Redis/redis:v2.0.1-stable Redis/redis:2.0.1 Redis/redis:v2.0.0-stable Redis/redis:2.0.0 Redis/redis:2.2-alpha1 Redis/redis:twitter-20100825 Redis/redis:twitter-20100804 Redis/redis:2.2-alpha0 Redis/redis:v2.0.0-rc4 Redis/redis:v2.0.0-rc3 Redis/redis:v2.0.0-rc2 Redis/redis:v2.0.0-rc1b Redis/redis:v2.1.1-watch Redis/redis:v2.0.0-rc1 Redis/redis:v1.3.12 Redis/redis:v1.3.11 Redis/redis:v1.3.10 Redis/redis:v1.3.9 Redis/redis:v1.3.8 Redis/redis:v1.3.7 Redis/redis:1.3.6 Redis/redis:vm-playpen
1155 Commits
6.2.19 ... 7.0.6
Author SHA1 Message Date
Oran Agra c0924a8361 Redis 7.0.6
CI / test-sanitizer-address (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
2022-12-12 17:36:34 +02:00
Binbin e2665c6fc0 Fix replication on expired key test timing issue, give it more chances (#11548) 
In replica, the key expired before master's `INCR` was arrived, so INCR
creates a new key in the replica and the test failed.
```
*** [err]: Replication of an expired key does not delete the expired key in tests/integration/replication-4.tcl
Expected '0' to be equal to '1' (context: type eval line 13 cmd {assert_equal 0 [$slave exists k]} proc ::test)
```

This test is very likely to do a false positive if the `wait_for_ofs_sync`
takes longer than the expiration time, so give it a few more chances.

The test was introduced in #9572.

(cherry picked from commit 06b577aad0)
 2022-12-12 17:36:34 +02:00
Binbin 3c525fab6a Fix redis-cli cluster add-node race in cli.tcl (#11349) 
There is a race condition in the test:
```
*** [err]: redis-cli --cluster add-node with cluster-port in tests/unit/cluster/cli.tcl
Expected '5' to be equal to '4' {assert_equal 5 [CI 0 cluster_known_nodes]} proc ::test)
```

When using cli to add node, there can potentially be a race condition
in which all nodes presenting cluster state o.k even though the added
node did not yet meet all cluster nodes.

This comment and the fix were taken from #11221. Also apply it in several
other similar places.

(cherry picked from commit a549b78c48)
 2022-12-12 17:36:34 +02:00
ranshid e1557e6c7a fix test Migrate the last slot away from a node using redis-cli (#11221) 
When using cli to add node, there can potentially be a race condition in
which all nodes presenting cluster state o.k even though the added node
did not yet meet all cluster nodes.
this adds another utility function to wait until all cluster nodes see the same cluster size

(cherry picked from commit c0ce97facc)
 2022-12-12 17:36:34 +02:00
Binbin b5784feabd Fix timing issue in cluster test (#11008) 
A timing issue like this was reported in freebsd daily CI:
```
*** [err]: Sanity test push cmd after resharding in tests/unit/cluster/cli.tcl
Expected 'CLUSTERDOWN The cluster is down' to match '*MOVED*'
```

We additionally wait for each node to reach a consensus on the cluster
state in wait_for_condition to avoid the cluster down error.

The fix just like #10495, quoting madolson's comment:
Cluster check just verifies the the config state is self-consistent,
waiting for cluster_state to be okay is an independent check that all
the nodes actually believe each other are healthy.

At the same time i noticed that unit/moduleapi/cluster.tcl has an exact
same test, may have the same problem, also modified it.

(cherry picked from commit 5ce64ab010)
 2022-12-12 17:36:34 +02:00
Binbin a43c51b297 Fix CLUSTERDOWN issue in cluster reshard unblock test (#11139) 
change the cluster-node-timeout from 1 to 1000

(cherry picked from commit 3a16ad30b7)
 2022-12-12 17:36:34 +02:00
David CARLIER d86408b7b9 Fixes build warning when CACHE_LINE_SIZE is already defined. (#11389) 
* Fixes build warning when CACHE_LINE_SIZE is already defined
* Fixes wrong CACHE_LINE_SIZE on some FreeBSD systems where it could be set to 128 (e.g. on MIPS)
* Fixes wrong CACHE_LINE_SIZE on Apple M1 (use 128 instead of 64)

Wrong cache line size in that case can some false sharing of array elements between threads, see #10892

(cherry picked from commit 871cc200a0)
 2022-12-12 17:36:34 +02:00
Oran Agra dca63da432 Avoid ASAN test errors on crash report tests 
Clang Address Sanitizer tests started reporting unknown-crash on these
tests due to the memcheck, disable the memcheck to avoid that noise.

(cherry picked from commit 18ff6a3269a34b9bfe549b34bda9d83c3eae7e2a)
 2022-12-12 17:36:34 +02:00
Oran Agra 4a5aba16c9 Try to fix a race in psync2 test (#11553) 
This test sets the master ping interval to 1 hour, in order to avoid
pings in the replicatoin stream incrementing the replication offset,
however, it didn't increase the repl-timeout so on slow machines
where the test took more than 60 seconds, the replicas would drop
and reconnect.

```
*** [err]: PSYNC2: Partial resync after restart using RDB aux fields in tests/integration/psync2.tcl
Replica didn't partial sync
```

The test would detect 4 additional partial syncs where it expects
only one.

(cherry picked from commit b0250b4508)
 2022-12-12 17:36:34 +02:00
Binbin 7275935641 Bump vmactions/freebsd-vm to 0.3.0 to fix FreeBSD daily (#11476) 
Our FreeBSD daily has been failing recently:
```
  Config file: freebsd-13.1.conf
  cd: /Users/runner/work/redis/redis: No such file or directory
  gmake: *** No targets specified and no makefile found.  Stop.
```

Upgrade vmactions/freebsd-vm to the latest version (0.3.0) can work.
I've tested it, but don't know why, but first let's fix it.

(cherry picked from commit 5246bf4544)
 2022-12-12 17:36:34 +02:00
Binbin 98410d0da5 Fix bgsaveerr issue in psync wrong offset test (#11043) 
The kill above is sometimes successful and sometimes already too late.
The PING in pysnc wrong offset test got rejected by bgsaveerr because
lastbgsave_status is C_ERR.

In theory, using diskless can avoid PING being affected, because when
the replica is dropped, we will kill the child with SIGUSR1, and this
will not affect lastbgsave_status.

Anyway, this kill is not particularly needed here, dropping the kill
is the best one, since we do have the waitForBgsave, so just let it
take care of the bgsave. No need for fast termination.

(cherry picked from commit e7144693e2)
 2022-12-12 17:36:34 +02:00
filipe oliveira 9ccb8dfd47 Reduce rewriteClientCommandVector usage on EXPIRE command (#11602) 
There is overhead on Redis 7.0 EXPIRE command that is not present on 6.2.7.

We could see that on the unstable profile there are around 7% of CPU cycles
spent on rewriteClientCommandVector that are not present on 6.2.7.
This was introduced in #8474.
This PR reduces the overhead by using 2X rewriteClientCommandArgument instead of
rewriteClientCommandVector. In this scenario rewriteClientCommandVector creates 4 arguments.
the above usage of rewriteClientCommandArgument reduces the overhead in half.

This PR should also improve PEXPIREAT performance by avoiding at all
rewriteClientCommandArgument usage.

Co-authored-by: Oran Agra <oran@redislabs.com>
(cherry picked from commit c3fb48da8b)
 2022-12-12 17:36:34 +02:00
Harkrishn Patro 05c9378b3f Optimize client memory usage tracking operation while client eviction is disabled (#11348) 
## Issue
During the client input/output buffer processing, the memory usage is
incrementally updated to keep track of clients going beyond a certain
threshold `maxmemory-clients` to be evicted. However, this additional
tracking activity leads to unnecessary CPU cycles wasted when no
client-eviction is required. It is applicable in two cases.

* `maxmemory-clients` is set to `0` which equates to no client eviction
  (applicable to all clients)
* `CLIENT NO-EVICT` flag is set to `ON` which equates to a particular
  client not applicable for eviction.

## Solution
* Disable client memory usage tracking during the read/write flow when
  `maxmemory-clients` is set to `0` or `client no-evict` is `on`.
  The memory usage is tracked only during the `clientCron` i.e. it gets
  periodically updated.
* Cleanup the clients from the memory usage bucket when client eviction
  is disabled.
* When the maxmemory-clients config is enabled or disabled at runtime,
  we immediately update the memory usage buckets for all clients (tested
  scanning 80000 took some 20ms)

Benchmark shown that this can improve performance by about 5% in
certain situations.

Co-authored-by: Oran Agra <oran@redislabs.com>
(cherry picked from commit c0267b3fa5)
 2022-12-12 17:36:34 +02:00
Binbin b322a77e99 Fix command line startup --sentinel problem (#11591) 
There is a issue with --sentinel:
```
[root]# src/redis-server sentinel.conf --sentinel --loglevel verbose

*** FATAL CONFIG FILE ERROR (Redis 255.255.255) ***
Reading the configuration file, at line 352
>>> 'sentinel "--loglevel" "verbose"'
Unrecognized sentinel configuration statement
```

This is because in #10660 (Redis 7.0.1), `--` prefix change break it.
In this PR, we will handle `--sentinel` the same as we did for `--save`
in #10866. i.e. it's a pseudo config option with no value.

(cherry picked from commit 8f13ac10b4)
 2022-12-12 17:36:34 +02:00
filipe oliveira 5e02338e5e GEOSEARCH BYBOX: Simplified haversine distance formula when longitude diff is 0 (#11579) 
This is take 2 of `GEOSEARCH BYBOX` optimizations based on haversine
distance formula when longitude diff is 0.
The first one was in #11535 .

- Given longitude diff is 0 the asin(sqrt(a)) on the haversine is asin(sin(abs(u))).
- arcsin(sin(x)) equal to x when x ∈[−𝜋/2,𝜋/2].
- Given latitude is between [−𝜋/2,𝜋/2] we can simplifiy arcsin(sin(x)) to x.

On the sample dataset with 60M datapoints, we've measured 55% increase
in the achievable ops/sec.

(cherry picked from commit e48ac075c0)
 2022-12-12 17:36:34 +02:00
filipe oliveira 6a6d8806a4 Reintroduce lua argument cache in luaRedisGenericCommand removed in v7.0 (#11541) 
This mechanism aims to reduce calls to malloc and free when
preparing the arguments the script sends to redis commands.
This is a mechanism was originally implemented in 48c49c4
and 4f68655, and was removed in #10220 (thinking it's not needed
and that it has no impact), but it now turns out it was wrong, and it
indeed provides some 5% performance improvement.

The implementation is a little bit too simplistic, it assumes consecutive
calls use the same size in the same arg index, but that's arguably
sufficient since it's only aimed at caching very small things.

We could even consider always pre-allocating args to the full
LUA_CMD_OBJCACHE_MAX_LEN (64 bytes) rather than the right size for the argument,
that would increase the chance they'll be able to be re-used.
But in some way this is already happening since we're using
sdsalloc, which in turn uses s_malloc_usable and takes ownership
of the full side of the allocation, so we are padded to the allocator
bucket size.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: sundb <sundbcn@gmail.com>
(cherry picked from commit 2d80cd7840)
 2022-12-12 17:36:34 +02:00
filipe oliveira 77570e3965 Speedup GEODIST with fixedpoint_d2string as an optimized version of snprintf %.4f (#11552) 
GEODIST used snprintf("%.4f") for the reply using addReplyDoubleDistance,
which was slow. This PR optimizes it without breaking compatibility by following
the approach of ll2string with some changes to match the use case of distance
and precision. I.e. we multiply it by 10000 format it as an integer, and then add
a decimal point. This can achieve about 35% increase in the achievable ops/sec.

Co-authored-by: Oran Agra <oran@redislabs.com>
(cherry picked from commit 61c85a2b20)
 2022-12-12 17:36:34 +02:00
Yossi Gottlieb 60ef6d2896 Improve TLS error handling. (#11563) 
* Remove duplicate code, propagating SSL errors into connection state.
* Add missing error handling in synchronous IO functions.
* Fix connection error reporting in some replication flows.

(cherry picked from commit 155acef51a)
 2022-12-12 17:36:34 +02:00
filipe oliveira 669a8dba37 changing addReplySds and sdscat to addReplyStatusLength() within luaReplyToRedisReply() (#11556) 
profiling EVALSHA\ we see that luaReplyToRedisReply takes 8.73% out of the
56.90% of luaCallFunction CPU cycles.

Using addReplyStatusLength instead of directly composing the protocol to avoid
sdscatprintf and addReplySds ( which imply multiple sdslen calls ).

The new approach drops
luaReplyToRedisReply CPU cycles to 3.77%

(cherry picked from commit 68e87eb088)
 2022-12-12 17:36:34 +02:00
filipe oliveira 7c4f6e179b Reduce eval related overhead introduced in v7.0 by evalCalcFunctionName (#11521) 
As being discussed in #10981 we see a degradation in performance
between v6.2 and v7.0 of Redis on the EVAL command.

After profiling the current unstable branch we can see that we call the
expensive function evalCalcFunctionName twice.

The current "fix" is to basically avoid calling evalCalcFunctionName and
even dictFind(lua_scripts) twice for the same command.
Instead we cache the current script's dictEntry (for both Eval and Functions)
in the current client so we don't have to repeat these calls.
The exception would be when doing an EVAL on a new script that's not yet
in the script cache. in that case we will call evalCalcFunctionName (and even
evalExtractShebangFlags) twice.

Co-authored-by: Oran Agra <oran@redislabs.com>
(cherry picked from commit 7dfd7b9197)
 2022-12-12 17:36:34 +02:00
zhaozhao.zz fba49b107d benchmark getRedisConfig exit only when meet NOAUTH error (#11096) 
redis-benchmark: when trying to get the CONFIG before benchmark,
avoid printing any warning on most errors (e.g. NOPERM error).
avoid aborting the benchmark on NOPERM.
keep the warning only when we abort the benchmark on a NOAUTH error

(cherry picked from commit f0005b5328)
 2022-12-12 17:36:34 +02:00
filipe oliveira 3d206f0fcf Simplified geoAppendIfWithinShape() and removed spurious calls do sdsdup and sdsfree (#11522) 
In scenarios in which we have large datasets and the elements are not
contained within the range we do spurious calls do sdsdup and sdsfree.
I.e. instead of pre-creating an sds before we know if we're gonna use it
or not, change the role of geoAppendIfWithinShape to just do geoWithinShape,
and let the caller create the string only when needed.

Co-authored-by: Oran Agra <oran@redislabs.com>
(cherry picked from commit 376b689b03)
 2022-12-12 17:36:34 +02:00
DevineLiu 81e72e8e29 [BUG] Fix announced ports not updating on local node when updated at runtime (#10745) 
The cluster-announce-port/cluster-announce-bus-port/cluster-announce-tls-port should take effect at runtime

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
(cherry picked from commit 25ffa79b64)
 2022-12-12 17:36:34 +02:00
filipe oliveira cffabb641f GEOSEARCH BYBOX: Reduce wastefull computation on geohashGetDistanceIfInRectangle and geohashGetDistance (#11535) 
Optimize geohashGetDistanceIfInRectangle when there are many misses.
It calls 3x geohashGetDistance. The first 2 times we call them to produce intermediate results.
This PR focus on optimizing for those 2 intermediate results.

1 Reduce expensive computation on intermediate geohashGetDistance with same long
2 Avoid expensive lon_distance calculation if lat_distance fails beforehand

Co-authored-by: Oran Agra <oran@redislabs.com>
(cherry picked from commit ae1de54900)
 2022-12-12 17:36:34 +02:00
sundb 4972f5768c Ignore -Wstringop-overread warning for SHA1Transform() on GCC 12 (#11538) 
Fix compile warning for SHA1Transform() method under alpine with GCC 12.

Warning:
```
In function 'SHA1Update',
    inlined from 'SHA1Final' at sha1.c:187:9:
sha1.c:144:13: error: 'SHA1Transform' reading 64 bytes from a region of size 0 [-Werror=stringop-overread]
  144 |             SHA1Transform(context->state, &data[i]);
      |             ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
sha1.c:144:13: note: referencing argument 2 of type 'const unsigned char[64]'
sha1.c: In function 'SHA1Final':
sha1.c:56:6: note: in a call to function 'SHA1Transform'
   56 | void SHA1Transform(uint32_t state[5], const unsigned char buffer[64])
      |      ^~~~~~~~~~~~~
```

This warning is a false positive because it has been determined in the loop judgment that there must be 64 chars after position `i`
```c
for ( ; i + 63 < len; i += 64) {
    SHA1Transform(context->state, &data[i]);
}
```

Reference: https://github.com/libevent/libevent/commit/e1d7d3e40a7fd50348d849046fbfd9bf976e643c
(cherry picked from commit fd80818552)
 2022-12-12 17:36:34 +02:00
Wen Hui d81c2a5f33 Add explicit error log message for AOF_TRUNCATED status when server load AOF file (#11484) 
Now, according to the comments, if the truncated file is not the last file,
it will be considered as a fatal error.
And the return code will updated to AOF_FAILED, then server will exit
without any error message to the client.

Similar to other error situations, this PR add an explicit error message
for this case and make the client know clearly what happens.

(cherry picked from commit 6e9724cb6a)
 2022-12-12 17:36:34 +02:00
Madelyn Olson a26ac7ebbc Explicitly send function commands to monitor (#11510) 
Both functions and eval are marked as "no-monitor", since we want to explicitly feed in the script command before the commands generated by the script. Note that we want this behavior generally, so that commands can redact arguments before being added to the monitor.

(cherry picked from commit d136bf2830)
 2022-12-12 17:36:34 +02:00
uriyage 7ad786db2e Module CLIENT_CHANGE, Fix crash on free blocked client with DB!=0 (#11500) 
In moduleFireServerEvent we change the real client DB to 0 on freeClient in case the event is REDISMODULE_EVENT_CLIENT_CHANGE.
It results in a crash if the client is blocked on a key on other than DB 0.

The DB change is not necessary even for module-client, as we set its DB to 0 on either createClient or moduleReleaseTempClient.

Co-authored-by: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
(cherry picked from commit e4eb18b303)
 2022-12-12 17:36:34 +02:00
Oran Agra abf83bc682 fixes for fork child exit and test: #11463 (#11499) 
Fix a few issues with the recent #11463
* use exitFromChild instead of exit
* test should ignore defunct process since that's what we expect to
  happen for thees child processes when the parent dies.
* fix typo

Co-authored-by: Binbin <binloveplay1314@qq.com>
(cherry picked from commit 4c54528f0f)
 2022-12-12 17:36:34 +02:00
Oran Agra 7c956d5cdf diskless master, avoid bgsave child hung when fork parent crashes (#11463) 
During a diskless sync, if the master main process crashes, the child would
have hung in `write`. This fix closes the read fd on the child side, so that if the
parent crashes, the child will get a write error and exit.

This change also fixes disk-based replication, BGSAVE and AOFRW.
In that case the child wouldn't have been hang, it would have just kept
running until done which may be pointless.

There is a certain degree of risk here. in case there's a BGSAVE child that could
maybe succeed and the parent dies for some reason, the old code would have let
the child keep running and maybe succeed and avoid data loss.
On the other hand, if the parent is restarted, it would have loaded an old rdb file
(or none), and then the child could reach the end and rename the rdb file (data
conflicting with what the parent has), or also have a race with another BGSAVE
child that the new parent started.

Note that i removed a comment saying a write error will be ignored in the child
and handled by the parent (this comment was very old and i don't think relevant).

(cherry picked from commit ccaef5c923)
 2022-12-12 17:36:34 +02:00
Moti Cohen 19d01b62ca Fix sentinel function that compares hostnames (if failed resolve) (#11419) 
Funcion sentinelAddrEqualsHostname() of sentinel makes DNS resolve
and based on it determines if two IP addresses are equal. Now, If the
DNS resolve command fails, the function simply returns 0, even if the
hostnames are identical.

This might become an issue in case of failover such that sentinel might
receives from Redis instance, response to regular INFO query it sent,
and wrongly decide that the instance is pointing to is different leader
than the one recorded because of this function, yet hostnames are
identical. In turn sentinel disconnects the connection between sentinel
and valid slave which leads to -failover-abort-no-good-slave.
See issue #11241.

I managed to reproduce only part of the flow in which the function
return wrong result and trigger +fix-slave-config.

The fix is even if the function failed to resolve then compare based on
hostnames. That is our best effort as long as the server is unavailable
for some reason. It is fine since Redis instance cannot have multiple
hostnames for a given setup

(cherry picked from commit bd23b15ad7)
 2022-12-12 17:36:34 +02:00
sundb 9fc20f4f9d Fix crash due to to reuse iterator entry after list deletion in module (#11383) 
In the module, we will reuse the list iterator entry for RM_ListDelete, but `listTypeDelete` will only update
`quicklistEntry->zi` but not `quicklistEntry->node`, which will result in `quicklistEntry->node` pointing to
a freed memory address if the quicklist node is deleted.

This PR sync `key->u.list.index` and `key->u.list.entry` to list iterator after `RM_ListDelete`.

This PR also optimizes the release code of the original list iterator.

Co-authored-by: Viktor Söderqvist <viktor@zuiderkwast.se>
(cherry picked from commit 6dd213558b)
 2022-12-12 17:36:34 +02:00
C Charles f95af7785b MIGTATE with AUTH that contains "keys" is getting wrong key names in migrateGetKeys, leads to ACL errors (#11253) 
When using the MIGRATE, with a destination Redis that has the user name or password set to the string "keys",
Redis would have determine the wrong set of key names the command is gonna access.
This lead to ACL returning wrong authentication result.

Destination instance:
```
127.0.0.1:6380> acl setuser default >keys
OK
127.0.0.1:6380> acl setuser keys on nopass ~* &* +@all
OK
```

Source instance:
```
127.0.0.1:6379> set a 123
OK
127.0.0.1:6379> acl setuser cc on nopass ~a* +@all
OK
127.0.0.1:6379> auth cc 1
OK
127.0.0.1:6379> migrate 127.0.0.1 6380 "" 0 1000 auth keys keys a
(error) NOPERM this user has no permissions to access one of the keys used as arguments
127.0.0.1:6379> migrate 127.0.0.1 6380 "" 0 1000 auth2 keys pswd keys a
(error) NOPERM this user has no permissions to access one of the keys used as arguments
```

Using `acl dryrun` we know that the parameters of `auth` and `auth2` are mistaken for the `keys` option.
```
127.0.0.1:6379> acl dryrun cc migrate whatever whatever "" 0 1000 auth keys keys a
"This user has no permissions to access the 'keys' key"
127.0.0.1:6379> acl dryrun cc migrate whatever whatever "" 0 1000 auth2 keys pswd keys a
"This user has no permissions to access the 'pswd' key"
```

Fix the bug by editing db.c/migrateGetKeys function, which finds the `keys` option and all the keys following.

(cherry picked from commit 9ab873d9d3)
 2022-12-12 17:36:34 +02:00
Oran Agra 92ad0b5c8b Improve linux overcommit check and warning (#11357) 
1. show the overcommit warning when overcommit is disabled (2),
   not just when it is set to heuristic (0).
2. improve warning text to mention the issue with jemalloc causing VM
   mapping fragmentation when set to 2.

(cherry picked from commit dd60c6c8d3)
 2022-12-12 17:36:34 +02:00
Meir Shpilraien (Spielrein) 6cf24fa437 RedisModule_ResetDataset should not clear the functions. (#11268) 
As mentioned on docs, `RM_ResetDataset` Performs similar operation to FLUSHALL.
As FLUSHALL do not clean the function, `RM_ResetDataset` should not clean the functions
as well.

(cherry picked from commit d2ad01ab3e)
 2022-12-12 17:36:34 +02:00
Meir Shpilraien (Spielrein) b1ede2120a Avoid crash on crash report when a bad function pointer was called (#11298) 
If Redis crashes due to calling an invalid function pointer,
the `backtrace` function will try to dereference this invalid pointer
which will cause a crash inside the crash report and will kill
the processes without having all the crash report information.

Example:

```
=== REDIS BUG REPORT START: Cut & paste starting from here ===
198672:M 19 Sep 2022 18:06:12.936 # Redis 255.255.255 crashed by signal: 11, si_code: 1
198672:M 19 Sep 2022 18:06:12.936 # Accessing address: 0x1
198672:M 19 Sep 2022 18:06:12.936 # Crashed running the instruction at: 0x1
// here the processes is crashing
```

This PR tries to fix this crash be:
1. Identify the issue when it happened.
2. Replace the invalid pointer with a pointer to some dummy function
   so that `backtrace` will not crash.

I identification is done by comparing `eip` to `info->si_addr`, if they
are the same we know that the crash happened on the same address it tries to
accesses and we can conclude that it tries to call and invalid function pointer.

To replace the invalid pointer we introduce a new function, `setMcontextEip`,
which is very similar to `getMcontextEip` and it knows to set the Eip for the
different supported OS's. After printing the trace we retrieve the old `Eip` value.

(cherry picked from commit 0bf90d9443)
 2022-12-12 17:36:34 +02:00
guybe7 5b2119c639 RM_CreateCommand should not set CMD_KEY_VARIABLE_FLAGS automatically (#11320) 
The original idea behind auto-setting the default (first,last,step) spec was to use
the most "open" flags when the user didn't provide any key-spec flags information.

While the above idea is a good approach, it really makes no sense to set
CMD_KEY_VARIABLE_FLAGS if the user didn't provide the getkeys-api flag:
in this case there's not way to retrieve these variable flags, so what's the point?

Internally in redis there was code to ignore this already, so this fix doesn't change
redis's behavior, it only affects the output of COMMAND command.

(cherry picked from commit 3330ea1864)
 2022-12-12 17:36:34 +02:00
Steffen Moser 929ab58a12 Fixing compilation by removing flock() when compiling on Solaris (#11327) 
SunOS/Solaris and its relatives don't support the flock() function.
While "redis" has been excluding setting up the lock using flock() on the cluster
configuration file when compiling under Solaris, it was still using flock() in the
unlock call while shutting down.

This pull request eliminates the flock() call also in the unlocking stage
for Oracle Solaris and its relatives.

Fix compilation regression from #10912

(cherry picked from commit 6aab4cb736)
 2022-12-12 17:36:34 +02:00
Binbin 79414d45fb Fix CLUSTER SHARDS showing empty hostname (#11297) 
* Fix CLUSTER SHARDS showing empty hostname

In #10290, we changed clusterNode hostname from `char*`
to `sds`, and the old `node->hostname` was changed to
`sdslen(node->hostname)!=0`.

But in `addNodeDetailsToShardReply` it is missing.
It results in the return of an empty string hostname
in CLUSTER SHARDS command if it unavailable.

Like this (note that we listed it as optional in the doc):
```
 9) "hostname"
10) ""
```

(cherry picked from commit 1de675b3d5)
 2022-12-12 17:36:34 +02:00
Shaya Potter b8fcd322c6 Add RM_SetContextUser to support acl validation in RM_Call (and scripts) (#10966) 
Adds a number of user management/ACL validaiton/command execution functions to improve a
Redis module's ability to enforce ACLs correctly and easily.

* RM_SetContextUser - sets a RedisModuleUser on the context, which RM_Call will use to both
  validate ACLs (if requested and set) as well as assign to the client so that scripts executed via
  RM_Call will have proper ACL validation.
* RM_SetModuleUserACLString - Enables one to pass an entire ACL string, not just a single OP
  and have it applied to the user
* RM_GetModuleUserACLString - returns a stringified version of the user's ACL (same format as dump
  and list).  Contains an optimization to cache the stringified version until the underlying ACL is modified.
* Slightly re-purpose the "C" flag to RM_Call from just being about ACL check before calling the
  command, to actually running the command with the right user, so that it also affects commands
  inside EVAL scripts. see #11231

(cherry picked from commit 6e993a5dfa)
 2022-12-12 17:36:34 +02:00
Valentino Geron d279ec2532 Replica that asks for rdb only should be closed right after the rdb part (#11296) 
The bug is that the the server keeps on sending newlines to the client.
As a result, the receiver might not find the EOF marker since it searches
for it only on the end of each payload it reads from the socket.
The but only affects `CLIENT_REPL_RDBONLY`.
This affects `redis-cli --rdb` (depending on timing)

The fixed consist of two steps:
1. The `CLIENT_REPL_RDBONLY` should be closed ASAP (we cannot
   always call to `freeClient` so we use `freeClientAsync`)
2. Add new replication state `SLAVE_STATE_RDB_TRANSMITTED`

(cherry picked from commit e53bf65245)
 2022-12-12 17:36:34 +02:00
Ariel Shtul a221fc85ed [PERF] use snprintf once in addReplyDouble (#11093) 
The previous implementation calls `snprintf` twice, the second time used to
'memcpy' the output of the first, which could be a very large string.
The new implementation reserves space for the protocol header ahead
of the formatted double, and then prepends the string length ahead of it.

Measured improvement of simple ZADD of some 25%.

(cherry picked from commit 90223759a3)
 2022-12-12 17:36:34 +02:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 1571907ea0 Bump vmactions/freebsd-vm from 0.2.3 to 0.2.4 (#11203)
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details 
Bumps [vmactions/freebsd-vm](https://github.com/vmactions/freebsd-vm) from 0.2.3 to 0.2.4.
- [Release notes](https://github.com/vmactions/freebsd-vm/releases)
- [Commits](https://github.com/vmactions/freebsd-vm/compare/v0.2.3...v0.2.4)

---
updated-dependencies:
- dependency-name: vmactions/freebsd-vm
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

(cherry picked from commit c66eaf4e4a)
 2022-09-21 22:42:01 +03:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> a06b68b917 Bump vmactions/freebsd-vm from 0.2.0 to 0.2.3 (#11072) 
Bumps [vmactions/freebsd-vm](https://github.com/vmactions/freebsd-vm) from 0.2.0 to 0.2.3.
- [Release notes](https://github.com/vmactions/freebsd-vm/releases)
- [Commits](https://github.com/vmactions/freebsd-vm/compare/v0.2.0...v0.2.3)

---
updated-dependencies:
- dependency-name: vmactions/freebsd-vm
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

(cherry picked from commit 4fe9242a7f)
 2022-09-21 22:42:01 +03:00
Yossi Gottlieb 7c0d160536 CI: Update vmaction. (#11013) 
(cherry picked from commit b550a55cbf)
 2022-09-21 22:42:01 +03:00
Oran Agra b2a250ce15 Redis 7.0.5 2022-09-21 22:42:01 +03:00
Oran Agra fa6815e14e Fix heap overflow vulnerability in XAUTOCLAIM (CVE-2022-35951) 
Executing an XAUTOCLAIM command on a stream key in a specific state, with a
specially crafted COUNT argument may cause an integer overflow, a subsequent
heap overflow, and potentially lead to remote code execution.
The problem affects Redis versions 7.0.0 or newer.
 2022-09-21 22:42:01 +03:00
Shaya Potter a1ec0caed0 Improve cmd_flags for script/functions in RM_Call (#11159) 
When RM_Call was used with `M` (reject OOM), `W` (reject writes),
as well as `S` (rejecting stale or write commands in "Script mode"),
it would have only checked the command flags, but not the declared
script flag in case it's a command that runs a script.

Refactoring: extracts out similar code in server.c's processCommand
to be usable in RM_Call as well.

(cherry picked from commit bed6d759bc)
 2022-09-21 22:42:01 +03:00
Madelyn Olson b8beda61a2 Cluster test infra (taken from #10920) 
* Taking just the test infrastrucutre from that commit.

(cherry picked from commit 8a4e3bcd8d)
 2022-09-21 22:42:01 +03:00
Shay Fadida c9eabbf9b1 Fix missing sections for INFO ALL with module (#11291) 
When using `INFO ALL <section>`, when `section` is a specific module section. 
Redis will not print the additional section(s).

The fix in this case, will search the modules info sections if the user provided additional sections to `ALL`.

Co-authored-by: Oran Agra <oran@redislabs.com>
(cherry picked from commit eedb8b1724)
 2022-09-21 22:42:01 +03:00
Binbin 842fd1fbdf Fix Invalid node address specified in redis-cli --cluster create/add-node (#11151) 
This bug was introduced in #10344 (7.0.3), and it breaks the
redis-cli --cluster create usage in #10436 (7.0 RC3).

At the same time, the cluster-port support introduced in #10344
cannot use the DNS lookup brought by #10436.

(cherry picked from commit c2b0c13d5c)
 2022-09-21 22:42:01 +03:00
sundb 01358df34e Fix crash due to delete entry from compress quicklistNode and wrongly split quicklistNode (#11242) 
This PR mainly deals with 2 crashes introduced in #9357,
and fix the QUICKLIST-PACKED-THRESHOLD mess in external test mode.

1. Fix crash due to deleting an entry from a compress quicklistNode
   When inserting a large element, we need to create a new quicklistNode first,
   and then delete its previous element, if the node where the deleted element is
   located is compressed, it will cause a crash.
   Now add `dont_compress` to quicklistNode, if we want to use a quicklistNode
   after some operation, we can use this flag like following:

    ```c
    node->dont_compress = 1; /* Prevent to be compressed */
    some_operation(node); /* This operation might try to compress this node */
    some_other_operation(node); /* We can use this node without decompress it */
    node->dont_compress = 0; /* Re-able compression */
    quicklistCompressNode(node);
    ```

   Perhaps in the future, we could just disable the current entry from being
   compressed during the iterator loop, but that would require more work.

2. Fix crash due to wrongly split quicklist
   before #9357, the offset param of _quicklistSplitNode() will not negative.
   For now, when offset is negative, the split extent will be wrong.
   following example:
    ```c
    int orig_start = after ? offset + 1 : 0;
    int orig_extent = after ? -1 : offset;
    int new_start = after ? 0 : offset;
    int new_extent = after ? offset + 1 : -1;
    # offset: -2, after: 1, node->count: 2
    # current wrong range: [-1,-1] [0,-1]
    # correct range: [1,-1] [0, 1]
    ```

   Because only `_quicklistInsert()` splits the quicklistNode and only
   `quicklistInsertAfter()`, `quicklistInsertBefore()` call _quicklistInsert(), 
   so `quicklistReplaceEntry()` and `listTypeInsert()` might occur this crash.
   But the iterator of `listTypeInsert()` is alway from head to tail(iter->offset is
   always positive), so it is not affected.
   The final conclusion is this crash only occur when we insert a large element
   with negative index into a list, that affects `LSET` command and `RM_ListSet`
   module api.
     
3. In external test mode, we need to restore quicklist packed threshold after
   when the end of test.
4. Show `node->count` in quicklistRepr().
5. Add new tcl proc `config_get_set` to support restoring config in tests.

(cherry picked from commit 13d25dd95e)
 2022-09-21 22:42:01 +03:00
zhaozhao.zz 2f6537dc7f fix infinite sleep in performEvictions when have lazyfree jobs (#11237) 
This bug is introduced in #7653. (Redis 6.2.0)

When `server.maxmemory_eviction_tenacity` is 100, `eviction_time_limit_us` is
`ULONG_MAX`, and if we cannot find the best key to delete (e.g. maxmemory-policy
is `volatile-lru` and all keys with ttl have been evicted), in `cant_free` redis will sleep
forever if some items are being freed in the lazyfree thread.

(cherry picked from commit 464aa04188)
 2022-09-21 22:42:01 +03:00
Viktor Söderqvist 6edbf70760 Avoid crash when a cluster node is a replica of a replica of itself (#11263) 
(cherry picked from commit 42e4241ece)
 2022-09-21 22:42:01 +03:00
Madelyn Olson 7fc1a8c5a0 Prevent use after free for inbound cluster link (#11255) 
(cherry picked from commit 6c03786b66)
 2022-09-21 22:42:01 +03:00
chendianqiang 8c702f8ddb Correctly handle scripts with shebang (not read-only) on a cluster replica (#11223) 
EVAL scripts are by default not considered `write` commands, so they were allowed on a replica.
But when adding a shebang, they become `write` command (unless the `no-writes` flag is added).
With this change we'll handle them as write commands, and reply with MOVED instead of
READONLY when executed on a redis cluster replica.

Co-authored-by: chendianqiang <chendianqiang@meituan.com>
(cherry picked from commit e42d98ed27)
 2022-09-21 22:42:01 +03:00
Huang Zhw 63db10ef6f Remove the NONDETERMINISTIC_OUTPUT flag from most CLUSTER sub-commands. (#11157) 
TLDR: the CLUSTER command originally had the `random` flag,
so all the sub-commands initially got that new flag, but in fact many
of them don't need it.
The only effect of this change is on the output of COMMAND INFO.

(cherry picked from commit a7da7473cb)
 2022-09-21 22:42:01 +03:00
Oran Agra a2a28b8034 Fix assertion when a key is lazy expired during cluster key migration (#11176) 
Redis 7.0 has #9890 which added an assertion when the propagation queue
was not flushed and we got to beforeSleep.
But it turns out that when processCommands calls getNodeByQuery and
decides to reject the command, it can lead to a key that was lazy
expired and is deleted without later flushing the propagation queue.

This change prevents lazy expiry from deleting the key at this stage
(not as part of a command being processed in `call`)

(cherry picked from commit c789fb0aa7)
 2022-09-21 22:42:01 +03:00
Itamar Haber 1e2876e016 Replaces a made-up term with a real one (#11169) 
(cherry picked from commit 407b5c912f)
 2022-09-21 22:42:01 +03:00
Itamar Haber dbb7337e49 Adds historical note about lower-case geo units support (#11162) 
This change was part of #9656 (Redis 7.0)

(cherry picked from commit 31ef410e88)
 2022-09-21 22:42:01 +03:00
Binbin fa60cd1369 Fix memory leak in moduleFreeCommand (#11147) 
Currently, we call zfree(cmd->args), but the argument array
needs to be freed recursively (there might be sub-args).
Also fixed memory leaks on cmd->tips and cmd->history.

Fixes #11145

(cherry picked from commit fc3956e8f4)
 2022-09-21 22:42:01 +03:00
DarrenJiang13 4dfc487f96 fix the client type in trackingInvalidateKey() (#11052) 
Fix bug with scripts ignoring client tracking NOLOOP and
send an invalidation message anyway.

(cherry picked from commit 44859a41ee)
 2022-09-21 22:42:01 +03:00
Huang Zhw ab524282e8 acl: bitfield with get and set|incrby can be executed with readonly permission (#11086) 
`bitfield` with `get` may not be readonly.

```
127.0.0.1:6384> acl setuser hello on nopass %R~* +@all
OK
127.0.0.1:6384> auth hello 1
OK
127.0.0.1:6384> bitfield hello set i8 0 1
(error) NOPERM this user has no permissions to access one of the keys used as arguments
127.0.0.1:6384> bitfield hello set i8 0 1 get i8 0
1) (integer) 0
2) (integer) 1
```

Co-authored-by: Oran Agra <oran@redislabs.com>
(cherry picked from commit ec5034a2e3)
 2022-09-21 22:42:01 +03:00
Wen Hui d1b3276b55 Fix function load error message (#10964) 
Update error messages for function load

(cherry picked from commit beb9746a9f)
 2022-09-21 22:42:01 +03:00
Rudi Floren 631eb3837e Fix wrong commands json docs for CLIENT KILL (#10970) 
The docs state that there is a new and an old argument format.
The current state of the arguments allows mixing the old and new format,
thus the need for two additional oneof blocks.
One for differentiating the new from the old format and then one to
allow setting multiple filters using the new format.

(cherry picked from commit 4ce3fd51b9)
 2022-09-21 22:42:01 +03:00
Huang Zhw 5632bbdcbd tracking pending invalidation message of flushdb sent by (#11068) 
trackingHandlePendingKeyInvalidations should use proto.

(cherry picked from commit 61451b02cb)
 2022-09-21 22:42:01 +03:00
Huang Zhw ba6ed445a3 When client tracking is on, invalidation message of flushdb in a (#11038) 
When FLUSHDB / FLUSHALL / SWAPDB is inside MULTI / EXEC, the
client side tracking invalidation message was interleaved with transaction response.


(cherry picked from commit 6f0a27e38e)
 2022-09-21 22:42:01 +03:00
Meir Shpilraien (Spielrein) ddf1bcf788 Fix #11030, use lua_rawget to avoid triggering metatables and crash. (#11032) 
Fix #11030, use lua_rawget to avoid triggering metatables.

#11030 shows how return `_G` from the Lua script (either function or eval), cause the
Lua interpreter to Panic and the Redis processes to exit with error code 1.
Though return `_G` only panic on Redis 7 and 6.2.7, the underline issue exists on older
versions as well (6.0 and 6.2). The underline issue is returning a table with a metatable
such that the metatable raises an error.

The following example demonstrate the issue:
```
127.0.0.1:6379> eval "local a = {}; setmetatable(a,{__index=function() foo() end}) return a" 0
Error: Server closed the connection
```
```
PANIC: unprotected error in call to Lua API (user_script:1: Script attempted to access nonexistent global variable 'foo')
```

The Lua panic happened because when returning the result to the client, Redis needs to
introspect the returning table and transform the table into a resp. In order to scan the table,
Redis uses `lua_gettable` api which might trigger the metatable (if exists) and might raise an error.
This code is not running inside `pcall` (Lua protected call), so raising an error causes the
Lua to panic and exit. Notice that this is not a crash, its a Lua panic that exit with error code 1.

Returning `_G` panics on Redis 7 and 6.2.7 because on those versions `_G` has a metatable
that raises error when trying to fetch a none existing key.

### Solution

Instead of using `lua_gettable` that might raise error and cause the issue, use `lua_rawget`
that simply return the value from the table without triggering any metatable logic.
This is promised not to raise and error.

The downside of this solution is that it might be considered as breaking change, if someone
rely on metatable in the returned value. An alternative solution is to wrap this entire logic
with `pcall` (Lua protected call), this alternative require a much bigger refactoring.

### Back Porting

The same fix will work on older versions as well (6.2, 6.0). Notice that on those version,
the issue can cause Redis to crash if inside the metatable logic there is an attempt to accesses
Redis (`redis.call`). On 7.0, there is not crash and the `redis.call` is executed as if it was done
from inside the script itself.

### Tests

Tests was added the verify the fix

(cherry picked from commit 020e046b42)
 2022-09-21 22:42:01 +03:00
Binbin 5da7fdb785 fsync the old aof file when open a new INCR AOF (#11004) 
In rewriteAppendOnlyFileBackground, after flushAppendOnlyFile(1),
and before openNewIncrAofForAppend, we should call redis_fsync
to fsync the aof file.

Because we may open a new INCR AOF in openNewIncrAofForAppend,
in the case of using everysec policy, the old AOF file may not
be fsynced in time (or even at all).

When using everysec, we don't want to pay the disk latency from
the main thread, so we will do a background fsync.

Adding a argument for bioCreateCloseJob, a `need_fsync` flag to
indicate that a fsync is required before the file is closed. So we will
fsync the old AOF file before we close it.

A cleanup, we make union become a union, since the free_* args and
the fd / fsync args are never used together.

Co-authored-by: Oran Agra <oran@redislabs.com>
(cherry picked from commit 03fff10ab4)
 2022-09-21 22:42:01 +03:00
chenyang8094 c039428627 Register abs-expire apis (#11025) 
RM_SetAbsExpire and RM_GetAbsExpire were not actually operational since
they were introduced, due to omission in API registration.

(cherry picked from commit 39d216a326)
 2022-09-21 22:42:01 +03:00
Pavel Krush 61a59b53a5 fixed complexity of bzmpop and zmpop commands (#11026) 
Swap M and N in the complexity formula of [B]ZMPOP

Co-authored-by: Pavel Krush <neon@pushwoosh.com>
(cherry picked from commit 5879e490b8)
 2022-09-21 22:42:01 +03:00
Tian babe90c90f Don't update node ip when peer fd is closed (#10696) 
(cherry picked from commit d00b8af892)
 2022-09-21 22:42:01 +03:00
guybe7 9e3dfa96f7 Adds LASTID to XCLAIM docs (#11017) 
It seems it was overlooked when we first created the json files

(cherry picked from commit 6d6e932fa6)
 2022-09-21 22:42:01 +03:00
Wen Hui d28314cb03 Fix EVALSHA_RO and EVAL_RO command json file (#11015) 
these are missing from the RO_ commands, present in the other ones.

Co-authored-by: Ubuntu <lucas.guang.yang1@huawei.com>
(cherry picked from commit 56828bab59)
 2022-09-21 22:42:01 +03:00
Wen Hui a8b6efea25 Add optional for FCALL and FCALL_RO command json file (#10988) 
According to the Redis functions documentation, FCALL command format could be
FCALL function_name numberOfKeys [key1, key2, key3.....] [arg1, arg2, arg3.....]

So in the json file of fcall and fcall_ro, we should add optional for key and arg part.
Just like EVAL...

Co-authored-by: Binbin <binloveplay1314@qq.com>
(cherry picked from commit 2d3240f31b)
 2022-09-21 22:42:01 +03:00
Oran Agra 888effe4be Redis 7.0.4
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
2022-07-18 16:04:07 +03:00
Oran Agra 15ae4e29e5 Fix heap overflow corruption in XAUTOCLAIM (CVE-2022-31144) (#11002) 
The temporary array for deleted entries reply of XAUTOCLAIM was
insufficient, but also in fact the COUNT argument should be used to
control the size of the reply, so instead of terminating the loop by
only counting the claimed entries, we'll count deleted entries as well.

Fix #10968
Addresses CVE-2022-31144

(cherry picked from commit 2825b6057b)
 2022-07-18 16:04:07 +03:00
Valentino Geron 21fd252a62 remove boolean usage and use 0/1 instead (#10997) 
If we do not use jemalloc (mostly with valgrind) and use an old compiler that does not support C11
we will get compilation error

Co-authored-by: Valentino Geron <valentino@redis.com>
(cherry picked from commit 82b8203555)
 2022-07-18 16:04:07 +03:00
Oran Agra 76b9c13d1d Merge pull request #10962 from oranagra/release-7.0.3
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details 
Release 7.0.3
 2022-07-11 17:44:20 +03:00
Oran Agra 85abb7cf2a 7.0.3 2022-07-11 15:22:28 +03:00
Oran Agra 7b9fc6fca0 Merge 'origin/unstable' into 7.0 for 7.0.3 2022-07-11 15:22:06 +03:00
Binbin 693acc0114 Trying to fix cluster test (#10963) 
#10942 break the new test added in #10449
```
Testing unit: 29-slot-migration-response.tcl
Cluster Join and auto-discovery test: FAILED: Cluster failed to join into a full mesh.
```

It looks like we need to wait for the cluster in 28 to become stable.
 2022-07-11 15:21:35 +03:00
Oran Agra b4568fc78a update help.h (#10961) 2022-07-11 12:36:03 +03:00
BinbinandMadelyn Olson 35e8ae3eb5 Add cluster-port support to redis-cli --cluster (#10344) 
In #9389, we add a new `cluster-port` config and make cluster bus port configurable,
and currently redis-cli --cluster create/add-node doesn't support with a configurable `cluster-port` instance.
Because redis-cli uses the old way (port + 10000) to send the `CLUSTER MEET` command.

Now we add this support on redis-cli `--cluster`, note we don't need to explicitly pass in the
`cluster-port` parameter, we can get the real `cluster-port` of the node in `clusterManagerNodeLoadInfo`,
so the `--cluster create` and `--cluster add-node` interfaces have not changed.

We will use the `cluster-port` when we are doing `CLUSTER MEET`, also note that `CLUSTER MEET` bus-port
parameter was added in 4.0, so if the bus_port (the one in redis-cli) is 0, or equal (port + 10000),
we just call `CLUSTER MEET` with 2 arguments, using the old form.

Co-authored-by: Madelyn Olson <34459052+madolson@users.noreply.github.com>
 2022-07-11 11:23:31 +03:00
Binbin 1e85b89aef Add --check-system in redis-server usage (#10960) 2022-07-11 08:31:39 +03:00
Madelyn Olson e6a1b2ea95 Fix crash during handshake and cluster shards call (#10942) 
* Fix an engine crash when there are nodes in handshaking and a user calls cluster shards
 2022-07-10 22:00:44 -07:00
Madelyn Olson 1209dc2277 Only print ACL syntax errors once and include command names in errors (#10922) 
* Only print ACL syntax errors once and include command names in errors
 2022-07-09 21:02:22 -07:00
Binbin 8203461120 Fix some outdated comments and some typo (#10946) 
* Fix some outdated comments and some typo
 2022-07-06 20:31:59 -07:00
adasarpan404andOran Agra 0a2e51f9ae Renaming conduct to code of conduct and contributing files to .md (#10941) 
CONTRIBUTING to get better formatting
CONDUCT also because github doesn't seem recognize the code of conduct page

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-07-06 11:58:44 +03:00
Binbin 0132ed7544 Add pubsubshard_channels field in INFO STATS (#10929) 
We already have `pubsub_channels` and `pubsub_patterns`
in INFO stats, now add `pubsubshard_channels` (symmetry).

Sharded pubsub was added in #8621
 2022-07-06 09:50:08 +03:00
Yossi Gottlieb 45ae605332 TLS: Notify clients on connection shutdown. (#10931) 
Use SSL_shutdown(), in a best-effort manner, when closing a TLS
connection. This change better supports OpenSSL 3.x clients that will
not silently ignore the socket-level EOF.
 2022-07-05 08:41:17 +03:00
Harkrishn Patro 9bcdd1537e Avoid double multiplication of alloc_count (#10934) 2022-07-04 10:09:27 -07:00
Wen HuiUbuntuMadelyn Olson
 		f620e6ac73 Add tests for error messages during slot migrations (#10449) 
* Add tests for error messages during slot migrations

Co-authored-by: Ubuntu <lucas.guang.yang1@huawei.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
 2022-07-04 10:31:12 -05:00
Harkrishn Patro a3704d4e87 Optimize number of realloc syscall during multi/exec flow (#10921) 
## Issue
During the MULTI/EXEC flow, each command gets queued until the `EXEC`
command is received and during this phase on every command queue, a
`realloc` is being invoked. This could be expensive based on the realloc
behavior (if copy to a new memory location). 


## Solution
In order to reduce the no. of syscall, couple of optimization I've used.

1. By default, reserve memory for atleast two commands. `MULTI/EXEC` for a
single command doesn't have any significance. Hence, I believe customer wouldn't use it.
2. For further reservation, increase the memory allocation in exponent growth (power of 2).
This reduces the no. of `realloc` call from `N` to `log(N)` times.

## Other changes:

* Include multi exec queued command array in client memory consumption calculation
(affects client eviction too)
 2022-07-04 09:47:34 +03:00
Qu Chen 33b7ff387c Unlock cluster config file upon server shutdown. (#10912) 
Currently in cluster mode, Redis process locks the cluster config file when
starting up and holds the lock for the entire lifetime of the process.
When the server shuts down, it doesn't explicitly release the lock on the
cluster config file. We noticed a problem with restart testing that if you shut down
a very large redis-server process (i.e. with several hundred GB of data stored),
it takes the OS a while to free the resources and unlock the cluster config file.
So if we immediately try to restart the redis server process, it might fail to acquire
the lock on the cluster config file and fail to come up.

This fix explicitly releases the lock on the cluster config file upon a shutdown rather
than relying on the OS to release the lock, which is a cleaner and safer approach to
free up resources acquired.
 2022-07-04 09:38:19 +03:00
Harkrishn Patro 0ab885a685 Account sharded pubsub channels memory consumption (#10925) 
Account sharded pubsub channels memory consumption in client memory usage
computation to accurately evict client based on the set threshold for `maxmemory-clients`.
 2022-07-04 09:18:57 +03:00
Binbin 679344a2b0 Set aof rewrite status in some backgroundRewriteDoneHandler errors (#10923) 
We should also set aof_lastbgrewrite_status to C_ERR on these
errors. Because aof rewrite did fail, and we did not finish the
manifest update. Also maintain the stat_aofrw_consecutive_failures.
 2022-07-03 15:36:42 +03:00
Yossi Gottlieb 0b645d6319 Fix TLS issues with large replies (#10909) 
This problem was introduced by 496375f and seems to more easily reproduce on macOS since OpenSSL writes more frequently return with EAGAIN.
 2022-07-03 13:35:58 +03:00
Yossi Gottlieb 69d5576832 Fix TLS tests on newer tcl-tls/OpenSSL. (#10910) 
Before this commit, TLS tests on Ubuntu 22.04 would fail as dropped
connections result with an ECONNABORTED error thrown instead of an empty
read.
 2022-07-03 13:34:14 +03:00
Binbin 2ab6767744 Always set server.aof_last_write_errno in aof write error (#10917) 
The `can_log` variable prevents us from outputting too
many error logs. But it should not include the modification
of server.aof_last_write_errno.

We are doing this because:
1. In the short write case, we always set aof_last_write_errno
to ENOSPC, we don't care the `can_log` flag.

2. And we always set aof_last_write_status to C_ERR in aof write
error (except for FSYNC_ALWAYS, we exit). So there may be a chance
that `aof_last_write_errno` is not right.

An innocent bug or just a code cleanup.
 2022-07-03 08:34:39 +03:00
Binbin 35e836c26d Add SENTINEL command flag to CLIENT/COMMANDS subcommands (#10904) 
This was harmless because we marked the parent command
with SENTINEL flag. So the populateCommandTable was ok.
And we also don't show the flag (SENTINEL and ONLY-SENTNEL)
in COMMAND INFO.

In this PR, we also add the same CMD_SENTINEL and CMD_ONLY_SENTINEL
flags check when populating the sub-commands.
so that in the future it'll be possible to add some sub-commands to sentinel or sentinel-only but not others.
 2022-06-30 16:32:40 +03:00
Wen HuiUbuntuOran AgraBinbin
 		51da5c3dde Fix CLUSTER RESET command argument number issue (#10898) 
Fix regression of CLUSTER RESET command in redis 7.0.

cluster reset command format is:
CLUSTER RESET [ HARD | SOFT]

According to the cluster reset command doc and codes, the third argument is optional, so
the arity in json file should be -2 instead of 3.

Add test to verify future regressions with RESET and RESET SOFT that were not covered.

Co-authored-by: Ubuntu <lucas.guang.yang1@huawei.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Binbin <binloveplay1314@qq.com>
 2022-06-29 08:17:00 +03:00
jonnyomerredis 35c2ee8716 Add sharded pubsub keychannel count to client info (#10895) 
When calling CLIENT INFO/LIST, and in various debug prints, Redis is printing
the number of pubsub channels / patterns the client is subscribed to.
With the addition of sharded pubsub, it would be useful to print the number of
keychannels the client is subscribed to as well.
 2022-06-28 10:11:17 +03:00
Tian 069b30a2b3 A minor refinement to clusterbus extension estlen (#10902) 2022-06-27 20:42:55 -07:00
Viktor Söderqvist 6af021007a Add missing REDISMODULE_CLIENTINFO_INITIALIZER (#10885) 
The module API docs mentions this macro, but it was not defined (so no one could have used it).

Instead of adding it as is, we decided to add a _V1 macro, so that if / when we some day extend this struct,
modules that use this API and don't need the extra fields, will still use the old version
and still be compatible with older redis version (despite being compiled with newer redismodule.h)
 2022-06-27 08:29:05 +03:00
RinChanNOW! 2854637385 Support conversion between RedisModuleString and unsigned long long (#10889) 
Since the ranges of `unsigned long long` and `long long` are different, we cannot read an
`unsigned long long` integer from a `RedisModuleString` by `RedisModule_StringToLongLong` . 

So I added two new Redis Module APIs to support the conversion between these two types:
* `RedisModule_StringToULongLong`
* `RedisModule_CreateStringFromULongLong`

Signed-off-by: RinChanNOWWW <hzy427@gmail.com>
 2022-06-26 15:02:52 +03:00
Binbin d443e312ad redis-server command line arguments allow passing config name and value in the same arg (#10866) 
This commit has two topics.

## Passing config name and value in the same arg
In #10660 (Redis 7.0.1), when we supported the config values that can start with `--` prefix (one of the two topics of that PR),
we broke another pattern: `redis-server redis.config "name value"`, passing both config name
and it's value in the same arg, see #10865

This wasn't a intended change (i.e we didn't realize this pattern used to work).
Although this is a wrong usage, we still like to fix it.

Now we support something like:
```
src/redis-server redis.conf "--maxmemory '700mb'" "--maxmemory-policy volatile-lru" --proc-title-template --my--title--template --loglevel verbose
```

## Changes around --save
Also in this PR, we undo the breaking change we made in #10660 on purpose.
1. `redis-server redis.conf --save --loglevel verbose` (missing `save` argument before anotehr argument).
    In 7.0.1, it was throwing an wrong arg error.
    Now it will work and reset the save, similar to how it used to be in 7.0.0 and 6.2.x.
3. `redis-server redis.conf --loglevel verbose --save` (missing `save` argument as last argument).
    In 6.2, it did not reset the save, which was a bug (inconsistent with the previous bullet).
    Now we will make it work and reset the save as well (a bug fix).
 2022-06-26 14:36:39 +03:00
Viktor Söderqvist 6272ca609e Add RM_SetClientNameById and RM_GetClientNameById (#10839) 
Adding Module APIs to let the module read and set the client name of an arbitrary connection.
 2022-06-26 14:34:59 +03:00
Binbin d96cf3639a Sync argv_len var back after command call in execCommand (#10900) 
This is harmless, we only restore mstate to make sure we
free the right pointer in freeClientMultiState, but it'll
be nicer to also sync that argv_len var back.
 2022-06-26 12:32:34 +03:00
Elvina Yakubova e2cf386765 Replace regular array to array of structs to get rid of false sharing (#10892) 
There are a lot of false sharing cache misses in line 4013 inside getIOPendingCount function.
The reason is that elements of io_threads_pending array access the same cache line from different
threads, so it is better to represent it as an array of structures with fields aligned to cache line size.
This change should improve performance (in particular, it affects the latency metric, we saw up to 3% improvement).
 2022-06-26 08:39:30 +03:00
Steve Lorello a64b29485d changing min,max in ZRANGE -> start stop (#10097) 
In 6.2.0 with the introduction of the REV subcommand in ZRANGE, there was a semantic shift in the arguments of ZRANGE when the REV sub-command is executed. Without the sub-command `min` and `max` (the old names of the arguments) are appropriate because if you put the min value and the max value in everything works fine.

```bash
127.0.0.1:6379> ZADD myset 0 foo
(integer) 1
127.0.0.1:6379> ZADD myset 1 bar
(integer) 1
127.0.0.1:6379> ZRANGE myset 0 inf BYSCORE
1) "foo"
2) "bar"
``` 

However - if you add the `REV` subcommand, ordering the arguments `min` `max` breaks the command:

```bash
127.0.0.1:6379> ZRANGE myset 0 inf BYSCORE REV
(empty array)
```

why? because `ZRANGE` with the `REV` sub-command is expecting the `max` first and the `min` second (because it's a reverse range like `ZREVRANGEBYSCORE`):

```bash
127.0.0.1:6379> ZRANGE myset 0 inf BYSCORE REV
(empty array)
```
 2022-06-25 19:02:20 +03:00
WuYunlong 64205345bc migrateGetSocket() cleanup.. (#5546) 
I think parameter c is only useful to get client reply.
Besides, other commands' host and port parameters may not be the at index 1 and 2.
 2022-06-23 18:41:32 +03:00
Binbin 755b51a42c When dirCreateIfMissing or openNewIncrAofForAppend fail, set aof_lastbgrewrite_status to err (#10775) 
It will be displayed in the `aof_last_bgrewrite_status`
field of the INFO command.
 2022-06-23 18:40:20 +03:00
(╯°□°）╯︵ uᴉǝssnH ɐɟɐʇsoW c52922e1d9 command prompt help: add stdin config usage (#6313) 
Signed-off-by: Mostafa Hussein <mostafa.hussein91@gmail.com>
 2022-06-23 18:37:42 +03:00
judeng d2405b9b6b fix benchmark failure in daily test with TLS (#10896) 
The new test added in #10891 can fail with a different error.
see comment in networking.c saying
```c
        /* That's a best effort error message, don't check write errors.
         * Note that for TLS connections, no handshake was done yet so nothing
         * is written and the connection will just drop. */
```
 2022-06-23 18:19:36 +03:00
Binbin aabce8932a Add bus_port argument in cluster-meet.json (#10304) 
In `CLUSTER MEET`, bus-port argument was added in 11436b1.
For cluster announce ip / port implementation, part of the
4.0-RC1.

And in #9389, we add a new cluster-port config and make
cluster bus port configurable, part of the 7.0-RC1.
 2022-06-23 07:56:25 +03:00
judeng 49876158cc fix redis-benchmark's bug: check if clients are created successfully in idle mode (#10891) 
my maxclients config:
```
redis-cli config get maxclients
1) "maxclients"
2) "4064"
```

Before this bug was fixed, creating 4065 clients appeared to be successful, but only 4064 were actually created```
```
./redis-benchmark -c 4065 -I
Creating 4065 idle connections and waiting forever (Ctrl+C when done)
cients: 4065
```

now :
```
./redis-benchmark -c 4065 -I
Creating 4065 idle connections and waiting forever (Ctrl+C when done)
Error from server: ERR max number of clients reached

./redis-benchmark -c 4064 -I
Creating 4064 idle connections and waiting forever (Ctrl+C when done)
clients: 4064

```
 2022-06-22 19:30:22 +03:00
Viktor Söderqvist 02acb8fd3a Module API docs corrections (#10890) 
* Fix typo `RedisModule_CreatString` -> `RedisModule_CreateString` (multiple occurrences)
* Make the markdown gen script change all `RM_` to `RedisModule_` even in code examples, etc.
 2022-06-21 17:00:24 +03:00
Meir Shpilraien (Spielrein) 61baabd8d5 Fix crash on RM_Call with script mode. (#10886) 
The PR fixes 2 issues:

### RM_Call crash on script mode

`RM_Call` can potentially be called from a background thread where `server.current_client`
are not set. In such case we get a crash on `NULL` dereference.
The fix is to check first if `server.current_client` is `NULL`, if it does we should
verify disc errors and readonly replica as we do to any normal clients (no masters nor AOF).

### RM_Call block OOM commands when not needed

Again `RM_Call` can be executed on a background thread using a `ThreadSafeCtx`.
In such case `server.pre_command_oom_state` can be irrelevant and should not be
considered when check OOM state. This cause OOM commands to be blocked when
not necessarily needed.

In such case, check the actual used memory (and not the cached value). Notice that in
order to know if the cached value can be used, we check that the ctx that was used on
the `RM_Call` is a ThreadSafeCtx. Module writer can potentially abuse the API and use
ThreadSafeCtx on the main thread. We consider this as a API miss used.
 2022-06-21 10:01:13 +03:00
Vlad a3fdc9cd82 Outdated comments, replace COUNTER_INIT_VAL with LFU_INIT_VAL, fix typo (#10888) 
`COUNTER_INIT_VAL` doesn't exist in the code anymore so we can replace it with `LFU_INIT_VAL` entries
 2022-06-21 08:14:31 +03:00
Bar Shaul 091701f363 Set replicas' configEpoch to 0 when loaded from cluster configuration file (#10798) 
* Changed clusterLoadConfig to set the config epoch of replica nodes to 0 when loaded.
 2022-06-20 21:40:48 -07:00
judeng ff6419658b Optimize the performance of clusterSendPing for large clusters (#10624) 
Optimize the performance of clusterSendPing by improving speed of checking for duplicate items in gossip.
 2022-06-20 21:02:22 -07:00
Tian 99a425d0f3 Fsync directory while persisting AOF manifest, RDB file, and config file (#10737) 
The current process to persist files is `write` the data, `fsync` and `rename` the file,
but a underlying problem is that the rename may be lost when a sudden crash like
power outage and the directory hasn't been persisted.

The article [Ensuring data reaches disk](https://lwn.net/Articles/457667/) mentions
a safe way to update file should be:

1. create a new temp file (on the same file system!)
2. write data to the temp file
3. fsync() the temp file
4. rename the temp file to the appropriate name
5. fsync() the containing directory

This commit handles CONFIG REWRITE, AOF manifest, and RDB file (both for persistence,
and the one the replica gets from the master).
It doesn't handle (yet), ACL SAVE and Cluster configs, since these don't yet follow this pattern.
 2022-06-20 19:17:23 +03:00
Moti Cohenandmoticless 4c72a09b78 Fix sentinel acl change test. Timing issue. (#10868) 
Co-authored-by: moticless <moticless@github.com>
 2022-06-19 09:45:16 +03:00
Masahiro Ide f2387daa83 Fix a prototype inconsitency of _serverAssert between redisassert.h and redis.h (#10872) 
both redisassert.c and server.h + debug.c use const, but redisassert.h doesn't.
 2022-06-19 08:42:12 +03:00
Wen HuiandOran Agra c37d63e87a DEPRECATED rename-command config in sentinel (#10877) 
Clients could use this config in sentinel only when they already add rename-command in the redis conf file,
but becuase in Redis instance, we already DEPRECATED rename-command config,
thus we should deprecate this in the sentinel part as well.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-06-19 08:18:47 +03:00
Chris Lamb a4754e228f Correct 'certificate' typo. (#10867) 2022-06-15 13:37:39 +03:00
Oran Agra 2189100383 optimize zset conversion on large ZRANGESTORE (#10789) 
when we know the size of the zset we're gonna store in advance,
we can check if it's greater than the listpack encoding threshold,
in which case we can create a skiplist from the get go, and avoid
converting the listpack to skiplist later after it was already populated.
 2022-06-14 21:12:45 +03:00
Oran Agra 8ef4f1dbad Script that made modification will not break with unexpected NOREPLICAS error (#10855) 
If a script made a modification and then was interrupted for taking too long.
there's a chance redis will detect that a replica dropped and would like to reject
write commands with NOREPLICAS due to insufficient good replicas.
returning an error on a command in this case breaks the script atomicity.

The same could in theory happen with READONLY, MISCONF, but i don't think
these state changes can happen during script execution.
 2022-06-14 21:09:50 +03:00
Oran Agra ffa0077041 Allow ECHO in loading and stale modes (#10853) 
I noticed that scripting.tcl uses INFO from within a script and thought it's an
overkill and concluded it's nicer to use another CMD_STALE command,
decided to use ECHO, and then noticed it's not at all allowed in stale mode.
probably overlooked at #6843
 2022-06-14 08:48:08 +03:00
hdyztmdqd c65e5087e8 Fix 2 comments in dict.c & redis-cli.c (#10860) 
* fix comment in dict.c

fix comment in dict.c

* Fix comment in redis-cli.c

Fix comment in redis-cli.c
 2022-06-14 08:09:08 +03:00
Huang Zhw 78960ad57b Throw -TRYAGAIN instead of -ASK on migrating nodes for multi-key commands when the node only has some of the keys (#9526) 
* In cluster getNodeByQuery when target slot is in migrating state and
the slot lack some keys but have at least one key, should return TRYAGAIN.

Before this commit, when a node is in migrating state and recevies
multiple keys command, if some keys don't exist, the command emits
an `ASK` redirection.

After this commit, if some keys exist and some keys don't exist, the
command emits a TRYAGAIN error. If all keys don't exist, the command
emits an `ASK` redirection.
 2022-06-13 21:32:43 -07:00
Binbin 0a2f78837d redis-check-rdb add when_opcode check for module aux (#10859) 
In #9199, we add a `goto eoferr` in when_opcode check,
this means that if the when_opcode check fails, we will
abort the rdb loading, but this not reflected in the
rdb-check tool. So someone can modify when_opcode to make
rdb load fail, but rdb-check will report OK. Just a cleanup
or a code consistency issue.
```
serverLog: # Internal error in RDB reading offset 0, function at rdb.c:3055 -> bad when_opcode
[offset 0] Checking RDB file dump.rdb
[offset 109] \o/ RDB looks OK! \o/
```

Plus a minor memory leak fix like #9860, note that it will
exit immediately after the eoferr, so it is not strictly a
leak, so it is also a small cleanup.
 2022-06-13 18:23:53 +03:00
XiongDa d4595dd94f Fix typo in replication.c (#10854) 2022-06-13 12:12:46 +03:00
Johannes Truschnigg 9f3b410050 Correctly check for vm.overcommit_memory == 0 (#10841) 
A regression caused by #10636 (released in 7.0.1) causes Redis startup warning about overcommit to be missing when needed and printed when not. 

Also, using atoi() to convert the string's value to an integer cannot discern
between an actual 0 (zero) having been read, or a conversion error.
 2022-06-12 20:53:11 +03:00
Oran Agra 05833959e3 Merge pull request #10851 from oranagra/release702
CI / build-libc-malloc (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details 
Release 7.0.2
 2022-06-12 16:00:17 +03:00
Oran Agra 1973558b63 7.0.2 2022-06-12 13:06:03 +03:00
Oran Agra 5267d3803f Merge remote-tracking branch 'origin/unstable' into 7.0 2022-06-12 12:02:39 +03:00
XiongDa abb2ea7e3c Fix 3 comments in server.c (#10844) 2022-06-12 08:44:44 +03:00
BinbinandOran Agra 92fb4f4f61 Fixed SET and BITFIELD commands being wrongly marked movablekeys (#10837) 
The SET and BITFIELD command were added `get_keys_function` in #10148, causing
them to be wrongly marked movablekeys in `populateCommandMovableKeys`.

This was an unintended side effect introduced in #10148 (7.0 RC1)
which could cause some clients an extra round trip for these commands in cluster mode.

Since we define movablekeys as a way to determine if the legacy range [first, last, step]
doesn't find all keys, then we need a completely different approach.

The right approach should be to check if the legacy range covers all key-specs,
and if none of the key-specs have the INCOMPLETE flag. 
This way, we don't need to look at getkeys_proc of VARIABLE_FLAG at all.
Probably with the exception of modules, who may still not be using key-specs.

In this PR, we removed `populateCommandMovableKeys` and put its logic in
`populateCommandLegacyRangeSpec`.
In order to properly serve both old and new modules, we must probably keep relying
CMD_MODULE_GETKEYS, but do that only for modules that don't declare key-specs. 
For ones that do, we need to take the same approach we take with native redis commands.

This approach was proposed by Oran. Fixes #10833

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-06-12 08:22:18 +03:00
Binbin 62ac1ab007 Fix crash when overcommit_memory is inaccessible (#10848) 
When `/proc/sys/vm/overcommit_memory` is inaccessible, fp is NULL.
`checkOvercommit` will return -1 without setting err_msg, and then
the err_msg is used to print the log, crash the server.
Set the err_msg variables to Null when declaring it, seems safer.

And the overcommit_memory error log will print two "WARNING",
like `WARNING WARNING overcommit_memory is set to 0!`, this PR
also removes the second WARNING in `checkOvercommit`.

Reported in #10846. Fixes #10846. Introduced in #10636 (7.0.1)
 2022-06-11 22:27:44 +03:00
Christian KriegandYossi Gottlieb 032619b82b Fixing test to consider statically linked binaries (#10835) 
The test calls `ldd` on `redis-server` in order to find out whether the binary
was linked against `libmusl`; However, `ldd` returns a value different from `0`
when statically linking the binaries agains libc-musl, because `redis-server` is
not a dynamic executable (as given by the exception thrown by the failing test),
and `make test` terminates with an error::

   $ ldd src/redis-server
       not a dynamic executable
   $ echo $?
   1

This commit fixes the test by ignoring such failures.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
 2022-06-09 12:59:33 +03:00
Wen Hui 6e1c3ff132 Update some comments in stream command docs (#10821) 
some small documentation fixes.
 2022-06-09 10:16:56 +03:00
Steve Lorello 9e40b076dd Module api doc generator, fixing issue with negative lookback terminating at "." (#10832) 
There is a little regex that wraps up all the free-floating functions in the doc-block
e.g. malloc() with backticks.
in case of `redis.call()`, it used to wrap just `call()` in backticks.
 2022-06-08 15:30:04 +03:00
Oran Agra 2667c41235 Merge pull request #10829 from oranagra/release-7.0.1
External Server Tests / test-external-cluster (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details 
Release 7.0.1
 2022-06-08 12:56:24 +03:00
Oran Agra 595e725db4 7.0.1 2022-06-08 08:42:53 +03:00
Oran Agra a5d2df3473 Merge remote-tracking branch 'origin/unstable' into 7.0 2022-06-08 08:26:30 +03:00
Petr Vaněk f22bfe86b6 Update musl libc detection pattern (#10826) 
This change fixes failing `integration/logging.tcl` test in Gentoo with
musl libc, where `ldd` returns
```
libc.so => /lib/ld-musl-x86_64.so.1 (0x7f9d5f171000)
```
unlike Alpine's
```
libc.musl-x86_64.so.1 => /lib/ld-musl-x86_64.so.1 (0x7f82cfa16000)
```
The solution is to extend matching pattern introduced in #8532.
 2022-06-07 18:47:01 +03:00
Binbin ae9764ea0a Increment the stat_rdb_saves counter in SAVE command (#10827) 
Currently, we only increment stat_rdb_saves in rdbSaveBackground,
we should also increment it in the SAVE command.

We concluded there's no need to increment when:
1. saving a base file for an AOF
2. when saving an empty rdb file to delete an old one
3. when saving to sockets (not creating a persistence / snapshot file)

The stat counter was introduced in #10178

* fix a wrong comment in startSaving
 2022-06-07 17:38:31 +03:00
ls-2018 b8665b879b fix unused argument warning in ae_select.c (#10824) 2022-06-07 14:47:09 +03:00
Bjorn Svensson 1067cfb34b Documentation fixes of BITFIELD_RO and XINFO STREAM (#10823) 
Correcting the introduction version of the command `BITFIELD_RO`
Command added by commit: b3e4abf06e 

Add history info of the `FULL` modifier in `XINFO STREAM`
Modifier was added by commit: 1e2aee3919

(Includes output from `./utils/generate-command-code.py`)
 2022-06-07 13:26:47 +03:00
Binbin 3d56f607b7 Handle multiple_token flag in generate-command-help.rb (#10822) 
Currently generate-command.help.rb dose not handle the
multiple_token flag, handle this flag in this PR.
The format is the same as redis-cli rendering.
```diff
- bitfield_ro key GET encoding offset [encoding offset ...]
+ bitfield_ro key GET encoding offset [GET encoding offset ...]
```

Re run generate-command-code.py which was forget in #10820.
Also change the flag value from string to bool, like "true" to true
 2022-06-07 10:15:39 +03:00
ranshid b4772f6381 Fix bitfield_ro documentation (#10820) 
Current documentation only states a single GET token is needed which is not true. Marking the command with multiple_token.
 2022-06-06 18:11:58 -07:00
Oran Agra b0272252f7 update help.h (#10818) 
before releasing 7.0.1
 2022-06-06 11:39:53 +03:00
Oran Agra 475563e2e9 crash report instructions (#10816) 
Trying to avoid people opening crash report issues about module crashes and ARM QEMU bugs.
 2022-06-06 11:39:23 +03:00
DarrenJiang13 f558583493 Split instantaneous_repl_total_kbps to instantaneous_input_repl_kbps and instantaneous_output_repl_kbps. (#10810) 
A supplement to https://github.com/redis/redis/pull/10062
Split `instantaneous_repl_total_kbps` to `instantaneous_input_repl_kbps` and `instantaneous_output_repl_kbps`. 
## Work:
This PR:
- delete 1 info field:
    - `instantaneous_repl_total_kbps`
- add 2 info fields:
    - `instantaneous_input_repl_kbps / instantaneous_output_repl_kbps`
## Result:
- master
```
total_net_input_bytes:26633673
total_net_output_bytes:21716596
total_net_repl_input_bytes:0
total_net_repl_output_bytes:18433052
instantaneous_input_kbps:0.02
instantaneous_output_kbps:0.00
instantaneous_input_repl_kbps:0.00
instantaneous_output_repl_kbps:0.00
```
- slave
```
total_net_input_bytes:18433212
total_net_output_bytes:94790
total_net_repl_input_bytes:18433052
total_net_repl_output_bytes:0
instantaneous_input_kbps:0.00
instantaneous_output_kbps:0.05
instantaneous_input_repl_kbps:0.00
instantaneous_output_repl_kbps:0.00
```
 2022-06-06 08:29:24 +03:00
Mixficsol c751d8a686 Update cluster.c (#10773) 
On line 4068, redis has a logical nodeIsSlave(myself) on the outer if layer,
which you can delete without having to repeat the decision
 2022-06-06 08:17:18 +03:00
Steve Lorello 247e792bde some minor spelling/grammatical fixes to module.c (#10812) 2022-06-03 15:36:35 -07:00
Madelyn Olson 4ad166235e Update time independent string compare to use hash length (#9759) 
* Update time independent string compare to use hash length
 2022-06-03 09:30:28 -07:00
zhaozhao.zzandOran Agra a18c91d642 rewrite alias config to original name (#10811) 
Redis 7 adds some new alias config like `hash-max-listpack-entries` alias `hash-max-ziplist-entries`.

If a config file contains both real name and alias like this:
```
hash-max-listpack-entries 20
hash-max-ziplist-entries 20
```

after set `hash-max-listpack-entries` to 100 and `config rewrite`, the config file becomes to:
```
hash-max-listpack-entries 100
hash-max-ziplist-entries 20
```

we can see that the alias config is not modified, and users will get wrong config after restart.

6.0 and 6.2 doesn't have this bug, since they only have the `slave` word alias.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-06-02 14:03:47 +03:00
chenyang8094 aae0ec2553 Only print final log when aof is loaded successfully (#10808) 
Skip the print on AOF_NOT_EXIST status.
 2022-06-02 09:00:12 +03:00
zhugezyandOran Agra cf3323dba4 Fix bugs in CONFIG REWRITE, omitting rename-command and include lines, and inserting comments around module and acl configs (#10761) 
A regression from #10285 (redis 7.0).
CONFIG REWRITE would put lines with: `include`, `rename-command`,
`user`,  `loadmodule`, and any module specific config in a comment.

For ACL `user`, `loadmodule` and module specific configs would be
re-inserted at the end (instead of updating existing lines), so the only
implication is a messy config file full of comments.

But for `rename-command` and `include`, the implication would be that
they're now missing, so a server restart would lose them.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-06-02 08:36:55 +03:00
Ozan Tezcan c81b5e5594 Fix Lua compile warning (#10805) 
Apparently, GCC 11.2.0 has a new fancy warning for misleading indentations.
It prints a warning when BRET(b) is on the same line as the loop.
 2022-06-01 16:21:45 +03:00
Oran Agra df55861838 Expose script flags to processCommand for better handling (#10744) 
The important part is that read-only scripts (not just EVAL_RO
and FCALL_RO, but also ones with `no-writes` executed by normal EVAL or
FCALL), will now be permitted to run during CLIENT PAUSE WRITE (unlike
before where only the _RO commands would be processed).

Other than that, some errors like OOM, READONLY, MASTERDOWN are now
handled by processCommand, rather than the command itself affects the
error string (and even error code in some cases), and command stats.

Besides that, now the `may-replicate` commands, PFCOUNT and PUBLISH, will
be considered `write` commands in scripts and will be blocked in all
read-only scripts just like other write commands.
They'll also be blocked in EVAL_RO (i.e. even for scripts without the
`no-writes` shebang flag.

This commit also hides the `may_replicate` flag from the COMMAND command
output. this is a **breaking change**.

background about may_replicate:
We don't want to expose a no-may-replicate flag or alike to scripts, since we
consider the may-replicate thing an internal concern of redis, that we may
some day get rid of.
In fact, the may-replicate flag was initially introduced to flag EVAL: since
we didn't know what it's gonna do ahead of execution, before function-flags
existed). PUBLISH and PFCOUNT, both of which because they have side effects
which may some day be fixed differently.

code changes:
The changes in eval.c are mostly code re-ordering:
- evalCalcFunctionName is extracted out of evalGenericCommand
- evalExtractShebangFlags is extracted luaCreateFunction
- evalGetCommandFlags is new code
 2022-06-01 14:09:40 +03:00
Oran Agra b2061de2e7 Fix broken protocol in MISCONF error, RM_Yield bugs, RM_Call(EVAL) OOM check bug, and new RM_Call checks. (#10786) 
* Fix broken protocol when redis can't persist to RDB (general commands, not
  modules), excessive newline. regression of #10372 (7.0 RC3)
* Fix broken protocol when Redis can't persist to AOF (modules and
  scripts), missing newline.
* Fix bug in OOM check of EVAL scripts called from RM_Call.
  set the cached OOM state for scripts before executing module commands too,
  so that it can serve scripts that are executed by modules.
  i.e. in the past EVAL executed by RM_Call could have either falsely
  fail or falsely succeeded because of a wrong cached OOM state flag.
* Fix bugs with RM_Yield:
  1. SHUTDOWN should only accept the NOSAVE mode
  2. Avoid eviction during yield command processing.
  3. Avoid processing master client commands while yielding from another client
* Add new two more checks to RM_Call script mode.
  1. READONLY You can't write against a read only replica
  2. MASTERDOWN Link with MASTER is down and `replica-serve-stale-data` is set to `no`
* Add new RM_Call flag to let redis automatically refuse `deny-oom` commands
  while over the memory limit. 
* Add tests to cover various errors from Scripts, Modules, Modules
  calling scripts, and Modules calling commands in script mode.

Add tests:
* Looks like the MISCONF error was completely uncovered by the tests,
  add tests for it, including from scripts, and modules
* Add tests for NOREPLICAS from scripts
* Add tests for the various errors in module RM_Call, including RM_Call that
  calls EVAL, and RM_call in "eval mode". that includes:
  NOREPLICAS, READONLY, MASTERDOWN, MISCONF
 2022-06-01 13:04:22 +03:00
filipe oliveira 6a6e911f12 Moving client flags to a more cache friendly position within client struct (#10697) 
Move the client flags to a more cache friendly position within the client struct
we regain the lost 2% of CPU cycles since v6.2 ( from 630532.57 to 647449.80 ops/sec ).
These are due to higher rate of calls to getClientType due to changes in #9166 and #10020
 2022-06-01 10:50:01 +03:00
menwen 42fbf064d1 Add module API RM_MallocUsableSize (#10795) 
This allows the module to know the usable size of an allocation
it made, rather than the consumed size.
 2022-05-31 10:29:17 +03:00
zhaozhao.zz 805191c791 rename channel to shardchannel in sharded pubsub commands (#10738) 
since the sharded pubsub is different with pubsub, it's better to give users a hint to make it more clear.
 2022-05-31 08:55:25 +03:00
DarrenJiang13andOran Agra bb1de082ea Adds isolated netstats for replication. (#10062) 
The amount of `server.stat_net_output_bytes/server.stat_net_input_bytes`
is actually the sum of replication flow and users' data flow. 
It may cause confusions like this:
"Why does my server get such a large output_bytes while I am doing nothing? ". 

After discussions and revisions, now here is the change about what this
PR brings (final version before merge):
- 2 server variables to count the network bytes during replication,
     including fullsync and propagate bytes.
     - `server.stat_net_repl_output_bytes`/`server.stat_net_repl_input_bytes`
- 3 info fields to print the input and output of repl bytes and instantaneous
     value of total repl bytes.
     - `total_net_repl_input_bytes` / `total_net_repl_output_bytes`
     - `instantaneous_repl_total_kbps`
- 1 new API `rioCheckType()` to check the type of rio. So we can use this
     to distinguish between diskless and diskbased replication
- 2 new counting items to keep network statistics consistent between master
     and slave
    - rdb portion during diskless replica. in `rdbLoadProgressCallback()`
    - first line of the full sync payload. in `readSyncBulkPayload()`

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-05-31 08:07:33 +03:00
Harkrishn Patro 4065b4f27e Sharded pubsub publish messagebulk as smessage (#10792) 
To easily distinguish between sharded channel message and a global
channel message, introducing `smessage` (instead of `message`) as
message bulk for sharded channel publish message.

This is gonna be a breaking change in 7.0.1!

Background:
Sharded pubsub introduced in redis 7.0, but after the release we quickly
realized that the fact that it's problematic that the client can't distinguish
between normal (global) pubsub messages and sharded ones.
This is important because the same connection can subscribe to both,
but messages sent to one pubsub system are not propagated to the
other (they're completely separate), so if one connection is used to
subscribe to both, we need to assist the client library to know which
message it got so it can forward it to the correct callback.
 2022-05-31 08:03:59 +03:00
Wen HuiUbuntuguybe7
 		d7ae858745 Fix streamParseAddOrTrimArgsOrReply function minor comment issue (#10783) 
When I read the source codes, I have no idea where the option "age" come from.


Co-authored-by: Ubuntu <lucas.guang.yang1@huawei.com>
Co-authored-by: guybe7 <guy.benoish@redislabs.com>
 2022-05-31 07:57:59 +03:00
Madelyn Olson ed29d634b3 Add readonly flag to EVAL_RO, EVALSHA_RO and FCALL_RO (#10728) 
* Add readonly flag to EVAL_RO, EVALSHA_RO and FCALL_RO
* Require users to explicitly declare @scripting to get access to lua scripting.
 2022-05-29 23:42:56 -07:00
skygragon 7a550c8bbb fixes function cache_memory accounting stats bug (#10791) 
This bug resulted in wrong memory usage statistics after a redis function library is removed.

Signed-off-by: skygragon <skygragon@gmail.com>
 2022-05-29 22:45:56 +03:00
Binbin f18c9da3e1 loadAppendOnlyFiles and loadSingleAppendOnlyFile ret should be AOF_OK, not C_OK (#10790) 
The ret value should be AOF_OK instead of C_OK.
AOF_OK and C_OK are both 0, so this is just a cleanup.
Also updated some outdated comments.
 2022-05-29 13:33:16 +03:00
Lu JJ 2f5edd03e6 fix typo in quicklist.c (#10785) 
fix typo ` the largest possible limit is 16k` -> ` the largest possible limit is 64k`.
The count field is 16 bits so the largest possible limit is 64k(2**16).
 2022-05-29 08:51:33 +03:00
Binbin 1013cbeae2 Fix sentinel disconnect test timing issue after auth-pass change (#10784) 
There is a timing issue reported in test-sanitizer-address (gcc):
```
Sentinels (re)connection following SENTINEL SET mymaster auth-pass:
FAILED: Expected to be disconnected from master due to wrong password
```

The reason we reach it, is because the test is fast enough to modify
auth-pass and test sentinel connection status with the server,
before its scheduled operation got the chance to update connection
status with the server.

We need to wait for `sentinelTimer` to kick in, and then update the
connection status. Replace condition with wait_for_condition on the check.

Fix just like #10480 did
 2022-05-29 08:38:38 +03:00
Vitaly 6461f09f43 Fix ZRANGESTORE crash when zset_max_listpack_entries is 0 (#10767) 
When `zrangestore` is called container destination object is created. 
Before this PR we used to create a listpack based object even if `zset-max-ziplist-entries`
or equivalent`zset-max-listpack-entries` was set to 0.
This triggered immediate conversion of the listpack into a skiplist in `zrangestore`, which hits
an assertion resulting in an engine crash.

Added a TCL test that reproduces this issue.
 2022-05-27 22:34:00 +03:00
Binbin 2a099d49d4 Fix some commands key spec in json files (#10779) 
There are some commands that has the wrong key specs.
This PR adds a key-spec related check in generate-command-code.py.
Check if the index is valid, or if there is an unused index.

The check result will look like:
```
[root]# python utils/generate-command-code.py
Processing json files...
Linking container command to subcommands...
Checking all commands...
command: RESTORE_ASKING may have unused key_spec
command: RENAME may have unused key_spec
command: PFDEBUG may have unused key_spec
command: WATCH key_specs missing flags
command: LCS arg: key2 key_spec_index error
command: RENAMENX may have unused key_spec
Error: There are errors in the commands check, please check the above logs.
```

The following commands have been fixed according to the check results:
- RESTORE ASKING: add missing arguments section (and history section)
- RENAME: newkey's key_spec_index should be 1
- PFDEBUG: add missing arguments (and change the arity from -3 to 3)
- WATCH: add missing key_specs flags: RO, like EXIST (it allow you to know the key exists, or is modified, but doesn't "leak" the data)
- LCS: key2 key_spec_index error, there is only one key-spec
- RENAMENX: newkey's key_spec_index should be 1
 2022-05-27 12:58:00 +03:00
Binbin 6f7c1a8ce6 Fix outdated comment about flags in moduleCreateArgvFromUserFormat (#10781) 
Clearly more than one flag exists, also fixed some typos.
Fixes #10776
 2022-05-26 17:34:17 +03:00
chenyang8094 f28e2ce7a4 improve logging around AOF file creation and loading (#10763) 
instead of printing a log when a folder or a manifest is missing (level reduced), we print:

total time it took to load all the aof files
when creating a new base or incr file
starting to write to an existing incr file on startup
 2022-05-26 17:23:05 +03:00
Meir Shpilraien (Spielrein) ad25716a75 Added the function name/script sha to the script timeout log message. (#10780) 
Added the function name/script sha to the script timeout log message.
This info existed in the log in redis 6.2, was removed in the function refactoring
since was initially complicated, but later made simple.
 2022-05-26 16:11:24 +03:00
Valentino Geron 9eb97b5d94 Fix regex support in --only, --skipfile and --skiptest (#10741) 
The regex support was added in:
 * https://github.com/redis/redis/pull/9352
 * https://github.com/redis/redis/pull/9555
 * https://github.com/redis/redis/pull/10212

These commits break backword compatiblity with older versions.

This fix keeps the test suite infra compatible with old versions by
default. However, if you want regex, the string must start with `/`
 2022-05-25 18:25:38 +03:00
Binbin 450c88f368 Fix BZMPOP gets unblocked by non-key args and returns them (#10764) 
This bug was introduced in #9484 (7.0.0).
It result that BZMPOP blocked on non-key arguments.

Like `bzmpop 0 1 myzset min count 10`, this command will additionally
block in these keys (except for the first and the last argument) and can return their values:
- 0: timeout value
- 1: numkeys value
- min: min/max token
- count: count token
 2022-05-23 14:15:54 +03:00
yoav-steinberg 843a4cdc07 Add warning for suspected slow system clocksource setting (#10636) 
This PR does 2 main things:
1) Add warning for suspected slow system clocksource setting. This is Linux specific.
2) Add a `--check-system` argument to redis which runs all system checks and prints a report.

## System checks
Add a command line option `--check-system` which runs all known system checks and provides
a report to stdout of which systems checks have failed with details on how to reconfigure the
system for optimized redis performance.
The `--system-check` mode exists with an appropriate error code after running all the checks.

## Slow clocksource details
We check the system's clocksource performance by running `clock_gettime()` in a loop and then
checking how much time was spent in a system call (via `getrusage()`). If we spend more than
10% of the time in the kernel then we print a warning. I verified that using the slow clock sources:
`acpi_pm` (~90% in the kernel on my laptop) and `xen` (~30% in the kernel on an ec2 `m4.large`)
we get this warning.

The check runs 5 system ticks so we can detect time spent in kernel at 20% jumps (0%,20%,40%...).
Anything more accurate will require the test to run longer. Typically 5 ticks are 50ms. This means
running the test on startup will delay startup by 50ms. To avoid this we make sure the test is only
executed in the `--check-system` mode.

For a quick startup check, we specifically warn if the we see the system is using the `xen` clocksource
which we know has bad performance and isn't recommended (at least on ec2). In such a case the
user should manually rung redis with `--check-system` to force the slower clocksource test described
above.

## Other changes in the PR

* All the system checks are now implemented as functions in _syscheck.c_.
  They are implemented using a standard interface (see details in _syscheck.c_).
  To do this I moved the checking functions `linuxOvercommitMemoryValue()`,
  `THPIsEnabled()`, `linuxMadvFreeForkBugCheck()` out of _server.c_ and _latency.c_
  and into the new _syscheck.c_. When moving these functions I made sure they don't
  depend on other functionality provided in _server.c_ and made them use a standard
  "check functions" interface. Specifically:
  * I removed all logging out of `linuxMadvFreeForkBugCheck()`. In case there's some
    unexpected error during the check aborts as before, but without any logging.
    It returns an error code 0 meaning the check didn't not complete.
  * All these functions now return 1 on success, -1 on failure, 0 in case the check itself
    cannot be completed.
  * The `linuxMadvFreeForkBugCheck()` function now internally calls `exit()` and not
    `exitFromChild()` because the latter is only available in _server.c_ and I wanted to
    remove that dependency. This isn't an because we don't need to worry about the
    child process created by the test doing anything related to the rdb/aof files which
    is why `exitFromChild()` was created.

* This also fixes parsing of other /proc/\<pid\>/stat fields to correctly handle spaces
  in the process name and be more robust in general. Not that before this fix the rss
  info in `INFO memory` was corrupt in case of spaces in the process name. To
  recreate just rename `redis-server` to `redis server`, start it, and run `INFO memory`.
 2022-05-22 17:10:31 +03:00
Oran Agra b0e18f804d Scripts that declare the no-writes flag are implicitly allow-oom too. (#10699) 
Scripts that have the `no-writes` flag, cannot execute write commands,
and since all `deny-oom` commands are write commands, we now act
as if the `allow-oom` flag is implicitly set for scripts that set the `no-writes` flag.
this also implicitly means that the EVAL*_RO and FCALL_RO commands can
never fails with OOM error.

Note about a bug that's no longer relevant:
There was an issue with EVAL*_RO using shebang not being blocked correctly
in OOM state:
When an EVAL script declares a shebang, it was by default not allowed to run in
OOM state.
but this depends on a flag that is updated before the command is executed, which
was not updated in case of the `_RO` variants.
the result is that if the previous cached state was outdated (either true or false),
the script will either unjustly fail with OOM, or unjustly allowed to run despite
the OOM state.
It doesn't affect scripts without a shebang since these depend on the actual
commands they run, and since these are only read commands, they don't care
for that cached oom state flag.
it did affect scripts with shebang and no allow-oom flag, bug after the change in
this PR, scripts that are run with eval_ro would implicitly have that flag so again
the cached state doesn't matter.

p.s. this isn't a breaking change since all it does is allow scripts to run when they
should / could rather than blocking them.
 2022-05-22 16:02:59 +03:00
yoav-steinberg cb6933e346 Updated HDR histogram from upsteam after they merged our fix in #10606. (#10755) 
The code is based on upstream https://github.com/HdrHistogram/HdrHistogram_c
master branch latest commit (e4448cf6d1cd08fff519812d3b1e58bd5a94ac42).
The reason to pull this in now is that their final version of our optimization is even faster.
See: https://github.com/HdrHistogram/HdrHistogram_c/pull/107.
 2022-05-22 13:44:29 +03:00
Binbin 18cb4a7d93 Remove ziplist dead code in object.c (#10751) 
Remove some dead code in object.c, ziplist is no longer used in 7.0

Some backgrounds:
zipmap - hash: replaced by ziplist in #285
ziplist - hash: replaced by listpack in #8887
ziplist - zset: replaced by listpack in #9366
ziplist - list: replaced by quicklist (listpack) in #2143 / #9740

Moved the location of ziplist.h in the server.c
 2022-05-22 12:27:54 +03:00
Yuuoniy 4a7a4e42db Fix memory leak in streamGetEdgeID (#10753) 
si is initialized by streamIteratorStart(), we should call
streamIteratorStop() on it when done.

regression introduced in #9127 (redis 7.0)
 2022-05-22 12:15:26 +03:00
Ofir Luzon 00a9d6b314 Add SIGINT handler to redis-cli --bigkeys, --memkeys, --hotkeys, --scan (#10736) 
Finish current loop and display the scanned keys summery on SIGINT (Ctrl-C) signal.
It will also prepend the current scanned percentage to the scanned keys summery 1st line.

In this commit I've renamed and relocated `intrinsicLatencyModeStop` function as I'm using the exact same logic.
 2022-05-22 10:55:26 +03:00
Binbin 60250f50c2 Fix typos in module comment / documentation (#10740) 
minor cleanup in redismodule.h and module.c
 2022-05-18 08:29:39 +03:00
Ozan Tezcan f62d52a5ad Add const qualifier to config name parameter in RM_RegisterBoolConfig() (#10733) 
was present in the C file and missing in the header.
 2022-05-16 14:53:38 +03:00
Qu Chen 837c063baf Replica fail and retry the PSYNC if the master is unresponsive (#10726) 
We observed from our replication testing that when the master becomes unresponsive,
or the replication connection is broken during PSYNC so the replica doesn't get a
response from the master, it was not able to recognize that condition as a failure
and instead moved into the full-sync code path. This fix makes the replica fail and
retry the PSYNC with the master in such scenarios.
 2022-05-16 12:08:19 +03:00
Qu Chen a7d6ca9770 Make the check for if script is running or not consistent (#10725) 
sometimes it is using `scriptIsRunning()` and other times it is using `server.in_script`.
We should use the `scriptIsRunning()` method consistently throughout the code base.
Removed server.in_script sine it's no longer used / needed.
 2022-05-15 14:07:45 +03:00
Tian 4b262182d9 Remove a redundant free in freeClient (#10721) 2022-05-14 19:38:26 -07:00
Stephen Sullivan acd517c883 re-add SENTINEL SLAVES command, missing in redis 7.0 (#10723) 
Alias was mistakenly forgotten when the sub commands introduced as json files.
 2022-05-13 19:47:58 +03:00
Wen Hui 135998ed8d Update comments on command args, and a misleading error reply (#10645) 
Updated the comments for:
info command
lmpopCommand and blmpopCommand
sinterGenericCommand 

Fix the missing "key" words in the srandmemberCommand function
For LPOS command, when rank is 0, prompt user that rank could be
positive number or negative number, and add a test for it
 2022-05-13 17:55:49 +03:00
Binbin 586a16ad79 Fix race in module fork kill test (#10717) 
The purpose of the test is to kill the child while it is running.
From the last two lines we can see the child exits before being killed.
```
- Module fork started pid: 56998
* <fork> fork child started
- Killing running module fork child: 56998
* <fork> fork child exiting
signal-handler (1652267501) Received SIGUSR1 in child, exiting now.
```

In this commit, we pass an argument to `fork.create` indicating how
long it should sleep. For the fork kill test, we use a longer time to
avoid the child exiting before being killed.

Other changes:
use wait_for_condition instead of hardcoded `after 250`.
Unify the test for failing fork with the one for killing it (save time)
 2022-05-12 20:10:38 +03:00
yoav-steinberg b16d1c2713 Fix possible regression around TLS config changes. Add VOLATILE_CONFIG flag for volatile configurations. (#10713) 
This fixes a possible regression in Redis 7.0.0, in which doing CONFIG SET
on a TLS config would not reload the configuration in case the new config is
the same file as before.

A volatile configuration is a configuration value which is a reference to the
configuration data and not the configuration data itself. In such a case Redis
doesn't know if the config data changed under the hood and can't assume a
change happens only when the config value changes. Therefore it needs to
be applied even when setting a config value to the same value as it was before.
 2022-05-12 17:06:26 +03:00
Yossi Gottlieb b414605285 Update security page with supported versions. (#10712) 2022-05-11 16:18:02 +03:00
Yossi Gottlieb 8bdd2d5ddd Fix Makefile.dep generation with ICC. (#10708) 
Before this commit, all source files including those that are not going
to be compiled were used. Some of these files are platform specific and
won't even pre-process on another platform. With GCC/Clang, that's not
an issue and they'll simply ignore them, but ICC aborts in this case.

This commit only attempts to generate Makefile.dep from the actual set
of C source files that will be compiled.
 2022-05-11 12:06:33 +03:00
Binbin bfbb15f75d redis-server command line arguments support take one bulk string with spaces for MULTI_ARG configs parsing. And allow options value to use the -- prefix (#10660) 
## Take one bulk string with spaces for MULTI_ARG configs parsing
Currently redis-server looks for arguments that start with `--`,
and anything in between them is considered arguments for the config.
like: `src/redis-server --shutdown-on-sigint nosave force now --port 6380`

MULTI_ARG configs behave differently for CONFIG command, vs the command
line argument for redis-server.
i.e. CONFIG command takes one bulk string with spaces in it, while the
command line takes an argv array with multiple values.

In this PR, in config.c, if `argc > 1` we can take them as is,
and if the config is a `MULTI_ARG` and `argc == 1`, we will split it by spaces.

So both of these will be the same:
```
redis-server --shutdown-on-sigint nosave force now --shutdown-on-sigterm nosave force
redis-server --shutdown-on-sigint nosave "force now" --shutdown-on-sigterm nosave force
redis-server --shutdown-on-sigint nosave "force now" --shutdown-on-sigterm "nosave force"
```

## Allow options value to use the `--` prefix
Currently it decides to switch to the next config, as soon as it sees `--`, 
even if there was not a single value provided yet to the last config,
this makes it impossible to define a config value that has `--` prefix in it.

For instance, if we want to set the logfile to `--my--log--file`,
like `redis-server --logfile --my--log--file --loglevel verbose`,
current code will handle that incorrectly.

In this PR, now we allow a config value that has `--` prefix in it.
**But note that** something like `redis-server --some-config --config-value1 --config-value2 --loglevel debug`
would not work, because if you want to pass a value to a config starting with `--`, it can only be a single value.
like: `redis-server --some-config "--config-value1 --config-value2" --loglevel debug`

An example (using `--` prefix config value):
```
redis-server --logfile --my--log--file --loglevel verbose
redis-cli config get logfile loglevel
1) "loglevel"
2) "verbose"
3) "logfile"
4) "--my--log--file"
```

### Potentially breaking change
`redis-server --save --loglevel verbose` used to work the same as `redis-server --save "" --loglevel verbose`
now, it'll error!
 2022-05-11 11:33:35 +03:00
Binbin 783b210db4 FLUSHDB and FLUSHALL add call forceCommandPropagation / FLUSHALL reset dirty counter to 0 if we enable save (#10691) 
## FLUSHALL
We used to restore the dirty counter after `rdbSave` zeroed it if we enable save.
Otherwise FLUSHALL will not be replicated nor put into the AOF.

And then we do increment it again below.
Without that extra dirty++, when db was already empty, FLUSHALL
will not be replicated nor put into the AOF.

We now gonna replace all that dirty counter magic with a call
to forceCommandPropagation (REPL and AOF), instead of all the
messing around with the dirty counter.
Added tests to cover three part (dirty counter, REPL, AOF).

One benefit other than cleaner code is that the `rdb_changes_since_last_save` is correct in this case.

## FLUSHDB
FLUSHDB was not replicated nor put into the AOF when db was already empty.
Unlike DEL on a non-existing key, FLUSHDB always does something, and that's to call the module hook. 
So basically FLUSHDB is never a NOP, and thus it should always be propagated.
Not doing that, could mean that if a module does something in that hook, and wants to
avoid issues of that hook being missing on the replica if the db is empty, it'll need to do complicated things.

So now FLUSHDB add call forceCommandPropagation, we will always propagate FLUSHDB.
Always propagating FLUSHDB seems like a safe approach that shouldn't have any drawbacks (other than looking odd)

This was mentioned in #8972

## Test section:
We actually found it while solving a race condition in the BGSAVE test (other.tcl).
It was found in extra_ci Daily Arm64 (test-libc-malloc).
```
[exception]: Executing test client: ERR Background save already in progress.
ERR Background save already in progress
```

It look like `r flushdb` trigger (schedule) a bgsave right after `waitForBgsave r` and before `r save`.
Changing flushdb to flushall, FLUSHALL will do a foreground save and then set the dirty counter to 0.
 2022-05-11 11:21:16 +03:00
guybe7 815a6f846a Dediacted member to hold RedisModuleCommand (#10681) 
Fix #10552

We no longer piggyback getkeys_proc to hold the RedisModuleCommand struct, when exists

Others:
Use `doesCommandHaveKeys` in `RM_GetCommandKeysWithFlags` and `getKeysSubcommandImpl`.
It causes a very minor behavioral change in commands that don't have actual keys, but have a spec
with `CMD_KEY_NOT_KEY`.
For example, before this command `COMMAND GETKEYS SPUBLISH` would return
`Invalid arguments specified for command` but not it returns `The command has no key arguments`
 2022-05-10 14:56:12 +03:00
Mariya Markova c2d8d4e648 Replace float zero comparison to FP_ZERO comparison (#10675) 
I suggest to use "[fpclassify](https://en.cppreference.com/w/cpp/numeric/math/fpclassify)" for float
comparison with zero, because of expression "value == 0" with value very close to zero can be
considered as true with some performance compiler optimizations.

Note: this code was introduced by 9d520a7f to accept zset scores that get ERANGE in conversion
due to precision loss near 0.
But with Intel compilers, ICC and ICX, where optimizations for 0 check are more aggressive, "==0" is
true for mentioned functions, however should not be. Behavior is seen starting from O2.
This leads to a failure in the ZSCAN test in scan.tcl
 2022-05-10 14:55:09 +03:00
Binbin 2a1ea8c7d8 CLUSTER SHARDS should returns slots as integers, not strings (#10683) 
It used to returns slots as strings, like:
```
redis> cluster shards
1) 1) "slots"
   2) 1) "10923"
      2) "16383"
```

CLUSTER SHARDS docs and the top comment of #10293 says that it returns integers.
Note other commands like CLUSTER SLOTS, it returns slots as integers.
Use addReplyLongLong instead of addReplyBulkLongLong, now it returns slots as integers:
```
redis> cluster shards
1) 1) "slots"
   2) 1) (integer) 10923
      2) (integer) 16383
```

This is a small breaking change, introduced in 7.0.0 (7.0 RC3, #10293)

Fixes #10680
 2022-05-10 14:22:01 +03:00
Meir Shpilraien (Spielrein) 442e73ea09 Fix #10705, avoid relinking the same library twice. (#10706) 
Set `old_li` to NULL to avoid linking it again on error.
Before the fix, loading an already existing library will cause the existing library to be added again. This cause not harm other then wrong statistics. The statistics that are effected  by the issue are:
* `libraries_count` and `functions_count` returned by `function stats` command
* `used_memory_functions` returned on `info memory` command
* `functions.caches` returned on `memory stats` command
 2022-05-10 11:47:45 +03:00
Ozan Tezcan a3df2777e8 Fix cursor type in RedisModuleScanCursor (#10698) 
Changed cursor's type from `int` to `unsigned long`
allows handling database or key with more than 2^31 elements
 2022-05-09 18:45:51 +03:00
Lu JJ 6b44e4ea92 fix some typos in "t_zset.c" (#10670) 
fix some typo in "t_zset.c".
1. `zzlisinlexrange` the function name mentioned in the comment is misspelled.
2. fix typo in function name`zarndmemberReplyWithListpack` -> `zrandmemberReplyWithListpack`
 2022-05-09 15:04:39 +03:00
Oran Agra 2bcd890d8a Fix --save command line regression in redis 7.0.0 (#10690) 
Unintentional change in #9644 (since RC1) meant that an empty `--save ""` config
from command line, wouldn't have clear any setting from the config file

Added tests to cover that, and improved test infra to take additional
command line args for redis-server
 2022-05-09 13:37:49 +03:00
Oran Agra eb915a82a5 Bug fixes for enum configs with overlapping bit flags (module API) (#10661) 
If we want to support bits that can be overlapping, we need to make sure
that:
1. we don't use the same bit for two return values.
2. values should be sorted so that prefer ones (matching more
   bits) come first.
 2022-05-09 13:36:53 +03:00
Ozan Tezcan 8fc959216c Fix RM_Scan() documentation (#10693) 
Fixed RM_Scan() usage example: `RedisModuleCursor` -> `RedisModuleScanCursor`
 2022-05-09 12:38:45 +03:00
David CARLIERandOran Agra bdcd4b3df8 zmalloc_get_rss implementation for haiku. (#10687) 
also fixing already defined constants build warning while at it.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-05-08 15:12:17 +03:00
Shaya Potter 4e761eb7e2 update redismodule notify defines to be in sync with server (#10688) 
this seems to have been an oversight. syncing the flags so that NOTIFY_NEW is available to modules.
missing in #10512
 2022-05-08 15:05:24 +03:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> ff3a3577f2 Bump github/codeql-action from 1 to 2 (#10635) 
* Bump github/codeql-action from 1 to 2

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 1 to 2.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/v1...v2)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

* Avoid CodeQL on push error.

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
 2022-05-04 11:40:08 +03:00
Viktor SöderqvistandOran Agra ced710fc83 Module API doc script: Mark unreleased API functions (#10674) 
* Module API doc script: Mark unreleased API functions

* fix broken quotes in generate-module-api-doc.rb

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-05-03 10:34:18 +03:00
Meir Shpilraien (Spielrein) f44c343292 Expose Lua error in case of string error. (#10677) 
In general, our error handler make sure the error
object is always a table. In some rare cases (such
as OOM error), the error handler will not be called
and the error object will be a string. The PR expose
the error even if its a string and not a table.

Currently there is no way to test it but if it'll ever happen,
it is better to propagate this string upwards than just
generate a generic error without any specific info.
 2022-05-03 10:24:05 +03:00
Lu JJandOran Agra 87131a5fa6 fast path when SDIFF command has the same key as the first key (#10663) 
When user uses the same input key for SDIFF as the first one, the result must be empty, so we don't need to process the elements to test.

This method is like the one done in zset‘s `zsetChooseDiffAlgorithm`

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-05-02 16:18:11 +03:00
wutao_water 9c39256a28 cleanup: use ZIPLIST_ENTRY_END macro instead of 1 (#3672) 
update macros ZIPLIST_ENTRY_END i think the right definition is ((zl)+intrev32ifbe(ZIPLIST_BYTES(zl))-ZIPLIST_END_SIZE)
 2022-05-02 12:46:48 +03:00
David CARLIER ef68deb3c2 support tcp-keepalive config interval on macOs (#10667) 
Till now, on MacOS we only used to enable SO_KEEPALIVE,
but we didn't set the interval which is configurable via the `tcp-keepalive` config.
This adds support for that on MacOS, to match what we already do on Linux.
 2022-05-02 09:37:14 +03:00
Lu JJ 1666ffbe1f fix typo in 'setTypeRandomElement' (#10662) 
`the redis object pointer was populated.` -> `the sds pointer was populated.`
We don't populate the redis object pointer in this function.
 2022-05-01 12:09:44 +03:00
Lu JJandOran Agra 02080f2686 add comment to 'container' in 'quicklist.h' (#10656) 
add a comment to `container` in `quicklist.h`.
Because `PLAIN` and `PACKED` are not as easy to understand as `NONE`
and `LISTPACK` and we don't have a detailed comment on it.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-04-28 08:36:40 +03:00
Itamar Haber 6c65345edb Injects Hugo FrontMatter to module-api.md (#10658) 2022-04-28 08:16:20 +03:00
Wen Hui f36eac9f68 Update the comments of commands introduced or updated in redis 7.0 (#10659) 2022-04-28 08:13:04 +03:00
Oran Agra d375595d5e Merge pull request #10652 from oranagra/redis-7.0.0
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CodeQL / Analyze (cpp) (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details 
Redis 7.0.0
 2022-04-27 16:32:17 +03:00
Oran Agra c1f3020631 Redis 7.0.0 GA 2022-04-27 12:56:07 +03:00
Oran Agra 20618c713c Merge remote-tracking branch 'origin/unstable' into 7.0 2022-04-27 12:55:56 +03:00
Oran Agra 89772ed827 Merge pull request #10651 from oranagra/meir_lua_readonly_tables 
# Lua readonly tables
The PR adds support for readonly tables on Lua to prevent security vulnerabilities:
* (CVE-2022-24736) An attacker attempting to load a specially crafted Lua script
  can cause NULL pointer dereference which will result with a crash of the
  redis-server process. This issue affects all versions of Redis.
* (CVE-2022-24735) By exploiting weaknesses in the Lua script execution
  environment, an attacker with access to Redis can inject Lua code that will
  execute with the (potentially higher) privileges of another Redis user.

The PR is spitted into 4 commits.

### Change Lua to support readonly tables

This PR modifies the Lua interpreter code to support a new flag on tables. The new flag indicating that the table is readonly and any attempt to perform any writes on such a table will result in an error. The new feature can be turned off and on using the new `lua_enablereadonlytable` Lua API. The new API can be used **only** from C code. Changes to support this feature was taken from https://luau-lang.org/

### Change eval script to set user code on Lua registry

Today, Redis wrap the user Lua code with a Lua function. For example, assuming the user code is:

```
return redis.call('ping')
```

The actual code that would have sent to the Lua interpreter was:

```
f_b3a02c833904802db9c34a3cf1292eee3246df3c() return redis.call('ping') end
```

The warped code would have been saved on the global dictionary with the following name: `f_<script sha>` (in our example `f_b3a02c833904802db9c34a3cf1292eee3246df3c`). This approach allows one user to easily override the implementation of another user code, example:

```
f_b3a02c833904802db9c34a3cf1292eee3246df3c = function() return 'hacked' end
```

Running the above code will cause `evalsha b3a02c833904802db9c34a3cf1292eee3246df3c 0` to return `hacked` although it should have returned `pong`. Another disadvantage is that Redis basically runs code on the loading (compiling) phase without been aware of it. User can do code injection like this:

```
return 1 end <run code on compling phase> function() return 1
```

The warped code will look like this and the entire `<run code on compiling phase>` block will run outside of eval or evalsha context:

```
f_<sha>() return 1 end <run code on compling phase> function() return 1 end
```

The commits puts the user code on a special Lua table called the registry. This table is not accessible to the user so it can not be manipulated by him. Also there is no longer a need to warp the user code so there is no risk in code injection which will cause running code in the wrong context.

### Use `lua_enablereadonlytable` to protect global tables on eval and function

The commit uses the new `lua_enablereadonlytable` Lua API to protect the global tables of both evals scripts and functions. For eval scripts, the implementation is easy, We simply call `lua_enablereadonlytable` on the global table to turn it into a readonly table.

On functions its more complected, we want to be able to switch globals between load run and function run. To achieve this, we create a new empty table that acts as the globals table for function, we control the actual globals using metatable manipulations. Notice that even if the user gets a pointer to the original tables, all the tables are set to be readonly (using `lua_enablereadonlytable` Lua API) so he can not change them. The following better explains the solution:

```
Global table {} <- global table metatable {.__index = __real_globals__}
```

The `__real_globals__` is depends on the run context (function load or function call).

Why is this solution needed and its not enough to simply switch globals? When we run in the context of function load and create our functions, our function gets the current globals that was set when they were created. Replacing the globals after the creation will not effect them. This is why this trick it mandatory.

### Protect the rest of the global API and add an allowed list to the provided API

The allowed list is done by setting a metatable on the global table before initialising any library. The metatable set the `__newindex` field to a function that check the allowed list before adding the field to the table. Fields which is not on the
allowed list are simply ignored.

After initialisation phase is done we protect the global table and each table that might be reachable from the global table. For each table we also protect the table metatable if exists.

### Performance

Performance tests was done on a private computer and its only purpose is to show that this fix is not causing any performance regression.

case 1: `return redis.call('ping')`
case 2: `for i=1,10000000 do redis.call('ping') end`

|                             | Unstable eval | Unstable function | lua_readonly_tables eval | lua_readonly_tables function |
|-----------------------------|---------------|-------------------|--------------------------|------------------------------|
| case1 ops/sec               | 235904.70     | 236406.62         | 232180.16               | 230574.14                   |
| case1 avg latency ms        | 0.175         | 0.164             | 0.178                    | 0.149                        |
| case2 total time in seconds | 3.373         | 3.444s            | 3.268                   | 3.278                        |

### Breaking changes

* `print` function was removed from Lua because it can potentially cause the Redis processes to get stuck (if no one reads from stdout). Users should use redis.log. An alternative is to override the `print` implementation and print the message to the log file.

All the work by @MeirShpilraien, i'm just publishing it.
 2022-04-27 12:48:51 +03:00
chenyang8094 e24d46004b Delete renamed new incr when write manifest failed (#10649) 
Followup fix for #10616
 2022-04-27 08:07:52 +03:00
meir efa162bcd7 Protect any table which is reachable from globals and added globals white list. 
The white list is done by setting a metatable on the global table before initializing
any library. The metatable set the `__newindex` field to a function that check
the white list before adding the field to the table. Fields which is not on the
white list are simply ignored.

After initialization phase is done we protect the global table and each table
that might be reachable from the global table. For each table we also protect
the table metatable if exists.
 2022-04-27 00:37:40 +03:00
meir 3731580b6b Protect globals of both evals scripts and functions. 
Use the new `lua_enablereadonlytable` Lua API to protect the global tables of
both evals scripts and functions. For eval scripts, the implemetation is easy,
We simply call `lua_enablereadonlytable` on the global table to turn it into
a readonly table.

On functions its more complecated, we want to be able to switch globals between
load run and function run. To achieve this, we create a new empty table that
acts as the globals table for function, we control the actual globals using metatable
manipulation. Notice that even if the user gets a pointer to the original tables, all
the tables are set to be readonly (using `lua_enablereadonlytable` Lua API) so he can
not change them. The following inlustration better explain the solution:

```
Global table {} <- global table metatable {.__index = __real_globals__}
```

The `__real_globals__` is set depends on the run context (function load or function call).

Why this solution is needed and its not enough to simply switch globals?
When we run in the context of function load and create our functions, our function gets
the current globals that was set when they were created. Replacing the globals after
the creation will not effect them. This is why this trick it mandatory.
 2022-04-27 00:37:40 +03:00
meir 992f9e23c7 Move user eval function to be located on Lua registry. 
Today, Redis wrap the user Lua code with a Lua function.
For example, assuming the user code is:

```
return redis.call('ping')
```

The actual code that would have sent to the Lua interpreter was:

```
f_b3a02c833904802db9c34a3cf1292eee3246df3c() return redis.call('ping') end
```

The wraped code would have been saved on the global dictionary with the
following name: `f_<script sha>` (in our example `f_b3a02c833904802db9c34a3cf1292eee3246df3c`).

This approach allows one user to easily override the implementation a another user code, example:

```
f_b3a02c833904802db9c34a3cf1292eee3246df3c = function() return 'hacked' end
```

Running the above code will cause `evalsha b3a02c833904802db9c34a3cf1292eee3246df3c 0` to return
hacked although it should have returned `pong`.

Another disadventage is that Redis basically runs code on the loading (compiling) phase without been
aware of it. User can do code injection like this:

```
return 1 end <run code on compling phase> function() return 1
```

The wraped code will look like this and the entire `<run code on compling phase>` block will run outside
of eval or evalsha context:

```
f_<sha>() return 1 end <run code on compling phase> function() return 1 end
```
 2022-04-27 00:20:54 +03:00
meir 8b33d813a3 Added support for Lua readonly tables. 
The new feature can be turned off and on using the new `lua_enablereadonlytable` Lua API.
 2022-04-27 00:20:54 +03:00
Oran Agra 8192625458 Add module API flag for using enum configs as bit flags (#10643) 
Enables registration of an enum config that'll let the user pass multiple keywords that
will be combined with `|` as flags into the integer config value.

```
    const char *enum_vals[] = {"none", "one", "two", "three"};
    const int int_vals[] = {0, 1, 2, 4};

    if (RedisModule_RegisterEnumConfig(ctx, "flags", 3, REDISMODULE_CONFIG_DEFAULT | REDISMODULE_CONFIG_BITFLAGS, enum_vals, int_vals, 4, getFlagsConfigCommand, setFlagsConfigCommand, NULL, NULL) == REDISMODULE_ERR) {
        return REDISMODULE_ERR;
    }
```
doing:
`config set moduleconfigs.flags "two three"` will result in 6 being passed to`setFlagsConfigCommand`.
 2022-04-26 20:29:20 +03:00
chenyang8094 46ec6ad98e Fix bug when AOF enabled after startup. put the new incr file in the manifest only when AOFRW is done. (#10616) 
Changes:

- When AOF is enabled **after** startup, the data accumulated during `AOF_WAIT_REWRITE`
  will only be stored in a temp INCR AOF file. Only after the first AOFRW is successful, we will
  add it to manifest file.
  Before this fix, the manifest referred to the temp file which could cause a restart during that
  time to load it without it's base.
- Add `aof_rewrites_consecutive_failures` info field for  aofrw limiting implementation.

Now we can guarantee that these behaviors of MP-AOF are the same as before (past redis releases):
- When AOF is enabled after startup, the data accumulated during `AOF_WAIT_REWRITE` will only
  be stored in a visible place. Only after the first AOFRW is successful, we will add it to manifest file.
- When disable AOF, we did not delete the AOF file in the past so there's no need to change that
  behavior now (yet).
- When toggling AOF off and then on (could be as part of a full-sync), a crash or restart before the
  first rewrite is completed, would result with the previous version being loaded (might not be right thing,
  but that's what we always had).
 2022-04-26 16:31:19 +03:00
Eduardo SemprebonandOran Agra 3a1d14259d Allow configuring signaled shutdown flags (#10594) 
The SHUTDOWN command has various flags to change it's default behavior,
but in some cases establishing a connection to redis is complicated and it's easier
for the management software to use signals. however, so far the signals could only
trigger the default shutdown behavior.
Here we introduce the option to control shutdown arguments for SIGTERM and SIGINT.

New config options:
`shutdown-on-sigint [nosave | save] [now] [force]` 
`shutdown-on-sigterm [nosave | save] [now] [force]`

Implementation:
Support MULTI_ARG_CONFIG on createEnumConfig to support multiple enums to be applied as bit flags.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-04-26 14:34:04 +03:00
Binbin 156836bfe5 Fix syntax error in replicationErrorBehavior enum (#10642) 
Missing a typeof, we will get errors like this:
- multiple definition of `replicationErrorBehavior'
- ld: error: duplicate symbol: replicationErrorBehavior

Introduced in #10504
 2022-04-26 14:29:05 +03:00
Madelyn Olson 6fa8e4f7af Set replicas to panic on disk errors, and optionally panic on replication errors (#10504) 
* Till now, replicas that were unable to persist, would still execute the commands
  they got from the master, now they'll panic by default, and we add a new
  `replica-ignore-disk-errors` config to change that.
* Till now, when a command failed on a replica or AOF-loading, it only logged a
  warning and a stat, we add a new `propagation-error-behavior` config to allow
  panicking in that state (may become the default one day)

Note that commands that fail on the replica can either indicate a bug that could
cause data inconsistency between the replica and the master, or they could be
in some cases (specifically in previous versions), a result of a command (e.g. EVAL)
that failed on the master, but still had to be propagated to fail on the replica as well.
 2022-04-26 13:25:33 +03:00
Madelyn Olson efcd1bf394 By default prevent cross slot operations in functions and scripts with # (#10615) 
Adds the `allow-cross-slot-keys` flag to Eval scripts and Functions to allow
scripts to access keys from multiple slots.
The default behavior is now that they are not allowed to do that (unlike before).
This is a breaking change for 7.0 release candidates (to be part of 7.0.0), but
not for previous redis releases since EVAL without shebang isn't doing this check.

Note that the check is done on both the keys declared by the EVAL / FCALL command
arguments, and also the ones used by the script when making a `redis.call`.

A note about the implementation, there seems to have been some confusion
about allowing access to non local keys. I thought I missed something in our
wider conversation, but Redis scripts do block access to non-local keys.
So the issue was just about cross slots being accessed.
 2022-04-26 12:09:21 +03:00
Oran Agra 79ffc3524d fix broken protocol regression from #10612 (#10639) 
A change in #10612 introduced a regression.
when replying with garbage bytes to the caller, we must make sure it
doesn't include any newlines.

in the past it called rejectCommandFormat which did that trick.
but now it calls rejectCommandSds, which doesn't, so we need to make sure
to sanitize the sds.
 2022-04-26 00:34:01 +03:00
Binbin 119ec91a5a Fix typos and limit unknown command error message (#10634) 
minor cleanup for recent changes.
 2022-04-25 17:59:39 +03:00
Lu JJ 0c35f54f0a fix an unclear comment and add a comment to 'zi' in 'quicklist.h' (#10633) 
fix an unclear comment quicklist container formats to quicklist node container formats
Add a comment to 'zi' in quicklistIter (Where it first appeared)
Why do I add a comment to zi:
Because it is not a variable name with a clear meaning, and its name seems to be from the deprecated ziplist.
 2022-04-25 17:47:45 +03:00
guybe7 df787764e3 Fix regression not aborting transaction on error, and re-edit some error responses (#10612) 
1. Disk error and slave count checks didn't flag the transactions or counted correctly in command stats (regression from #10372  , 7.0 RC3)
2. RM_Call will reply the same way Redis does, in case of non-exisitng command or arity error
3. RM_WrongArtiy will consider the full command name
4. Use lowercase 'u' in "unknonw subcommand" (to align with "unknown command")

Followup work of #10127
 2022-04-25 13:08:13 +03:00
guybe7 21e39ec461 Test: RM_Call from within "expired" notification (#10613) 
This case is interesting because it originates from cron,
rather than from another command.

The idea came from looking at #9890 and #10573, and I was wondering if RM_Call
would work properly when `server.current_client == NULL`
 2022-04-25 13:05:06 +03:00
Yossi Gottlieb bd823c7fa3 Run large-memory tests as solo. (#10626) 
This avoids random memory spikes and enables --large-memory tests to run
on moderately sized systems.
 2022-04-24 17:29:35 +03:00
guybe7 8ad0cfa56c isSafeToPerformEvictions: Remove redundant condition (#10610) 
If was first added in #9890 to solve the problem of
CONFIG SET maxmemory causing eviction inside MULTI/EXEC,
but that problem is already fixed (CONFIG SET doesn't evict
directly, it just schedules a later eviction)

Keep that condition may hide bugs (i.e. performEvictions
should always expect to have an empty server.also_propagate)
 2022-04-24 16:12:00 +03:00
Binbin a6b3ce28a8 Fix timing issue in slowlog redact test (#10614) 
* Fix timing issue in slowlog redact test

This test failed once in my daily CI (test-sanitizer-address (clang))
```
*** [err]: SLOWLOG - Some commands can redact sensitive fields in tests/unit/slowlog.tcl
Expected 'migrate 127.0.0.1 25649 key 9 5000 AUTH2 (redacted) (redacted)' to match '* key 9 5000 AUTH (redacted)' (context: type eval line 12 cmd {assert_match {* key 9 5000 AUTH (redacted)} [lindex [lindex [r slowlog get] 1] 3]} proc ::test)
```

The reason is that with slowlog-log-slower-than 10000,
slowlog get will have a chance to exceed 10ms.

Change slowlog-log-slower-than from 10000 to -1, disable it.
Also handles a same potentially problematic test above.
This is actually the same timing issue as #10432.

But also avoid repeated calls to `SLOWLOG GET`
 2022-04-24 12:16:30 +03:00
Lu JJ 698e7cbba6 Fix typo in function name "harndfieldReplyWithListpack" to "hrandfieldReplyWithListpack" (#10623) 2022-04-24 10:21:04 +03:00
Lu JJ 719b766586 fix typo in "lcsCommand" doc comment (#10622) 
fix typo. `LCS[j+(blen+1)*j]` -> `LCS[j+(blen+1)*i]`
 2022-04-24 10:19:46 +03:00
filipe oliveiraandOran Agra 3cd8baf616 Optimization: Use either monotonic or wall-clock to measure command execution time, to regain up to 4% execution time (#10502) 
In #7491 (part of redis 6.2), we started using the monotonic timer instead of mstime to measure
command execution time for stats, apparently this meant sampling the clock 3 times per command
rather than two (wince we also need the wall-clock time).
In some cases this causes a significant overhead.

This PR fixes that by avoiding the use of monotonic timer, except for the cases were we know it
should be extremely fast.
This PR also adds a new INFO field called `monotonic_clock` that shows which clock redis is using.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-04-20 14:00:30 +03:00
Oran Agra ee220599b0 Fixes around clients that must be obeyed. Replica report disk errors in PING. (#10603) 
This PR unifies all the places that test if the current client is the
master client or AOF client, and uses a method to test that on all of
these.

Other than some refactoring, these are the actual implications:
- Replicas **don't** ignore disk error when processing commands not
  coming from their master.
  **This is important for PING to be used for health check of replicas**
- SETRANGE, APPEND, SETBIT, BITFIELD don't do proto_max_bulk_len check for AOF
- RM_Call in SCRIPT_MODE ignores disk error when coming from master /
  AOF
- RM_Call in cluster mode ignores slot check when processing AOF
- Scripts ignore disk error when processing AOF
- Scripts **don't** ignore disk error on a replica, if the command comes
  from clients other than the master
- SCRIPT KILL won't kill script coming from AOF
- Scripts **don't** skip OOM check on replica if the command comes from
  clients other than the master

Note that Script, AOF, and module clients don't reach processCommand,
which is why some of the changes don't actually have any implications.

Note, reverting the change done to processCommand in 2f4240b9d9
should be dead code due to the above mentioned fact.
 2022-04-20 11:11:21 +03:00
menwenandOran Agra 046654a70e Stop RDB child before flushing and parsing the RDB in Diskless replication too (#10602) 
We should stop RDB child in advance before flushing to reduce COW in diskless replication too.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-04-20 09:54:55 +03:00
yoav-steinbergandfilipe oliveira 5075e74366 Optimized hdr_value_at_percentile (#10606) 
`hdr_value_at_percentile()` is part of the Hdr_Histogram library
used when generating `latencystats` report. 

There's a pending optimization for this function which greatly
affects the performance of `info latencystats`.
https://github.com/HdrHistogram/HdrHistogram_c/pull/107

This PR:
1. Upgrades the sources in _deps/hdr_histogram_ to the latest Hdr_Histogram
  version 0.11.5
2. Applies the referenced optimization.
3. Adds minor documentation about the hdr_histogram dependency which was
  missing under _deps/README.md_.

benchmark on my machine:
running: `redis-benchmark -n 100000 info latencystats` on a clean build with no data.

| benchmark | RPS |
| ---- | ---- |
| before upgrade to v0.11.05  | 7,681 |
| before optimization | 12,474 |
| after optimization | 52,606 |

Co-authored-by: filipe oliveira <filipecosta.90@gmail.com>
 2022-04-20 09:38:20 +03:00
David CARLIER aba2865c86 Add socket-mark-id support for marking sockets. (#10349) 
Add a configuration option to attach an operating system-specific identifier to Redis sockets, supporting advanced network configurations using iptables (Linux) or ipfw (FreeBSD).
 2022-04-20 09:29:37 +03:00
Oran Agra a1c85eebf4 Tests: improve skip tags around maxmemory and resp3 (#10597) 
some skip tags where missing on some tests....
 2022-04-19 14:57:28 +03:00
Moti CohenmoticlessOran Agra
 		295cbf297f Add comment to sdsConfigSet() (#10605) 
Improve comments to explain the code

Co-authored-by: moticless <moticless@github.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-04-19 14:22:16 +03:00
judengchenyang8094Oran Agra
 		d4cbd8140b Fixes around AOF failed rewrite rate limiting (#10582) 
Changes:
1. Check the failed rewrite time threshold only when we actually consider triggering a rewrite.
  i.e. this should be the last condition tested, since the test has side effects (increasing time threshold)
  Could have happened in some rare scenarios 
2. no limit in startup state (e.g. after restarting redis that previously failed and had many incr files)
3. the “triggered the limit” log would be recorded only when the limit status is returned
4. remove failure count in log (could be misleading in some cases)

Co-authored-by: chenyang8094 <chenyang8094@users.noreply.github.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-04-19 12:06:39 +03:00
sundb 1a93804645 Return 0 when config set out-of-range oom-score-adj-values (#10601) 
When oom-score-adj-values is out of range, setConfigOOMScoreAdjValuesOption
should return 0, not -1, otherwise it will be considered as success.
 2022-04-19 11:31:15 +03:00
Moti Cohen 85899e359e Fix sdsConfigRewrite() to prevent freeing invalid memory(#10598) 2022-04-18 20:04:10 -07:00
Oran Agra 7d1ad6ca96 Fix RM_Yield bug processing future commands of the current client. (#10573) 
RM_Yield was missing a call to protectClient to prevent redis from
processing future commands of the yielding client.

Adding tests that fail without this fix.

This would be complicated to solve since nested calls to RM_Call used to
replace the current_client variable with the module temp client.

It looks like it's no longer necessary to do that, since it was added
back in #9890 to solve two issues, both already gone:
1. call to CONFIG SET maxmemory could trigger a module hook calling
   RM_Call. although this specific issue is gone, arguably other hooks
   like keyspace notification, can do the same.
2. an assertion in lookupKey that checks the current command of the
   current client, introduced in #9572 and removed in #10248
 2022-04-18 14:56:00 +03:00
Binbinandsundb fe4b4806b3 Fix long long to double implicit conversion warning (#10595) 
There is a implicit conversion warning in clang:
```
util.c:574:23: error: implicit conversion from 'long long' to 'double'
changes value from -4611686018427387903 to -4611686018427387904
[-Werror,-Wimplicit-const-int-float-conversion]
    if (d < -LLONG_MAX/2 || d > LLONG_MAX/2)
```

introduced in #10486

Co-authored-by: sundb <sundbcn@gmail.com>
 2022-04-18 08:34:22 +03:00
Oran Agra 0c4733c8d7 Optimize integer zset scores in listpack (converting to string and back) (#10486) 
When the score doesn't have fractional part, and can be stored as an integer,
we use the integer capabilities of listpack to store it, rather than convert it to string.
This already existed before this PR (lpInsert dose that conversion implicitly).

But to do that, we would have first converted the score from double to string (calling `d2string`),
then pass the string to `lpAppend` which identified it as being an integer and convert it back to an int.
Now, instead of converting it to a string, we store it using lpAppendInteger`.

Unrelated:
---
* Fix the double2ll range check (negative and positive ranges, and also the comparison operands
  were slightly off. but also, the range could be made much larger, see comment).
* Unify the double to string conversion code in rdb.c with the one in util.c
* Small optimization in lpStringToInt64, don't attempt to convert strings that are obviously too long.

Benchmark;
---
Up to 20% improvement in certain tight loops doing zzlInsert with large integers.
(if listpack is pre-allocated to avoid realloc, and insertion is sorted from largest to smaller)
 2022-04-17 17:16:46 +03:00
guybe7 f49ff156ec Add RM_PublishMessageShard (#10543) 
since PUBLISH and SPUBLISH use different dictionaries for channels and clients,
and we already have an API for PUBLISH, it only makes sense to have one for SPUBLISH

Add test coverage and unifying some test infrastructure.
 2022-04-17 15:43:22 +03:00
Meir Shpilraien (Spielrein) 789c94fece Added test to verify loading Lua binary payload is not possible (#10583) 
The tests verify that loading a binary payload to the Lua interpreter raises an error.
The Lua code modification was done here: fdf9d45509
which force the Lau interpreter to always use the text parser.
 2022-04-17 15:28:50 +03:00
Wang Yuan a9d5cfa99b Optimize the call of prepareReplicasToWrite (#10588) 
From #9166, we call several times of prepareReplicasToWrite when propagating
one write command to replication stream (once per argument, same as we do for
normal clients), that is not necessary. Now we only call it one time per command
at the begin of feeding replication stream.

This results in reducing CPU consumption and slightly better performance,
specifically when there are many replicas.
 2022-04-17 09:41:46 +03:00
guybe7 fe1c096b18 Add RM_MallocSizeString, RM_MallocSizeDict (#10542) 
Add APIs to allow modules to compute the memory consumption of opaque objects owned by redis.
Without these, the mem_usage callbacks of module data types are useless in many cases.

Other changes:
Fix streamRadixTreeMemoryUsage to include the size of the rax structure itself
 2022-04-17 08:31:57 +03:00
Madelyn Olson effa707e9d Fix incorrect error code for eval scripts and fix test error checking (#10575) 
By the convention of errors, there is supposed to be a space between the code and the name.
While looking at some lua stuff I noticed that interpreter errors were not adding the space,
so some clients will try to map the detailed error message into the error.

We have tests that hit this condition, but they were just checking that the string "starts" with ERR.
I updated some other tests with similar incorrect string checking. This isn't complete though, as
there are other ways we check for ERR I didn't fix.

Produces some fun output like:
```
# Errorstats
errorstat_ERR:count=1
errorstat_ERRuser_script_1_:count=1
```
 2022-04-14 11:18:32 +03:00
Oran Agra 95050f2683 solve corrupt dump fuzzer crash in streams (#10579) 
we had a panic in streamLastValidID when the stream metadata
said it's not empty, but the rax is empty.
 2022-04-14 08:29:35 +03:00
Wen HuiandUbuntu ca913a5de0 Fix several document error and function comments (#10580) 
This PR fix the following minor errors before Redis 7 release:

ZRANGEBYLEX command in deprecated in 6.2.0, and could be replaced by ZRANGE with the
BYLEX argument, but in the document, the words is written incorrect in " by ZRANGE with the BYSCORE argument"

Fix function zpopmaxCommand incorrect comment

The comments of function zmpopCommand and bzmpopCommand are not consistent with document description, fix them

Co-authored-by: Ubuntu <lucas.guang.yang1@huawei.com>
 2022-04-13 21:18:37 +03:00
filipe oliveiraandOran Agra a642947e04 Optimize stream id sds creation on XADD key * (~20% saved cpu cycles) (#10574) 
we can observe that when adding to a stream without ID there is a duplicate work
on sds creation/freeing/sdslen that costs ~11% of the CPU cycles. 

This PR avoids it by not freeing the sds after the first reply. 
The expected reduction in CPU cycles is around 9-10% 

Additionally, we now pre-allocate the sds to the right size, to avoid realloc.
this brought another ~10% improvement

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-04-13 12:29:24 +03:00
Luke Palmer bb7891f080 Keyspace event for new keys (#10512) 
Add an optional keyspace event when new keys are added to the db.

This is useful for applications where clients need to be aware of the redis keyspace.
Such an application can SCAN once at startup and then listen for "new" events (plus
others associated with DEL, RENAME, etc).
 2022-04-13 11:36:38 +03:00
guybe7 e875ff89ec Add the deprecated_since field in command args of COMMAND DOCS (#10545) 
Apparently, some modules can afford deprecating command arguments
(something that was never done in Redis, AFAIK), so in order to represent
this piece of information, we added the `deprecated_since` field to redisCommandArg
(in symmetry to the already existing `since` field).

This commit adds `const char *deprecated_since` to `RedisModuleCommandArg`,
which is technically a breaking change, but since 7.0 was not released yet, we decided to let it slide
 2022-04-13 11:33:36 +03:00
Yossi Gottlieb bd8da0ca29 Fix error/warning on Arm due to unsigned char. (#10572) 2022-04-12 18:55:11 +03:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 6b403f56a5 Bump actions/upload-artifact from 2 to 3 (#10566) 
Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 2 to 3.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/v2...v3)

---
updated-dependencies:
- dependency-name: actions/upload-artifact
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
 2022-04-12 13:23:41 +03:00
Madelyn Olson 8bd01a07ae Allow specifying ACL reason for module log entry (#10559) 
Allow specifying an ACL log reason, which is shown in the log. Right now it always shows "unknown", which is a little bit cryptic. This is a breaking change, but this API was added as part of 7 so it seems ok to stabilize it still.
 2022-04-11 22:16:17 -07:00
Ernesto Rodriguez ReinaandMadelyn Olson 4c6d9bbd62 Extending the use of hashTypeGetValue. (#10567) 
* Extending the use of hashTypeGetValue.

Functions hashTypeExists, hashTypeGetValueLength and addHashFieldToReply
have a similar pattern on calling hashTypeGetFromHashTable or
hashTypeGetFromZipList depending on the underlying data structure. What
does functions are duing is exactly what hashTypeGetValue does. Those
functions were changed to use existing function hashTypeGetValue making
the code more consistent.

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
 2022-04-11 16:05:45 -07:00
zhaozhao.zz 1a7765cb7c Durability enhancement for appendfsync=always policy (#9678) 
Durability of database is a big and old topic, in this regard Redis use AOF to
support it, and `appendfsync=alwasys` policy is the most strict level, guarantee
all data is both written and synced on disk before reply success to client.

But there are some cases have been overlooked, and could lead to durability broken.

1. The most clear one is about threaded-io mode
   we should also set client's write handler with `ae_barrier` in
   `handleClientsWithPendingWritesUsingThreads`, or the write handler would be
   called after read handler in the next event loop, it means the write command result
   could be replied to client before flush to AOF.
2. About blocked client (mostly by module)
   in `beforeSleep()`, `handleClientsBlockedOnKeys()` should be called before
   `flushAppendOnlyFile()`, in case the unblocked clients modify data without persistence
   but send reply.
3. When handling `ProcessingEventsWhileBlocked`
   normally it takes place when lua/function/module timeout, and we give a chance to users
   to kill the slow operation, but we should call `flushAppendOnlyFile()` before
   `handleClientsWithPendingWrites()`, in case the other clients in the last event loop get
   acknowledge before data persistence.
   for a instance:
   ```
   in the same event loop
   client A executes set foo bar
   client B executes eval "for var=1,10000000,1 do end" 0
   ```
   after the script timeout, client A will get `OK` but lose data after restart (kill redis when
   timeout) if we don't flush the write command to AOF.
4. A more complex case about `ProcessingEventsWhileBlocked`
   it is lua timeout in transaction, for example
   `MULTI; set foo bar; eval "for var=1,10000000,1 do end" 0; EXEC`, then client will get set
   command's result before the whole transaction done, that breaks atomicity too.
   fortunately, it's already fixed by #5428 (although it's not the original purpose just a side
   effect : )), but module timeout should be fixed too.

case 1, 2, 3 are fixed in this commit, the module issue in case 4 needs a followup PR.
 2022-04-11 11:08:39 +03:00
Evan 574ed6b0ce modules: add RedisModuleKey* return type to RM_OpenKey (#3719) 
Change `RM_OpenKey` to return `RedisModuleKey*` instead of `void*`.
Which is the input type of other APIs that take the value from RM_OpenKey.
 2022-04-11 08:34:43 +03:00
guybe7 eeb0f1426c Add RM_TryAlloc (#10541) 
Similarly to LCS, some modules would want to try to allocate memory, and
fail gracefully if the allocation fails
 2022-04-10 11:43:59 +03:00
guybe7 719db14ec7 COMMAND DOCS shows module name, where applicable (#10544) 
Add field to COMMAND DOCS response to denote the name of the module
that added that command.
COMMAND LIST can filter by module, but if you get the full commands list,
you may still wanna know which command belongs to which module.
The alternative would be to do MODULE LIST, and then multiple calls to COMMAND LIST
 2022-04-10 11:41:31 +03:00
cyhone 6cb5cbb28f fix indent space (#10562) 2022-04-10 10:38:57 +03:00
zhl49andOran Agra dd3b545d89 Fix an mistake in comment (#10560) 
copy paste error..

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-04-10 09:29:50 +03:00
王恒andMadelyn Olson ee17e7af8d improve malloc efficiency: reduce call times of zrealloc (#10533) 
* improve malloc efficiency: reduce call times of zrealloc

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
 2022-04-09 19:52:36 -07:00
chenyang8094Oran Agrayoav-steinberg
 		625bdaf3d1 Fix auto-aof-rewrite-percentage based AOFRW trigger after restart (#10550) 
The `auto-aof-rewrite-percentage` config defines at what growth percentage
an automatic AOF rewrite is triggered.
This normally works OK since the size of the AOF file at the end of a rewrite
is stored in `server.aof_rewrite_base_size`.
However, on startup, redis used to store the entire size of the AOF file into that
variable, resulting in a wrong automatic AOF rewrite trigger (could have been
triggered much later than desired).
This issue would only affect the first AOFRW after startup, after that future AOFRW
would have been triggered correctly.
This bug existed in all previous versions of Redis.

This PR unifies the meaning of `server.aof_rewrite_base_size`, which only represents
the size of BASE AOF.
Note that after an AOFRW this size includes the size of the incremental file (all the
commands that executed during rewrite), so that auto-aof-rewrite-percentage is the
ratio from the size of the AOF after rewrite.
However, on startup, it is complicated to know that size, and we compromised on
taking just the size of the base file, this means that the first rewrite after startup can
happen a little bit too soon.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: yoav-steinberg <yoav@redislabs.com>
 2022-04-07 14:47:07 +03:00
Oran Agra 451531f1c8 Fix RM_Yield bug (#10548) 
The bug was when using REDISMODULE_YIELD_FLAG_CLIENTS.
in that case we would have only set the CLIENTS type flag in
server.busy_module_yield_flags and then clear that flag when exiting
RM_Yield, so we would never call unblockPostponedClients when the
context is destroyed.

This didn't really have any actual implication, which is why the tests
couldn't (and still can't) find that since the bug only happens when
using CLIENT, but in this case we won't have any clients to un-postpone
i.e. clients will get rejected with BUSY error, rather than being
postponed.

Unrelated:
* Adding tests for nested contexts, just in case.
* Avoid nested RM_Yield calls
 2022-04-07 11:52:28 +03:00
Itamar Haber 3e09a8c097 Fixes commands' syntices (#10534) 
Fixes in command argument in json files
* Fixes BITFIELD's syntax ("sub-commands" can be repeated, and OVERFLOW is only valid for SET and INCR)
* Improves readability of SET (reordered)
* Fixes GEOSEARCH and GEOSEARCH_RO syntices (use `oneof` for mutually exclusive group instead of `optional`)
* Fixes MIGRATE syntax (use `oneof` for mutually exclusive group instead of `optional`)
* Fixes MODULE LOADEX syntax (the `CONFIG` token should be repeated too when using multiple configs)

other:
* make generate-command-help.rb accept a path to commands.json, or read it from stdin (e.g. `generate-commands-json.py | generate-command-help.rb -`)
 2022-04-06 09:33:33 +03:00
Lu JJ f110de4b23 Fix the bug that caused hash encoding errors when using hincrbyfloat or hincrby commands (#10479) 
Fixed a bug that used the `hincrbyfloat` or `hincrby` commands to make the field or value exceed the
`hash_max_listpack_value` but did not change the object encoding of the hash structure.

Add a length check for field and value, check the length of value first, if the length of value does not
exceed `hash_max_listpack_value` then check the length of field.

If the length of field or value is too long, it will reduce the efficiency of listpack, and the object encoding
will become hashtable after AOF restart, so this is also to keep the same before and after AOF restart.
 2022-04-05 21:45:45 +03:00
judeng 8a7049d363 use $^ instead of $< for linker in module makefile (#10530) 2022-04-05 17:08:27 +03:00
Moti Cohen e342bedc83 Stabilize Sentinel tests - refine failover-timeout & tilt-period (#10518) 
Sentinel once in a while experience Sentinel TILT period or leader election 
failure cycle. The problem is that those default timeout are too big and once 
it happens, it breaks our tests.  Suggesting:
- Reducing failover-timeout from 20 to 10sec (actually it is multiplied by 2 
  and reach 40sec of timeout) 
- Modify tilt-period from default of 30sec to 5sec. When TILT period happens 
  it might lead to failover in our tests, and might cause also to failover cycle
  cycle failure.

Sentinel tests should `wait_for_condition` up to 50seconds, where needed, 
to be stable in case having single TILT period or failover failure cycle.

In addition relax timing configuration for "manual failover" Sentinel test 
(was modified several months ago as part of an effort to reduce tests runtime)
 2022-04-05 17:07:59 +03:00
Oran Agra fb4e0d400f Merge pull request #10532 from oranagra/7.0-rc3
CI / build-debian-old (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CodeQL / Analyze (cpp) (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details 
Release 7.0 rc3
 2022-04-05 14:25:02 +03:00
Oran Agra 8b242ef977 7.0-RC3 2022-04-05 14:06:45 +03:00
Oran Agra d7971f96b3 Merge remote-tracking branch 'origin/unstable' into 7.0 2022-04-05 13:36:18 +03:00
Oran Agra acfb4f7a44 update static help.h from redis.io commands.json (#10529) 2022-04-05 12:09:51 +03:00
Madelyn Olson 4ffcec29af Update json command files so they only include syntax related information (#10398) 
The command json documents should just include information about the "arguments" and the "outputs".
I removed all of the 'functional wording' so it's clear.
 2022-04-05 10:36:06 +03:00
Meir Shpilraien (Spielrein) ae020e3d56 Functions: Move library meta data to be part of the library payload. (#10500) 
## Move library meta data to be part of the library payload.

Following the discussion on https://github.com/redis/redis/issues/10429 and the intention to add (in the future) library versioning support, we believe that the entire library metadata (like name and engine) should be part of the library payload and not provided by the `FUNCTION LOAD` command. The reasoning behind this is that the programmer who developed the library should be the one who set those values (name, engine, and in the future also version). **It is not the responsibility of the admin who load the library into the database.**

The PR moves all the library metadata (engine and function name) to be part of the library payload. The metadata needs to be provided on the first line of the payload using the shebang format (`#!<engine> name=<name>`), example:

```lua
#!lua name=test
redis.register_function('foo', function() return 1 end)
```

The above script will run on the Lua engine and will create a library called `test`.

## API Changes (compare to 7.0 rc2)

* `FUNCTION LOAD` command was change and now it simply gets the library payload and extract the engine and name from the payload. In addition, the command will now return the function name which can later be used on `FUNCTION DELETE` and `FUNCTION LIST`.
* The description field was completely removed from`FUNCTION LOAD`, and `FUNCTION LIST`


## Breaking Changes (compare to 7.0 rc2)

* Library description was removed (we can re-add it in the future either as part of the shebang line or an additional line).
* Loading an AOF file that was generated by either 7.0 rc1 or 7.0 rc2 will fail because the old command syntax is invalid.

## Notes

* Loading an RDB file that was generated by rc1 / rc2 **is** supported, Redis will automatically add the shebang to the libraries payloads (we can probably delete that code after 7.0.3 or so since there's no need to keep supporting upgrades from an RC build).
 2022-04-05 10:27:24 +03:00
bugwzandMadelyn Olson 2db0d898f8 Cluster node name sanity check (#10391) 
* Limit cluster node id length for CLUSTER commands loading
* Cluster node name sanity check for length and values

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
 2022-04-04 22:51:51 -07:00
judeng 9578b67e0e delete obsolete REDISMODULE_EXPERIMENTAL_API define in module demos (#10527) 
This macro was recently removed from redismodule.h, so no longer needed.
 2022-04-05 08:21:41 +03:00
Binbin a95ae56b30 Remove outdated utils/hashtable/rehashing.c (#10526) 
It is really outdated and i guessing no one will use it now. 
Remove the corresponding utils/hashtable directory.
closes #10235
 2022-04-04 19:12:47 -07:00
chenyang8094 cb625844bf Fix null pointer subtraction warning (#10498) 
The warning:
```
pqsort.c:106:7: warning: performing pointer subtraction with a null pointer has undefined behavior [-Wnull-pointer-subtraction]
loop:   SWAPINIT(a, es);
        ^~~~~~~~~~~~~~~
pqsort.c:65:47: note: expanded from macro 'SWAPINIT'
#define SWAPINIT(a, es) swaptype = ((char *)a - (char *)NULL) % sizeof(long) || \
```
Clang version:
```
Apple clang version 13.1.6 (clang-1316.0.21.2)
Target: x86_64-apple-darwin21.3.0
Thread model: posix
InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin
```
 2022-04-04 18:38:18 +03:00
Bofang Liu 8500ca4ae8 Fix mistake / outdated doc comment (#10521) 
Unlike original reference code, Redis uses 64bit variable, so half the bits make it 32bit, not 16
 2022-04-04 15:35:49 +03:00
Meir Shpilraien (Spielrein) 047b609335 Fix #10508, on error, pop function and error handler from Lua stack. (#10519) 
If, for some reason, Redis decides not to execute the script, we need
to pop the function and error handler from Lua stack. Otherwise, eventually
the Lua stack will explode.

Relevant only for 7.0-rc1 and 7.0-rc2.
 2022-04-04 10:58:59 +03:00
judengandjudeng 6cdaa27703 fix typos in aof.c (#10513) 
function aofRewriteLimited in aof.c, deley->delay and NAX->MAX

Co-authored-by: judeng <judeng@didiglobal.com>
 2022-04-04 08:38:41 +03:00
Moti Cohenandmoticless 37beb5e67e Fix sentinel ACL test. Timing issue. (#10510) 
Fix by replacing in test blind sleep with wait_for_condition().

Co-authored-by: moticless <moticless@github.com>
 2022-04-03 10:56:15 +03:00
Viktor Söderqvist b53c7f2c0b Turn into replica on SETSLOT (#10489) 
* Fix race condition where node loses its last slot and turns into replica

When a node has lost its last slot and finds out from the SETSLOT command
before the cluster bus PONG from the new owner arrives. In this case, the
node didn't turn itself into a replica of the new slot owner.

This commit adds the same logic to the SETSLOT command as already exists
for the cluster bus PONG processing.

* Revert "Fix new / failing cluster slot migration test (#10482)"

This reverts commit 0b21ef8d49.

In this test, the old slot owner finds out that it has lost its last
slot in a nondeterministic way. Either the cluster bus PONG from the
new slot owner and sometimes in a SETSLOT command from redis-cli. In
both cases, the result should be the same and the old owner should
turn itself into a replica of the new slot owner.
 2022-04-02 14:58:07 -07:00
sundb b8eb2a7340 Fix failing moduleconfigs tests and memory leak (#10501) 
Fix global `strval` not reset to NULL after being freed, causing a crash on alpine
(most likely because the dynamic library loader doesn't init globals on reload)
By the way, fix the memory leak of using `RedisModule_Free` to free `RedisModuleString`,
and add a corresponding test.
 2022-03-31 15:26:10 +03:00
Madelyn Olson e81bd15e99 Prevent replica failover during manual takeover test (#10499) 
During 11-manual-takeover.tcl, if the killing of the instances happens
too slowly, one of the replicas might be able to promote itself.
I'm not sure why it was slow, but it was observed taking 6 seconds
which is enough time to do an election.
I was able to verify the error locally by adding a small delay (1 second)
during ASAN CI. A fix is just to disable automated failover until all the
nodes are confirmed dead.
 2022-03-31 08:15:00 +03:00
Binbin a3075ca4fe Fix cluster slot migration test (#10495) 
Fix three timing issues in the test
 2022-03-30 20:14:21 -07:00
Ozan Tezcan 7da1cc3e90 Use exit code 1 on error in redis-cli (#10468) 
On error, redis-cli was returning `REDIS_ERR` on some cases by mistake. `REDIS_ERR` is `-1` which becomes `255` as exit code. This commit changes it and returns `1` on errors to be consistent.
 2022-03-30 21:16:02 +03:00
Binbin 6075f50663 Move restart_killed_instances and verify_sentinel_auto_discovery to utils (#10497) 
Create a utils.tcl in sentinel/tests/includes, and move two procs to it.
Allow sentinel test 08-hostname-conf run on its own.
 2022-03-30 20:42:51 +03:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 4e55d557eb Bump actions/checkout from 2 to 3 (#10390) 
Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v2...v3)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
 2022-03-30 16:18:03 +03:00
dependabot[bot]anddependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> 8df37363db Bump actions/cache from 2 to 3 (#10463) 
Bumps [actions/cache](https://github.com/actions/cache) from 2 to 3.
- [Release notes](https://github.com/actions/cache/releases)
- [Commits](https://github.com/actions/cache/compare/v2...v3)

---
updated-dependencies:
- dependency-name: actions/cache
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
 2022-03-30 16:16:21 +03:00
Nick ChunMadelyn OlsonMadelyn OlsonsundbMadelyn OlsonOran AgraYossi Gottlieb
 		bda9d74dad Module Configurations (#10285) 
This feature adds the ability to add four different types (Bool, Numeric,
String, Enum) of configurations to a module to be accessed via the redis
config file, and the CONFIG command.

**Configuration Names**:

We impose a restriction that a module configuration always starts with the
module name and contains a '.' followed by the config name. If a module passes
"config1" as the name to a register function, it will be registered as MODULENAME.config1.

**Configuration Persistence**:

Module Configurations exist only as long as a module is loaded. If a module is
unloaded, the configurations are removed.
There is now also a minimal core API for removal of standardConfig objects
from configs by name.

**Get and Set Callbacks**:

Storage of config values is owned by the module that registers them, and provides
callbacks for Redis to access and manipulate the values.
This is exposed through a GET and SET callback.

The get callback returns a typed value of the config to redis. The callback takes
the name of the configuration, and also a privdata pointer. Note that these only
take the CONFIGNAME portion of the config, not the entire MODULENAME.CONFIGNAME.

```
 typedef RedisModuleString * (*RedisModuleConfigGetStringFunc)(const char *name, void *privdata);
 typedef long long (*RedisModuleConfigGetNumericFunc)(const char *name, void *privdata);
 typedef int (*RedisModuleConfigGetBoolFunc)(const char *name, void *privdata);
 typedef int (*RedisModuleConfigGetEnumFunc)(const char *name, void *privdata);
```

Configs must also must specify a set callback, i.e. what to do on a CONFIG SET XYZ 123
or when loading configurations from cli/.conf file matching these typedefs. *name* is
again just the CONFIGNAME portion, *val* is the parsed value from the core,
*privdata* is the registration time privdata pointer, and *err* is for providing errors to a client.

```
typedef int (*RedisModuleConfigSetStringFunc)(const char *name, RedisModuleString *val, void *privdata, RedisModuleString **err);
typedef int (*RedisModuleConfigSetNumericFunc)(const char *name, long long val, void *privdata, RedisModuleString **err);
typedef int (*RedisModuleConfigSetBoolFunc)(const char *name, int val, void *privdata, RedisModuleString **err);
typedef int (*RedisModuleConfigSetEnumFunc)(const char *name, int val, void *privdata, RedisModuleString **err);
```

Modules can also specify an optional apply callback that will be called after
value(s) have been set via CONFIG SET:

```
typedef int (*RedisModuleConfigApplyFunc)(RedisModuleCtx *ctx, void *privdata, RedisModuleString **err);
```

**Flags:**
We expose 7 new flags to the module, which are used as part of the config registration.

```
#define REDISMODULE_CONFIG_MODIFIABLE 0 /* This is the default for a module config. */
#define REDISMODULE_CONFIG_IMMUTABLE (1ULL<<0) /* Can this value only be set at startup? */
#define REDISMODULE_CONFIG_SENSITIVE (1ULL<<1) /* Does this value contain sensitive information */
#define REDISMODULE_CONFIG_HIDDEN (1ULL<<4) /* This config is hidden in `config get <pattern>` (used for tests/debugging) */
#define REDISMODULE_CONFIG_PROTECTED (1ULL<<5) /* Becomes immutable if enable-protected-configs is enabled. */
#define REDISMODULE_CONFIG_DENY_LOADING (1ULL<<6) /* This config is forbidden during loading. */
/* Numeric Specific Configs */
#define REDISMODULE_CONFIG_MEMORY (1ULL<<7) /* Indicates if this value can be set as a memory value */
```

**Module Registration APIs**:

```
int (*RedisModule_RegisterBoolConfig)(RedisModuleCtx *ctx, char *name, int default_val, unsigned int flags, RedisModuleConfigGetBoolFunc getfn, RedisModuleConfigSetBoolFunc setfn, RedisModuleConfigApplyFunc applyfn, void *privdata);
int (*RedisModule_RegisterNumericConfig)(RedisModuleCtx *ctx, const char *name, long long default_val, unsigned int flags, long long min, long long max, RedisModuleConfigGetNumericFunc getfn, RedisModuleConfigSetNumericFunc setfn, RedisModuleConfigApplyFunc applyfn, void *privdata);
int (*RedisModule_RegisterStringConfig)(RedisModuleCtx *ctx, const char *name, const char *default_val, unsigned int flags, RedisModuleConfigGetStringFunc getfn, RedisModuleConfigSetStringFunc setfn, RedisModuleConfigApplyFunc applyfn, void *privdata);
int (*RedisModule_RegisterEnumConfig)(RedisModuleCtx *ctx, const char *name, int default_val, unsigned int flags, const char **enum_values, const int *int_values, int num_enum_vals, RedisModuleConfigGetEnumFunc getfn, RedisModuleConfigSetEnumFunc setfn, RedisModuleConfigApplyFunc applyfn, void *privdata);
int (*RedisModule_LoadConfigs)(RedisModuleCtx *ctx);
```

The module name will be auto appended along with a "." to the front of the name of the config.

**What RM_Register[...]Config does**:

A RedisModule struct now keeps a list of ModuleConfig objects which look like:
```
typedef struct ModuleConfig {
    sds name; /* Name of config without the module name appended to the front */
    void *privdata; /* Optional data passed into the module config callbacks */
    union get_fn { /* The get callback specificed by the module */
        RedisModuleConfigGetStringFunc get_string;
        RedisModuleConfigGetNumericFunc get_numeric;
        RedisModuleConfigGetBoolFunc get_bool;
        RedisModuleConfigGetEnumFunc get_enum;
    } get_fn;
    union set_fn { /* The set callback specified by the module */
        RedisModuleConfigSetStringFunc set_string;
        RedisModuleConfigSetNumericFunc set_numeric;
        RedisModuleConfigSetBoolFunc set_bool;
        RedisModuleConfigSetEnumFunc set_enum;
    } set_fn;
    RedisModuleConfigApplyFunc apply_fn;
    RedisModule *module;
} ModuleConfig;
```
It also registers a standardConfig in the configs array, with a pointer to the
ModuleConfig object associated with it.

**What happens on a CONFIG GET/SET MODULENAME.MODULECONFIG:**

For CONFIG SET, we do the same parsing as is done in config.c and pass that
as the argument to the module set callback. For CONFIG GET, we call the
module get callback and return that value to config.c to return to a client.

**CONFIG REWRITE**:

Starting up a server with module configurations in a .conf file but no module load
directive will fail. The flip side is also true, specifying a module load and a bunch
of module configurations will load those configurations in using the module defined
set callbacks on a RM_LoadConfigs call. Configs being rewritten works the same
way as it does for standard configs, as the module has the ability to specify a
default value. If a module is unloaded with configurations specified in the .conf file
those configurations will be commented out from the .conf file on the next config rewrite.

**RM_LoadConfigs:**

`RedisModule_LoadConfigs(RedisModuleCtx *ctx);`

This last API is used to make configs available within the onLoad() after they have
been registered. The expected usage is that a module will register all of its configs,
then call LoadConfigs to trigger all of the set callbacks, and then can error out if any
of them were malformed. LoadConfigs will attempt to set all configs registered to
either a .conf file argument/loadex argument or their default value if an argument is
not specified. **LoadConfigs is a required function if configs are registered.
** Also note that LoadConfigs **does not** call the apply callbacks, but a module
can do that directly after the LoadConfigs call.

**New Command: MODULE LOADEX [CONFIG NAME VALUE] [ARGS ...]:**

This command provides the ability to provide startup context information to a module.
LOADEX stands for "load extended" similar to GETEX. Note that provided config
names need the full MODULENAME.MODULECONFIG name. Any additional
arguments a module might want are intended to be specified after ARGS.
Everything after ARGS is passed to onLoad as RedisModuleString **argv.

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Madelyn Olson <matolson@amazon.com>
Co-authored-by: sundb <sundbcn@gmail.com>
Co-authored-by: Madelyn Olson <34459052+madolson@users.noreply.github.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
 2022-03-30 15:47:06 +03:00
Binbin e2fa6aa158 command json files cleanups (#10473) 
This PR do some command json files cleanups:

1. Add COMMAND TIPS to some commands
- command-docs: add `NONDETERMINISTIC_OUTPUT_ORDER`
- command-info: add `NONDETERMINISTIC_OUTPUT_ORDER`
- command-list: add `NONDETERMINISTIC_OUTPUT_ORDER`
- command: change `NONDETERMINISTIC_OUTPUT` to `NONDETERMINISTIC_OUTPUT_ORDER`
- function-list: add `NONDETERMINISTIC_OUTPUT_ORDER`
- latency-doctor: add `NONDETERMINISTIC_OUTPUT`, `REQUEST_POLICY:ALL_NODES` and `RESPONSE_POLICY:SPECIAL`
- latency-graph: add `NONDETERMINISTIC_OUTPUT`, `REQUEST_POLICY:ALL_NODES` and `RESPONSE_POLICY:SPECIAL`
- memory-doctor: add `REQUEST_POLICY:ALL_SHARDS` and `RESPONSE_POLICY:SPECIAL`
- memory-malloc-stats: add `REQUEST_POLICY:ALL_SHARDS` and `RESPONSE_POLICY:SPECIAL`
- memory-purge: add `REQUEST_POLICY:ALL_SHARDS` and `RESPONSE_POLICY:ALL_SUCCEEDED`
- module-list: add `NONDETERMINISTIC_OUTPUT_ORDER`
- msetnx: add `REQUEST_POLICY:MULTI_SHARD` and `RESPONSE_POLICY:AGG_MIN`
- object-refcount: add `NONDETERMINISTIC_OUTPUT`
3. Only (mostly) indentation and formatting changes:
- cluster-shards
- latency-history
- pubsub-shardchannels
- pubsub-shardnumsub
- spublish
- ssubscribe
- sunsubscribe
4. add doc_flags (DEPRECATED) to cluster-slots,  replaced_by `CLUSTER SHARDS` in 7.0
5. command-getkeysandflags: a better summary (the old one is copy from command-getkeys)
6. adjustment of command parameter types
- `port` is integer, not string (`MIGRATE`, `REPLICAOF`, `SLAVEOF`)
- `replicationid` is string, not integer (`PSYNC`)
- `pattern` is pattern, not string (`PUBSUB CHANNELS`, `SENTINEL RESET`, `SORT`, `SORT_RO`)
 2022-03-30 09:39:26 +03:00
Oran Agra 16d206ee36 fix daily.yaml skip filters (#10490) 
* missing parenthesis meant that the ubuntu and centos jobs were not
  skipped
* the recently divided freebsd, macos, and valgrind jobs, which are now
  split into distict jobs for redis, modules, sentinel, cluster. were
  all executed, producing a build, but not running anything.
  now they're filtered at the job level
* iothreads was missing from the skip list defaults, so was not skipped
 2022-03-29 18:35:17 +03:00
Viktor Söderqvist 35bb021254 redis-cli: Do DNS lookup before sending CLUSTER MEET (#10436) 
Affects `--cluster create` and `--cluster add-node`.
 2022-03-29 15:45:14 +03:00
zhaozhao.zz 001e192575 show cluster.links in MEMORY STATS (#10302) 2022-03-29 16:54:45 +08:00
Oran Agra 3b1e65a32b improve malloc efficiency for cluster slots_info_pairs (#10488) 
This commit improve malloc efficiency of the slots_info_pairs mechanism in cluster.c
by changing adlist into an array being realloced with greedy growth mechanism

Recently the cluster tests are consistently failing when executed with ASAN in the CI.
I tried to track down the commit that started it, and it appears to be #10293.
Looking at the commit, i realize it didn't affect this test / flow, other than the
replacement of the slots_info_pairs from sds to list.

I concluded that what could be happening is that the slot range is very fragmented,
and that results in many allocations.
with sds, it results in one allocation and also, we have a greedy growth mechanism,
but with adlist, we just have many many small allocations.
this probably causes stress on ASAN, and causes it to be slow at termination.
 2022-03-29 10:05:06 +03:00
Oran Agra 14b198868f introduce MAX_D2STRING_CHARS instead of 128 const (#10487) 
There are a few places that use a hard coded const of 128 to allocate a buffer for d2string.
Replace these with a clear macro.
Note that In theory, converting double into string could take as much as nearly 400 chars,
but since d2string uses `%g` and not `%f`, it won't pass some 40 chars.

unrelated:
restore some changes to auto generated commands.c that got accidentally reverted in #10293
 2022-03-28 18:35:56 +03:00
Binbin 3f28d7d712 Make redis-cli --cluster help output to stdout (#10485) 
redis-cli --cluster help currently outputs the help on stderr.
This is similar to #9124
 2022-03-28 14:47:36 +03:00
Moti Cohen 63f77698cf Fix sentinel test SDOWN is triggered by non-responding instance (#10484) 
A timing issue of debug sleep master isn't long enough to ensure
that master is down and let the test identify it. Replaced the code
with suspend PID until verified master-is-down.
 2022-03-28 12:40:52 +03:00
DarrenJiang13andOran Agra ae771ea77b fix crash in debug protocol push (#10483) 
a missing of resp3 judgement which may lead to the crash using `debug protocol push`
introduced in #9235
Similar improvement in RM_ReplySetAttributeLength in case the module ignored the
error that returned from RM_ReplyWithAttribute.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-03-28 11:54:34 +03:00
Oran Agra 0b21ef8d49 Fix new / failing cluster slot migration test (#10482) 
#10381 fixed an issue in `redis-cli --cluster reshard` that used to fail it (redis-cli) because
of a race condition.
the race condition is / was that when moving the last slot from a node, sometimes the PONG
messages delivering the configuration change arrive to that node before the SETSLOT arrives
to it, and it becomes a replica.
other times the the SETSLOT arrive first, and then PONG **doesn't** demote it.

**however**, the PR also added a new test that suffers from exactly the same race condition,
and the tests started failing a lot.

The fact is (if i understand it correctly), that this test (the one being deleted here), isn't related
to the fix that PR fixed (which was to fix redis-cli).
The race condition in the cluster code still happens, and as long as we don't solve it, there's
no reason to test it.

For now, even if my understandings are wrong, i'm gonna delete that failing test, since as far as
i understand, #10381 didn't introduce any new risks for that matter (which are gonna be
compromised by removing this check), this race existed since forever, and still exists, and the
fact that redis-cli is now immune to it is still being tested.

Additional work should be carried to fix it, and i live it for other PRs to handle.
 2022-03-27 18:39:19 +03:00
Moti Cohen 37d761ba29 Fix Sentinel reconnect test following ACL change (#10480) 
Replace condition with wait_for_condition On "Verify sentinel that restarted 
failed to reconnect master after ACL change"

The reason we reach it, is because the test is fast enough to modify ACL and
test sentinel connection status with the server - before its scheduled operation
got the chance to update connection status with the server:
```
/* Perform scheduled operations for the specified Redis instance. */
void sentinelHandleRedisInstance(sentinelRedisInstance *ri) {
    /* ========== MONITORING HALF ============ */
    /* Every kind of instance */
    sentinelReconnectInstance(ri);
```
 2022-03-27 17:56:21 +03:00
Binbin f25e688e2a Cleanups in redis.conf (#10452) 
Did some cleanups:
1. local local typo
2. replace the only slave word in the file
3. add FUNCTION FLUSH to `lazyfree-lazy-user-flush` description
4. thought it would be better to use these, there are actually "four" options
5. the the typo
6. remove a extra space
7. change comment next to `activedefrag no` to match the default value
 2022-03-27 12:03:38 +03:00
小令童鞋 3bcc67d14f Add appendonlydir-* in utils/create-cluster/.gitignore (#10471) 2022-03-26 09:53:10 +03:00
zhaozhao.zz 78bef6e1fe optimize(remove) usage of client's pending_querybuf (#10413) 
To remove `pending_querybuf`, the key point is reusing `querybuf`, it means master client's `querybuf` is not only used to parse command, but also proxy to sub-replicas.

1. add a new variable `repl_applied` for master client to record how many data applied (propagated via `replicationFeedStreamFromMasterStream()`) but not trimmed in `querybuf`.

2. don't sdsrange `querybuf` in `commandProcessed()`, we trim it to `repl_applied` after the whole replication pipeline processed to avoid fragmented `sdsrange`. And here are some scenarios we cannot trim to `qb_pos`:
    * we don't receive complete command from master
    * master client blocked because of client pause
    * IO threads operate read, master client flagged with CLIENT_PENDING_COMMAND

    In these scenarios, `qb_pos` points to the part of the current command or the beginning of next command, and the current command is not applied yet, so the `repl_applied` is not equal to `qb_pos`.

Some other notes:
* Do not do big arg optimization on master client, since we can only sdsrange `querybuf` after data sent to replicas.
* Set `qb_pos` and `repl_applied` to 0 when `freeClient` in `replicationCacheMaster`.
* Rewrite `processPendingCommandsAndResetClient` to `processPendingCommandAndInputBuffer`, let `processInputBuffer` to be called successively after `processCommandAndResetClient`.
 2022-03-25 10:45:40 +08:00
Oran Agra 1a57af629c Split daily CI into smaller chunks (#10469) 
this should aid find the CI issues with freebsd and macos runs, and also
get faster results from valgrind and tls
 2022-03-22 17:38:01 +02:00
Oran Agra 6761d10cc3 crash log, print killer pid only when si_code is SI_USER (#10454) 
Avoid printing "Killed by PID" when si_code != SI_USER.
Apparently SI_USER isn't always set to 0. e.g. on Mac it's 0x10001 and the check that did <= was wrong.
 2022-03-22 17:37:34 +02:00
Meir Shpilraien (Spielrein) f3855a0930 Add new RM_Call flags for script mode, no writes, and error replies. (#10372) 
The PR extends RM_Call with 3 new capabilities using new flags that
are given to RM_Call as part of the `fmt` argument.
It aims to assist modules that are getting a list of commands to be
executed from the user (not hard coded as part of the module logic),
think of a module that implements a new scripting language...

* `S` - Run the command in a script mode, this means that it will raise an
  error if a command which are not allowed inside a script (flaged with the
  `deny-script` flag) is invoked (like SHUTDOWN). In addition, on script mode,
  write commands are not allowed if there is not enough good replicas (as
  configured with `min-replicas-to-write`) and/or a disk error happened.

* `W` - no writes mode, Redis will reject any command that is marked with `write`
  flag. Again can be useful to modules that implement a new scripting language
  and wants to prevent any write commands.

* `E` - Return errors as RedisModuleCallReply. Today the errors that happened
  before the command was invoked (like unknown commands or acl error) return
  a NULL reply and set errno. This might be missing important information about
  the failure and it is also impossible to just pass the error to the user using
  RM_ReplyWithCallReply. This new flag allows you to get a RedisModuleCallReply
  object with the relevant error message and treat it as if it was an error that was
  raised by the command invocation.

Tests were added to verify the new code paths.

In addition small refactoring was done to share some code between modules,
scripts, and `processCommand` function:
1. `getAclErrorMessage` was added to `acl.c` to unified to log message extraction
  from the acl result
2. `checkGoodReplicasStatus` was added to `replication.c` to check the status of
  good replicas. It is used on `scriptVerifyWriteCommandAllow`, `RM_Call`, and
  `processCommand`.
3. `writeCommandsGetDiskErrorMessage` was added to `server.c` to get the error
  message on persistence failure. Again it is used on `scriptVerifyWriteCommandAllow`,
  `RM_Call`, and `processCommand`.
 2022-03-22 14:13:28 +02:00
yiyuaner 08aed7e7dd Fix an off by one error in zzlStrtod (#10465) 
When vlen = sizeof(buf), the statement buf[vlen] = '\0' accessing the buffer buf is an off by one error.
 2022-03-22 10:46:16 +02:00
zhaozhao.zz 79db037a4f config rewrite enhancement, in case of line longer than 1024 (#8009) 
When rewrite the config file, we need read the old config file first,
but the CONFIG_MAX_LEN is 1024, so if some lines are longer than it,
it will generate a wrong config file, and redis cannot reboot from
the new config file.

Rename CONFIG_MAX_LINE to CONFIG_READ_LEN
 2022-03-22 16:34:01 +08:00
郭伟光 93dda65354 config rewrite failed save errno in case of being tainted (#10461) 
errno would be potentially tainted during the serverLog() by the file io functions, such as fopen and fflush.
 2022-03-21 14:20:23 +02:00
Ozan Tezcan 4517fadb59 Use exit code 1 if redis-cli fails to connect (#10438) 
Use exit code 1 if redis-cli fails to connect.

Before https://github.com/redis/redis/pull/10382/, on a connection failure,
exit code would be 1.  After this PR, whether connection is established or not,
`noninteractive()` return value is used as the exit code. On a failure, this function
returns `REDIS_ERR` which is `-1`. It becomes `255` as exit codes are between `0-255`.

There is nothing wrong by returning 1 or 255 on failure as far as I know but it'll break
things that expect to see 1 as exit code on a connection failure. This is also how we
realized the issue. With this PR, changing behavior back to using 1 as exit code to
preserve backward compatibility.
 2022-03-21 13:40:02 +02:00
guybe7 e82c1aedea BITSET and BITFIELD SET should propagate even if just length changed (#10459) 
Bug introduced in #9403, caused inconsistency between master and
replica in case just the length (i.e. set a high-index bit to 0)
changed.
 2022-03-21 11:33:27 +02:00
Meir Shpilraien (Spielrein) 2f9cdcd733 Increase function tests timeout (#10458) 
Increase function tests timeout to avoid false failures on
slow systems.
 2022-03-21 11:00:27 +02:00
Madelyn Olson 557222d1e0 Fix timing issue in shards test and fix displayed TLS port (#10450) 2022-03-20 22:08:40 -07:00
郭伟光 fae5b1a19d unblockClient: avoid to reset client when the client was shutdown-blocked (#10440) 
fix #10439. see https://github.com/redis/redis/pull/9872
When executing SHUTDOWN we pause the client so we can un-pause it
if the shutdown fails.
this could happen during the timeout, if the shutdown is aborted, but could
also happen from withing the initial `call()` to shutdown, if the rdb save fails.
in that case when we return to `call()`, we'll crash if `c->cmd` has been set to NULL.

The call stack is:
```
unblockClient(c)
replyToClientsBlockedOnShutdown()
cancelShutdown()
finishShutdown()
prepareForShutdown()
shutdownCommand()
```

what's special about SHUTDOWN in that respect is that it can be paused,
and then un-paused before the original `call()` returns.
tests where added for both failed shutdown, and a followup successful one.
 2022-03-20 15:18:53 +02:00
sundb b9656adbd9 Restore ::singledb after cluster test (#10441) 
When ::singledb is 0, we will use db 9 for the test db.
Since ::singledb is set to 1 in the cluster-related tests, but not restored, some subsequent
tests associated with db 9 will fail.
 2022-03-18 14:10:24 +02:00
carlosfu b69636d377 fix wrong comment about cluster_announce_hostname (#10274) 2022-03-16 20:26:30 -07:00
Madelyn Olson e8771efda9 Fixed incorrect parsing of hostname information from nodes.conf (#10435) 2022-03-16 14:07:24 -07:00
Viktor Söderqvist 69017fa232 Fix redis-cli CLUSTER SETSLOT race conditions (#10381) 
After migrating a slot, send CLUSTER SETSLOT NODE to the destination
node first to make sure the slot isn't left without an owner in case
the destination node crashes before it is set as new owner.

When informing the source node, it can happen that the destination
node has already informed it and if the source node has lost its
last slot, it has already turned itself into a replica. Redis-cli
should ignore this error in this case.
 2022-03-16 10:11:38 -07:00
Binbin 61b7e5916d Fix module redact test for valgrind (#10432) 
The new module redact test will fail with valgrind:
```
[err]: modules can redact arguments in tests/unit/moduleapi/auth.tcl
Expected 'slowlog reset' to be equal to 'auth.redact 1 (redacted) 3 (redacted)' (context: type eval line 12 cmd {assert_equal {slowlog reset} [lindex [lindex [r slowlog get] 2] 3]} proc ::test)
```

The reason is that with `slowlog-log-slower-than 10000`,
`slowlog get` will have a chance to exceed 10ms.

Made two changes to avoid failure:
1. change `slowlog-log-slower-than` from 10000 to -1, distable it.
2. assert to use the previous execution result.

In theory, the second one can actually be left unchanged, but i
think it will be better if it is changed.
 2022-03-16 08:53:57 +02:00
Harkrishn PatroandMadelyn Olson 45ccae89bb Add new cluster shards command (#10293) 
Implement a new cluster shards command, which provides a flexible and extensible API for topology discovery.

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
 2022-03-15 18:24:40 -07:00
Madelyn Olson 416c9ac2ef Add module API for redacting command arguments (#10425) 
Add module API for redacting client commands
 2022-03-15 18:21:13 -07:00
Wen Hui c30de7073c Sentinel: update command json files (#10374) 2022-03-15 18:38:07 +02:00
ranshidMadelyn OlsonOran Agra
 		1078e30c5f make sort/ro commands validate external keys access patterns (#10106) (#10340) 
Currently the sort and sort_ro can access external keys via `GET` and `BY`
in order to make sure the user cannot violate the authorization ACL
rules, the decision is to reject external keys access patterns unless ACL allows
SORT full access to all keys.
I.e. for backwards compatibility, SORT with GET/BY keeps working, but
if ACL has restrictions to certain keys, these features get permission denied.

### Implemented solution
We have discussed several potential solutions and decided to only allow the GET and BY
arguments when the user has all key permissions with the SORT command. The reasons
being that SORT with GET or BY is problematic anyway, for instance it is not supported in
cluster mode since it doesn't declare keys, and we're not sure the combination of that feature
with ACL key restriction is really required.
**HOWEVER** If in the fullness of time we will identify a real need for fine grain access
support for SORT, we would implement the complete solution which is the alternative
described below.

### Alternative (Completion solution):
Check sort ACL rules after executing it and before committing output (either via store or
to COB). it would require making several changes to the sort command itself. and would
potentially cause performance degradation since we will have to collect all the get keys
instead of just applying them to a temp array and then scan the access keys against the
ACL selectors. This solution can include an optimization to avoid the overheads of collecting
the key names, in case the ACL rules grant SORT full key-access, or if the ACL key pattern
literal matches the one used in GET/BY. It would also mean that authorization would be
O(nlogn) since we will have to complete most of the command execution before we can
perform verification

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-03-15 17:14:53 +02:00
yoav-steinberg cf6dcb7bf1 Optimization: remove updateClientMemUsage from i/o threads. (#10401) 
In a benchmark we noticed we spend a relatively long time updating the client
memory usage leading to performance degradation.
Before #8687 this was performed in the client's cron and didn't affect performance.
But since introducing client eviction we need to perform this after filling the input
buffers and after processing commands. This also lead me to write this code to be
thread safe and perform it in the i/o threads.

It turns out that the main performance issue here is related to atomic operations
being performed while updating the total clients memory usage stats used for client
eviction (`server.stat_clients_type_memory[]`). This update needed to be atomic
because `updateClientMemUsage()` was called from the IO threads.

In this commit I make sure to call `updateClientMemUsage()` only from the main thread.
In case of threaded IO I call it for each client during the "fan-in" phase of the read/write
operation. This also means I could chuck the `updateClientMemUsageBucket()` function
which was called during this phase and embed it into `updateClientMemUsage()`.

Profiling shows this makes `updateClientMemUsage()` (on my x86_64 linux) roughly x4 faster.
 2022-03-15 14:18:23 +02:00
Binbin 871fa12fec Sentinel: fix reconnect test timing issue (#10424) 
We need to wait for `sentinelTimer` to kick in, and then trigger the reconnect.

As for another change, we should better call `server_set_password` before calling SENTINEL SET auth-pass.

Fixes problem introeuced in #10400
 2022-03-14 11:13:14 +02:00
DarrenJiang13 38ed6c6007 improve string2ll() to avoid extra conversion for long integer string. (#10408) 
For an integer string like "123456789012345678901" which could cause
overflow-failure in string2ll() conversion, we could compare its length at
the beginning to avoid extra work.

* move LONG_STR_SIZE to be in declared in util.h, next to MAX_LONG_DOUBLE_CHARS
 2022-03-14 08:22:57 +02:00
郭伟光 dc7a9d3a31 Cleanup: replicationFeedMonitors use the monitor list arg it got (#10417) 
Better check the monitors list argument instead of server.monitors in the function,
although they are basically the same in the context, so this doesn't have any
impact on the current code.
 2022-03-13 16:19:42 +02:00
Moti Cohen a6bf509810 Sentinel: fix no reconnect after auth-pass is changed (#10400) 
When updating SENTINEL with master’s new password (command:
`SENTINEL SET mymaster auth-pass some-new-password`), 
sentinel might still keep the old connection and avoid reconnecting 
with the new password. This is because of wrong logic that traces 
the last ping (pong) time to servers. In fact it worked fine until 8631e64 
changed the condition to send ping. To resolve it with minimal risk, 
let’s disconnect master and replicas once changing password/user. 

Based on earlier work of yz1509.
 2022-03-13 10:13:47 +02:00
Wen Hui b576fbc474 Remove redundancy GETNAME in client help command message (#10418) 
probably a copy paste error.
 2022-03-13 08:27:41 +02:00
Binbin 1797330e2e Initialize help when using redis-cli help or redis-cli ? (#10382) 
The following usage will output an empty newline:
```
> redis-cli help set
empty line
```

The reason is that in interactive mode, we have called
`cliInitHelp`, which initializes help.

When using `redis-cli help xxx` or `redis-cli help ? xxx`,
we can't match the command due to empty `helpEntries`,
so we output an empty newline.

In this commit, we will call `cliInitHelp` to init the help.
Note that in this case, we need to call `cliInitHelp` (COMMAND DOCS)
every time, which i think is acceptable.

So now the output will look like:
```
[redis]# src/redis-cli help get

  GET key
  summary: Get the value of a key
  since: 1.0.0
  group: string

[redis]#
```

Fixes #10378

This PR also fix a redis-cli crash when using `--ldb --eval`:
```
[root]# src/redis-cli --ldb --eval test.lua test 1
Lua debugging session started, please use:
quit    -- End the session.
restart -- Restart the script in debug mode again.
help    -- Show Lua script debugging commands.

* Stopped at 1, stop reason = step over
-> 1   local num = redis.call('GET', KEYS[1]);
redis-cli: redis-cli.c:718: cliCountCommands: Assertion
`commandTable->element[i]->type == 1' failed.
Aborted
```
Because in ldb mode, `COMMAND DOCS` or `COMMAND` will
return an array, only with one element, and the type
is `REDIS_REPLY_STATUS`, the result is `<error> Unknown
Redis Lua debugger command or wrong number of arguments`.

So if we are in the ldb mode, and init the Redis HELP, we
will get the wrong response and crash the redis-cli.
In ldb mode we don't initialize HELP, help is only initialized
after the lua debugging session ends.

It was broken in #10043
 2022-03-10 18:20:01 +02:00
ranshid 11b071a22b ACL DRYRUN does not validate the verified command args. (#10405) 
As a result we segfault when parsing and matching the command keys.
 2022-03-10 10:08:41 +02:00
zhugezy a26cab9dd6 set "disable-thp" config immutable (#10409) 
It's confusing for this config to be modifiable since it only takes effect on startup
 2022-03-10 09:52:49 +02:00
rangerzhang 4e012daee9 Fix outdated comments on updateSlavesWaitingBgsave (#10394) 
* fix-replication-comments

The described capacity
 `and to schedule a new BGSAVE if there are slaves that attached while a BGSAVE was in progress`
was moved to `checkChildrenDone()`  named by `replicationStartPendingFork`

But the comment was not changed, may misleading others.

* remove-misleading-comments

The described capacity
 `to schedule a new BGSAVE if there are slaves that attached while a BGSAVE was in progress` 
and 
`or when the replication RDB transfer strategy is modified from disk to socket or the other way around` 
were not correct now.
 2022-03-10 09:51:55 +02:00
Oran Agra 311a757cb1 Some adjustments to command hints. (#10375) 
* stats and latency commands have non-deterministic output.
* the ones about latency should be sent to ALL_NODES (considering
  reads from replicas)
* the ones about running scripts and memory usage only to masters.
* stats aggregation is SPECIAL (like in INFO)
 2022-03-09 16:40:27 +02:00
a2tt 86ca9b25e2 fix typos (#10402) 2022-03-09 13:58:23 +02:00
蔡相跃 24da71e507 Fix typo "the the" (#10399) 2022-03-09 13:55:17 +02:00
sundb adc5a3217c Use dismissMemory to dismiss COW of client output buffer (#10403) 
c->buf is not sds, so we should use dismissMemory instead of dismissSds to dismiss it.
This is a recent regression from #10371
 2022-03-09 13:32:03 +02:00
Ronald Petty b104f3cabc Update redis.conf (#10396) 
Typo in conf file comment.
 2022-03-08 12:52:54 -08:00
guybe7 2a2954086a XREADGROUP: Unblock client if stream is deleted (#10306) 
Deleting a stream while a client is blocked XREADGROUP should unblock the client.

The idea is that if a client is blocked via XREADGROUP is different from
any other blocking type in the sense that it depends on the existence of both
the key and the group. Even if the key is deleted and then revived with XADD
it won't help any clients blocked on XREADGROUP because the group no longer
exist, so they would fail with -NOGROUP anyway.
The conclusion is that it's better to unblock these clients (with error) upon
the deletion of the key, rather than waiting for the first XADD. 

Other changes:
1. Slightly optimize all `serveClientsBlockedOn*` functions by checking `server.blocked_clients_by_type`
2. All `serveClientsBlockedOn*` functions now use a list iterator rather than looking at `listFirst`, relying
  on `unblockClient` to delete the head of the list. Before this commit, only `serveClientsBlockedOnStreams`
  used to work like that.
3. bugfix: CLIENT UNBLOCK ERROR should work even if the command doesn't have a timeout_callback
  (only relevant to module commands)
 2022-03-08 17:10:36 +02:00
zhaozhao.zz 728e62523e script should not allow may-replicate commands when client pause write (#10364) 
In some special commands like eval_ro / fcall_ro we allow no-writes commands.
But may-replicate commands are no-writes too, that leads crash when client pause write:
 2022-03-08 16:53:11 +02:00
Oran Agra b3fe4f31a2 dismiss COW of client output buffer now that it's dynamic (#10371) 
since #9822, the static reply buffer is no longer part of the client structure, so we need to dismiss it.
 2022-03-08 15:17:15 +02:00
zhugezy 4f19b4d0c1 remove a piece of redundant comment (#10392) 
introduced in #10147 since we blocked the first-arg mechanism on subcommands
 2022-03-08 13:35:37 +02:00
Yossi Gottlieb 38052fd702 Fix redis-benchmark --cluster with IPv6. (#10393) 
Currently, CLUSTER NODES is parsed and was not done correctly for IPv6
addresses.
 2022-03-08 13:10:24 +02:00
Binbin 7ef68cd5ec Add missing doc_flags to cluster-slaves (#10387) 
Add `DEPRECATED` doc_flag.
 2022-03-07 17:59:51 +02:00
Shaya Potter 23f03e7965 Modules: Add REDISMODULE_EVENT_CONFIG (#10311) 
Add a new REDISMODULE_EVENT_CONFIG event type for notifying modules when Redis configuration changes.
 2022-03-07 17:37:57 +02:00
Binbin 45d83fb2d4 Fix timing issue in rehash test (#10388) 
`Expected '*table size: 4096*' to match '*table size: 8192*'`

This test failed once on daily macOS, the reason is because
the bgsave has not stopped after the kill and `after 200`.
So there is a child process and no rehash triggered.

This commit use `waitForBgsave` to wait for it to finish.
 2022-03-07 13:44:07 +02:00
Yossi Gottlieb 6740e1753d Fix redis-cli test issues on tcl8.5. (#10386) 
Apparently using `\x` produces different results between tclsh 8.5 and
8.6, whereas `\u` is more consistent.
 2022-03-06 13:02:35 +02:00
Yuta Hongo e3ef73dc2a redis-cli: Better --json Unicode support and --quoted-json (#10286) 
Normally, `redis-cli` escapes non-printable data received from Redis, using a custom scheme (which is also used to handle quoted input). When using `--json` this is not desired as it is not compatible with RFC 7159, which specifies JSON strings are assumed to be Unicode and how they should be escaped.

This commit changes `--json` to follow RFC 7159, which means that properly encoded Unicode strings in Redis will result with a valid Unicode JSON.

However, this introduces a new problem with `--json` and data that is not valid Unicode (e.g., random binary data, text that follows other encoding, etc.). To address this, we add `--quoted-json` which produces JSON strings that follow the original redis-cli quoting scheme.

For example, a value that consists of only null (0x00) bytes will show up as:
* `"\u0000\u0000\u0000"` when using `--json`
* `"\\x00\\x00\\x00"` when using `--quoted-json`
 2022-03-05 21:25:52 +02:00
Binbin af6d5c5932 Constrain cluster node name logging (#10376) 
Cluster node name is not null terminated, so need to be constrained.
 2022-03-02 22:41:31 -08:00
Henry feb032fd42 A faster and more robust code of zslRandomLevel using RAND_MAX (#5539) 
1. since ZSKIPLIST_P is float, using it directly inside the condition used to causes floating point code to be used (gcc/x86)
2. In some operating system(eg.Windows), the largest value returned from random() is 0x7FFF(15bit), so after bitwise AND with 0xFFFF, the probability of the less operation returning true in the while loop's condition is no more equal to ZSKIPLIST_P.
3. In case some library has random() returning int in range [0~ZSKIPLIST_P*65535], the while loop will be an infinite loop.
4. on Linux where RAND_MAX is higher than 0xFFFF, this change actually improves precision (despite not matching the result against a float value)
 2022-03-02 10:40:39 +02:00
ranshidandyoav-steinberg yoav@redislabs.com 9b15dd288e Introduce debug command to disable reply buffer resizing (#10360) 
In order to resolve some flaky tests which hard rely on examine memory footprint.
we introduce the following fixes:

# Fix in client-eviction test - by @yoav-steinberg 
Sometime the libc allocator can use different size client struct allocations.
this may cause unexpected memory calculations to fail the test.

# Introduce new DEBUG command for disabling reply buffer resizing
In order to eliminate reply buffer resizing during specific tests.
we introduced the ability to disable (and enable) the resizing cron job

Co-authored-by: yoav-steinberg yoav@redislabs.com
 2022-03-01 14:40:29 +02:00
Madelyn Olson 4a45386e3c Move most of the configuration to a hashtable (#10323) 
* Moved configuration storage from a list to a hash table
* Configs are returned in a non-deterministic order. It's possible that a client was relying on order (hopefully not).
* Fixed an esoteric bug where if you did a set with an alias with an error, it would throw an error indicating a bug with the preferred name for that config.
 2022-02-28 23:02:47 -08:00
Harkrishn Patro 21aabab401 Fix acl dryrun to return the tested common permission error. (#10359) 2022-02-28 20:26:58 -08:00
Vitah Lin dff153ff24 Fix memory leak in RM_StreamIteratorStop and moduleFreeKeyIterator (#10353) 
* Fix memory leak in RM_StreamIteratorStop
* Fix memory leak in moduleFreeKeyIterator
 2022-02-28 17:06:39 +02:00
Oran Agra d2b5a579dd Merge pull request #10355 from oranagra/release-7.0-rc2
CI / build-libc-malloc (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CodeQL / Analyze (cpp) (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
CI / build-32bit (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
CI / test-ubuntu-latest (push) Has been cancelled
Details 
Release 7.0 RC2
 2022-02-28 15:35:46 +02:00
Oran Agra 9478d5a134 enable daily CI on release branches (#10357) 2022-02-28 13:17:56 +02:00
Oran Agra 10dc57ab22 7.0 RC2 2022-02-28 11:39:04 +02:00
Oran Agra a83e36637a Merge remote-tracking branch 'origin/unstable' into 7.0 2022-02-28 11:35:34 +02:00
ranshidandOran Agra 5860fa3d9c deflake client-eviction test "evict clients only until below limit" (#10354) 
After introducing #9822 need to prevent client reply buffer shrink
to maintain correct client memory math.

add needs:debug missing one one test.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-28 11:32:42 +02:00
Oran Agra fa5d90069f update redis-cli help.h (#10352) 
re-generate help.h from commands.json
 2022-02-28 10:59:16 +02:00
Binbin ad7a6275ff Fix node-id type in cluster-setslot (#10348) 
* The type of node-id should be string, not integer.
* Also improve the CLUSTER SETSLOT help message.
 2022-02-27 23:46:57 -08:00
chenyang8094 a6fd237537 Make git ignore all files starting with appendonly.aof (#10351) 2022-02-28 09:24:47 +02:00
Meir Shpilraien (Spielrein)andOran Agra aa856b39f2 Sort out the mess around Lua error messages and error stats (#10329) 
This PR fix 2 issues on Lua scripting:
* Server error reply statistics (some errors were counted twice).
* Error code and error strings returning from scripts (error code was missing / misplaced).

## Statistics
a Lua script user is considered part of the user application, a sophisticated transaction,
so we want to count an error even if handled silently by the script, but when it is
propagated outwards from the script we don't wanna count it twice. on the other hand,
if the script decides to throw an error on its own (using `redis.error_reply`), we wanna
count that too.
Besides, we do count the `calls` in command statistics for the commands the script calls,
we we should certainly also count `failed_calls`.
So when a simple `eval "return redis.call('set','x','y')" 0` fails, it should count the failed call
to both SET and EVAL, but the `errorstats` and `total_error_replies` should be counted only once.

The PR changes the error object that is raised on errors. Instead of raising a simple Lua
string, Redis will raise a Lua table in the following format:

```
{
    err='<error message (including error code)>',
    source='<User source file name>',
    line='<line where the error happned>',
    ignore_error_stats_update=true/false,
}
```

The `luaPushError` function was modified to construct the new error table as describe above.
The `luaRaiseError` was renamed to `luaError` and is now simply called `lua_error` to raise
the table on the top of the Lua stack as the error object.
The reason is that since its functionality is changed, in case some Redis branch / fork uses it,
it's better to have a compilation error than a bug.

The `source` and `line` fields are enriched by the error handler (if possible) and the
`ignore_error_stats_update` is optional and if its not present then the default value is `false`.
If `ignore_error_stats_update` is true, the error will not be counted on the error stats.

When parsing Redis call reply, each error is translated to a Lua table on the format describe
above and the `ignore_error_stats_update` field is set to `true` so we will not count errors
twice (we counted this error when we invoke the command).

The changes in this PR might have been considered as a breaking change for users that used
Lua `pcall` function. Before, the error was a string and now its a table. To keep backward
comparability the PR override the `pcall` implementation and extract the error message from
the error table and return it.

Example of the error stats update:

```
127.0.0.1:6379> lpush l 1
(integer) 2
127.0.0.1:6379> eval "return redis.call('get', 'l')" 0
(error) WRONGTYPE Operation against a key holding the wrong kind of value. script: e471b73f1ef44774987ab00bdf51f21fd9f7974a, on @user_script:1.

127.0.0.1:6379> info Errorstats
# Errorstats
errorstat_WRONGTYPE:count=1

127.0.0.1:6379> info commandstats
# Commandstats
cmdstat_eval:calls=1,usec=341,usec_per_call=341.00,rejected_calls=0,failed_calls=1
cmdstat_info:calls=1,usec=35,usec_per_call=35.00,rejected_calls=0,failed_calls=0
cmdstat_lpush:calls=1,usec=14,usec_per_call=14.00,rejected_calls=0,failed_calls=0
cmdstat_get:calls=1,usec=10,usec_per_call=10.00,rejected_calls=0,failed_calls=1
```

## error message
We can now construct the error message (sent as a reply to the user) from the error table,
so this solves issues where the error message was malformed and the error code appeared
in the middle of the error message:

```diff
127.0.0.1:6379> eval "return redis.call('set','x','y')" 0
-(error) ERR Error running script (call to 71e6319f97b0fe8bdfa1c5df3ce4489946dda479): @user_script:1: OOM command not allowed when used memory > 'maxmemory'.
+(error) OOM command not allowed when used memory > 'maxmemory' @user_script:1. Error running script (call to 71e6319f97b0fe8bdfa1c5df3ce4489946dda479)
```

```diff
127.0.0.1:6379> eval "redis.call('get', 'l')" 0
-(error) ERR Error running script (call to f_8a705cfb9fb09515bfe57ca2bd84a5caee2cbbd1): @user_script:1: WRONGTYPE Operation against a key holding the wrong kind of value
+(error) WRONGTYPE Operation against a key holding the wrong kind of value script: 8a705cfb9fb09515bfe57ca2bd84a5caee2cbbd1, on @user_script:1.
```

Notica that `redis.pcall` was not change:
```
127.0.0.1:6379> eval "return redis.pcall('get', 'l')" 0
(error) WRONGTYPE Operation against a key holding the wrong kind of value
```


## other notes
Notice that Some commands (like GEOADD) changes the cmd variable on the client stats so we
can not count on it to update the command stats. In order to be able to update those stats correctly
we needed to promote `realcmd` variable to be located on the client struct.

Tests was added and modified to verify the changes.

Related PR's: #10279, #10218, #10278, #10309

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-27 13:40:57 +02:00
filipe oliveira 9f30dd03cd Enable redis-benchmark to use RESP3 protocol mode (#10335) 
Adds `-3` option to cause redis-benchmark to send a `HELLO 3`
to it can benchmark the effects of RESP3 on the server.
 2022-02-27 10:30:39 +02:00
Madelyn Olson 35fccd875a Fixed typo in variable name (#10347) 2022-02-26 19:52:09 -08:00
filipe oliveiraandOran Agra 1dc89e2d02 Optimization: Avoid deferred array reply on ZRANGE commands BYRANK (#10337) 
Avoid deferred array reply on genericZrangebyrankCommand() when consumer type is client.
I.e. any ZRANGE / ZREVRNGE (when tank is used).
This was a performance regression introduced in #7844 (v 6.2) mainly affecting pipelined workloads.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-24 14:20:00 +02:00
Yossi Gottlieb e8c5b66ed2 Update to latest hiredis (#10297) 
This is basically just a subtree pull of the latest (unreleased) hiredis.

Unfortunately, the `sds -> hisds` patch was pulled as a subtree update from a remote branch rather than a local redis change. Because of that, it goes away on every subtree update. It is now applied as a local commit so it should survive in the future.
 2022-02-24 10:51:04 +02:00
Binbin b4079abed4 Fix XGROUP HELP message missing a newline (#10339) 
Add a comma, this would have resulted in missing newline in the message.
Forgot to add in #9127
 2022-02-24 08:31:39 +02:00
Itamar HaberGuy BenoishOran Agra
 		c81c7f51c3 Add stream consumer group lag tracking and reporting (#9127) 
Adds the ability to track the lag of a consumer group (CG), that is, the number
of entries yet-to-be-delivered from the stream.

The proposed constant-time solution is in the spirit of "best-effort."

Partially addresses #8737.

## Description of approach

We add a new "entries_added" property to the stream. This starts at 0 for a new
stream and is incremented by 1 with every `XADD`.  It is essentially an all-time
counter of the entries added to the stream.

Given the stream's length and this counter value, we can trivially find the logical
"entries_added" counter of the first ID if and only if the stream is contiguous.
A fragmented stream contains one or more tombstones generated by `XDEL`s.
The new "xdel_max_id" stream property tracks the latest tombstone.

The CG also tracks its last delivered ID's as an "entries_read" counter and
increments it independently when delivering new messages, unless the this
read counter is invalid (-1 means invalid offset). When the CG's counter is
available, the reported lag is the difference between added and read counters.

Lastly, this also adds a "first_id" field to the stream structure in order to make
looking it up cheaper in most cases.

## Limitations

There are two cases in which the mechanism isn't able to track the lag.
In these cases, `XINFO` replies with `null` in the "lag" field.

The first case is when a CG is created with an arbitrary last delivered ID,
that isn't "0-0", nor the first or the last entries of the stream. In this case,
it is impossible to obtain a valid read counter (short of an O(N) operation).
The second case is when there are one or more tombstones fragmenting
the stream's entries range.

In both cases, given enough time and assuming that the consumers are
active (reading and lacking) and advancing, the CG should be able to
catch up with the tip of the stream and report zero lag.
Once that's achieved, lag tracking would resume as normal (until the
next tombstone is set).

## API changes

* `XGROUP CREATE` added with the optional named argument `[ENTRIESREAD entries-read]`
  for explicitly specifying the new CG's counter.
* `XGROUP SETID` added with an optional positional argument `[ENTRIESREAD entries-read]`
  for specifying the CG's counter.
* `XINFO` reports the maximal tombstone ID, the recorded first entry ID, and total
  number of entries added to the stream.
* `XINFO` reports the current lag and logical read counter of CGs.
* `XSETID` is an internal command that's used in replication/aof. It has been added with
  the optional positional arguments `[ENTRIESADDED entries-added] [MAXDELETEDID max-deleted-entry-id]`
  for propagating the CG's offset and maximal tombstone ID of the stream.

## The generic unsolved problem

The current stream implementation doesn't provide an efficient way to obtain the
approximate/exact size of a range of entries. While it could've been nice to have
that ability (#5813) in general, let alone specifically in the context of CGs, the risk
and complexities involved in such implementation are in all likelihood prohibitive.

## A refactoring note

The `streamGetEdgeID` has been refactored to accommodate both the existing seek
of any entry as well as seeking non-deleted entries (the addition of the `skip_tombstones`
argument). Furthermore, this refactoring also migrated the seek logic to use the
`streamIterator` (rather than `raxIterator`) that was, in turn, extended with the
`skip_tombstones` Boolean struct field to control the emission of these.

Co-authored-by: Guy Benoish <guy.benoish@redislabs.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-23 22:34:58 +02:00
filipe oliveiraandOran Agra b857928ba7 Optimize deferred replies to use shared objects instead of sprintf (#10334) 
Avoid sprintf/ll2string on setDeferredAggregateLen()/addReplyLongLongWithPrefix() when we can used shared objects.
In some pipelined workloads this achieves about 10% improvement.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-23 18:15:12 +02:00
Moti Cohen a7179e7570 Sentinel: fix a free-after-use issue re-registering Sentinels. (#10333) 
In case HELLO message received from another sentinel, with same address like another instance registered in the past but with different runid. Then there was cumbersome logic to modify the instance the port to 0 to in order to mark as invalid and later on to delete it. But the deletion is happening during update of instances in such a way that we might end up accessing an instance that was deleted just before.

Didn't find a good reason why to postpone the deletion action of an obsolete instance (deletion is taking place instantly, for other cases ) -> Lets delete at once
There is a mixture of logic of Sentinel address update with the logic of deletion of Sentinels that match a given Address -> Split to two!
 2022-02-23 17:59:22 +02:00
Binbin 488aecb3ab Fix timing issue in EXEC fail on lazy expired WATCHed key test (#10332) 
The test will fail on slow machines (valgrind or FreeBsd).
Because in #10256 when WATCH is called on a key that's already
logically expired, we will add an `expired` flag, and we will
skip it in `isWatchedKeyExpired` check.

Apparently we need to increase the expiration time so that
the key can not expire logically then the WATCH is called.
Also added retries to make sure it doesn't fail. I suppose
100ms is enough in valgrind, tested locally, no need to retry.
 2022-02-23 08:47:16 +02:00
Wen Hui de6be8850f Fix PUBSUB SHARDNUMSUB command json file (#10331) 
argument was missing, affecting redis.io docs
 2022-02-22 23:29:29 +02:00
Andy Pan 496375fc36 Reduce system calls of write for client->reply by introducing writev (#9934) 
There are scenarios where it results in many small objects in the reply list,
such as commands heavily using deferred array replies (`addReplyDeferredLen`).
E.g. what COMMAND command and CLUSTER SLOTS used to do (see #10056, #7123),
but also in case of a transaction or a pipeline of commands that use just one differed array reply.

We used to have to run multiple loops along with multiple calls to `write()` to send data back to
peer based on the current code, but by means of `writev()`, we can gather those scattered
objects in reply list and include the static reply buffer as well, then send it by one system call,
that ought to achieve higher performance.

In the case of TLS,  we simply check and concatenate buffers into one big buffer and send it
away by one call to `connTLSWrite()`, if the amount of all buffers exceeds `NET_MAX_WRITES_PER_EVENT`,
then invoke `connTLSWrite()` multiple times to avoid a huge massive of memory copies.

Note that aside of reducing system calls, this change will also reduce the amount of
small TCP packets sent.
 2022-02-22 14:00:37 +02:00
Viktor SöderqvistOran Agrazhaozhao.zz
 		e9ae03787e Delete key doesn't dirty client who watched stale key (#10256) 
When WATCH is called on a key that's already logically expired, avoid discarding the
transaction when the keys is actually deleted.

When WATCH is called, a flag is stored if the key is already expired
at the time of watch. The expired key is not deleted, only checked.

When a key is "touched", if it is deleted and it was already expired
when a client watched it, the client is not marked as dirty.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: zhaozhao.zz <zhaozhao.zz@alibaba-inc.com>
 2022-02-22 12:09:46 +02:00
ranshidOran AgraYoav Steinberg
 		47c51d0c78 introduce dynamic client reply buffer size - save memory on idle clients (#9822) 
Current implementation simple idle client which serves no traffic still
use ~17Kb of memory. this is mainly due to a fixed size reply buffer
currently set to 16kb.

We have encountered some cases in which the server operates in a low memory environments.
In such cases a user who wishes to create large connection pools to support potential burst period,
will exhaust a large amount of memory  to maintain connected Idle clients.
Some users may choose to "sacrifice" performance in order to save memory.

This commit introduce a dynamic mechanism to shrink and expend the client reply buffer based on
periodic observed peak.
the algorithm works as follows:
1. each time a client reply buffer has been fully written, the last recorded peak is updated: 
new peak = MAX( last peak, current written size)
2. during clients cron we check for each client if the last observed peak was:
     a. matching the current buffer size - in which case we expend (resize) the buffer size by 100%
     b. less than half the buffer size - in which case we shrink the buffer size by 50%
3. In any case we will **not** resize the buffer in case:
    a. the current buffer peak is less then the current buffer usable size and higher than 1/2 the
      current buffer usable size
    b. the value of (current buffer usable size/2) is less than 1Kib
    c. the value of  (current buffer usable size*2) is larger than 16Kib
4. the peak value is reset to the current buffer position once every **5** seconds. we maintain a new
   field in the client structure (buf_peak_last_reset_time) which is used to keep track of how long it
   passed since the last buffer peak reset.

### **Interface changes:**
**CIENT LIST** - now contains 2 new extra fields:
rbs= < the current size in bytes of the client reply buffer >
rbp=< the current value in bytes of the last observed buffer peak position >

**INFO STATS** - now contains 2 new statistics:
reply_buffer_shrinks = < total number of buffer shrinks performed >
reply_buffer_expends = < total number of buffer expends performed >

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Yoav Steinberg <yoav@redislabs.com>
 2022-02-22 11:19:38 +02:00
Madelyn Olson 71204f9632 Implemented module getchannels api and renamed channel keyspec (#10299) 
This implements the following main pieces of functionality:
* Renames key spec "CHANNEL" to be "NOT_KEY", and update the documentation to
  indicate it's for cluster routing and not for any other key related purpose.
* Add the getchannels-api, so that modules can now define commands that are subject to
  ACL channel permission checks. 
* Add 4 new flags that describe how a module interacts with a command (SUBSCRIBE, PUBLISH,
  UNSUBSCRIBE, and PATTERN). They are all technically composable, however not sure how a
  command could both subscribe and unsubscribe from a command at once, but didn't see
  a reason to add explicit validation there.
* Add two new module apis RM_ChannelAtPosWithFlags and RM_IsChannelsPositionRequest to
  duplicate the functionality provided by the keys position APIs.
* The RM_ACLCheckChannelPermissions (only released in 7.0 RC1) was changed to take flags
  rather than a boolean literal.
* The RM_ACLCheckKeyPermissions (only released in 7.0 RC1) was changed to take flags
  corresponding to keyspecs instead of custom permission flags. These keyspec flags mimic
  the flags for ACLCheckChannelPermissions.
 2022-02-22 11:00:03 +02:00
Binbin c4c68f5d41 Remove ALLOW_BUSY from REPLICAOF and add it to REPLCONF. Add DEPRECATED doc flag to SLAVEOF, mark it as deprecated. (#10315) 
* Remove ALLOW_BUSY from REPLICAOF and add it to REPLCONF
* mark SLAVEOF as deprecated
 2022-02-22 10:55:54 +02:00
chenyang8094 4916d79fbd Fix path copy error and add more logs. (#10324) 
Since we didn't copy the null terminator to temp_filepath, dirname could return the wrong result.
 2022-02-22 10:09:34 +02:00
YaacovHazanchenyang8094Oran Agra
 		65e4bce0e7 fix return value of loadAppendOnlyFiles (#10295) 
Make sure the status return from loading multiple AOF files reflects the overall
result, not just the one of the last file.

When one of the AOF files succeeded to load, but the last AOF file
was empty, the loadAppendOnlyFiles will return AOF_EMPTY.
This commit changes this behavior, and return AOF_OK in that case.

This can happen for example, when loading old AOF file, and no more commands processed,
the manifest file will include base AOF file with data, and empty incr AOF file.

Co-authored-by: chenyang8094 <chenyang8094@users.noreply.github.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-22 08:59:23 +02:00
Oran Agra fad0b0d2a6 Fix error stats and failed command stats for blocked clients (#10309) 
This is a followup work for #10278, and a discussion about #10279

The changes:
- fix failed_calls in command stats for blocked clients that got error.
  including CLIENT UNBLOCK, and module replying an error from a thread.
- fix latency stats for XREADGROUP that filed with -NOGROUP

Theory behind which errors should be counted:
- error stats represents errors returned to the user, so an error handled by a
  module should not be counted.
- total error counter should be the same.
- command stats represents execution of commands (even with RM_Call, and if
  they fail or get rejected it counts these calls in commandstats, so it should
  also count failed_calls)

Some thoughts about Scripts:
for scripts it could be different since they're part of user code, not the infra (not an extension to redis)
we certainly want commandstats to contain all calls and errors
a simple script is like mult-exec transaction so an error inside it should be counted in error stats
a script that replies with an error to the user (using redis.error_reply) should also be counted in error stats
but then the problem is that a plain `return redis.call("SET")` should not be counted twice (once for the SET
and once for EVAL)
so that's something left to be resolved in #10279
 2022-02-21 11:20:41 +02:00
yoav-steinberg b59bb9b476 Fix script active defrag test (#10318) 
This includes two fixes:
* We forgot to count non-key reallocs in defragmentation stats.
* Fix the script defrag tests so to make dict entries less signigicant in fragmentation by making the scripts larger.
This assures active defrage will complete and reach desired results.
Some inherent fragmentation might exists in dict entries which we need to ignore.
This lead to occasional CI failures.
 2022-02-21 09:37:25 +02:00
qetu3790andBinbin b2d393b990 Fix geo search bounding box check causing missing results (#10018) 
Consider the following example:
1. geoadd k1 -0.15307903289794921875 85 n1 0.3515625 85.00019260486917005437 n2.
2. geodist k1 n1 n2 returns  "4891.9380"
3. but GEORADIUSBYMEMBER k1 n1 4891.94 m only returns n1.
n2 is in the  boundingbox but out of search areas.So we let  search areas contain boundingbox to get n2.

Co-authored-by: Binbin <binloveplay1314@qq.com>
 2022-02-21 08:06:58 +02:00
rangerzhang 9b0fd9f4d0 Fix a mistake in comments (#10312) 
There is no variable named by REPL_STATE_RECEIVE_PSYNC_REPLY, it should be REPL_STATE_RECEIVE_PSYNC_REPLY according to the contexts.
 2022-02-20 08:09:19 +02:00
Binbin c0ea77f0e1 Show publishshard_sent stat in cluster info (#10314) 
publishshard was added in #8621 (7.0 RC1), but the publishshard_sent
stat is not shown in CLUSTER INFO command.

Other changes:
1. Remove useless `needhelp` statements, it was removed in 3dad819.
2. Use `LL_WARNING` log level for some error logs (I/O error, Connection failed).
3. Fix typos that saw by the way.
 2022-02-19 21:11:20 -08:00
yoav-steinberg 56fa48ffc1 aof rewrite and rdb save counters in info (#10178) 
Add aof_rewrites and rdb_snapshots counters to info.
This is useful to figure our if a rewrite or snapshot happened since last check.
This was part of the (ongoing) effort to provide a safe backup solution for multipart-aof backups.
 2022-02-17 14:32:48 +02:00
chenyang8094 a50aa29bde Adapt redis-check-aof tool for Multi Part Aof (#10061) 
Modifications of this PR:
1. Support the verification of `Multi Part AOF`, while still maintaining support for the
  old-style `AOF/RDB-preamble`. `redis-check-aof` will automatically choose which
  mode to use according to the incoming file format.
   
`Usage: redis-check-aof [--fix|--truncate-to-timestamp $timestamp] <AOF/manifest>`
 
2. Refactor part of the code to make it easier to understand
3. Currently only supports truncate  (`--fix` or `--truncate-to-timestamp`) the last AOF
  file (may be `BASE` or `INCR`)

The reasons for 3 above:
- for `--fix`: Only the last AOF may be truncated, this is guaranteed by redis
- for `--truncate-to-timestamp`:  Normally, we only have `BASE` + `INCR` files
  at most, and `BASE` cannot be truncated(It only contains a timestamp annotation
  at the beginning of the file), so only `INCR` can be truncated. If we have a
  `BASE+INCR1+INCR2` file (meaning we have an interrupted AOFRW), Only `INCR2`
  files can be truncated at this time. If we still insist on truncate `INCR1`, we need to
  manually delete `INCR2` and update the manifest file, then re-run `redis-check-aof`
- If we want to support truncate any file, we need to add very complicated code to support
  the atomic modification of multiple file deletion and update manifest, I think this is unnecessary
 2022-02-17 08:13:28 +02:00
Ping Xie f7f68c654a Use sds for clusterNode.hostname (#10290) 
* Provide a fallback static_assert implementation
* Use sds for clusterNode.hostname
 2022-02-16 13:35:49 -08:00
Matteo Baccan cca3577503 Removed double semicolon at the end of line (#10305) 2022-02-16 16:42:04 +02:00
Yossi Gottlieb 3881f7850f Fix OpenSSL 3.0.x related issues. (#10291) 
* Drop obsolete initialization calls.
* Use decoder API for DH parameters.
* Enable auto DH parameters if not explicitly used, which should be the
  preferred configuration going forward.
 2022-02-15 08:37:06 +02:00
michael-grunder 47400870ae Reapply: Rename sds calls so they don't conflict. 
Reapply this commit on top of hiredis as a local change. Previosuly it
was pulled from a private hiredis branch, which resulted with it going
away on subtree pull.
 2022-02-14 14:00:57 +02:00
Yossi Gottlieb 886a04c2c3 Merge commit '418de21d8feb35303904ec718fcd1387e4699d2f' into refresh-hiredis 2022-02-14 13:52:57 +02:00
Yossi Gottlieb 418de21d8f Squashed 'deps/hiredis/' changes from 00272d669..f8de9a4bd 
f8de9a4bd Merge pull request #1046 from redis/rockylinux-ci
a41c9bc8b CentOS 8 is EOL, switch to RockyLinux
be41ed60d Avoid incorrect call to the previous reply's callback (#1040)
f2e8010d9 fix building on AIX and SunOS (#1031)
e73ab2f23 Add timeout support for libuv adapter (#1016)
f2ce5980e Allow sending commands after sending an unsubscribe (#1036)
ff860e55d Correction for command timeout during pubsub (#1038)
24d534493 CMakeLists.txt: allow building without a C++ compiler (#872)
4ece9a02e Fix adapters/libevent.h compilation for 64-bit Windows (#937)
799edfaad Don't link with crypto libs if USE_SSL isn't set.
f74b08182 Makefile: move SSL options into a block and refine rules
f347743b7 Update CMakeLists.txt for more portability (#1005)
f2be74802 Fix integer overflow when format command larger than 4GB (#1030)
58aacdac6 Handle array response in parallell with pubsub using RESP3 (#1014)
d3384260e Support PING while subscribing (RESP2) (#1027)
e3a479e40 FreeBSD build fixes + CI (#1026)
da5a4ff36 Add asynchronous test for pubsub using RESP3 (#1012)
b5716ee82 Valgrind returns error exit code when errors found (#1011)
1aed21a8c Move to using make directly in Cygwin (#1020)
a83f4b890 Correct CMake warning for libevent adapter example
c4333203e Remove unused parameter warning in libev adapter
7ad38dc4a Small tweaks of the async tests
4021726a6 Add asynchronous test for pubsub using RESP2
648763c36 Add build options for enabling async tests
c98c6994d Correcting the build target `coverage` for enabled SSL (#1009)
30ff8d850 Run SSL tests in CI
4a126e8a9 Add valgrind and CMake to tests
b73c2d410 Add Centos8
e9f647384 We should run actions on PRs
6ad4ccf3c Add Cygwin build test
783a3789c Add Windows tests in GitHub actions
0cac8dae1 Switch to GitHub actions
fa900ef76 Fix unused variable warning.
e489846b7 Minor refactor of CVE-2021-32765 fix.
51c740824 Remove extra comma from cmake var. Or it'll be treated as part of the var name.
632bf0718 Merge branch 'release/v1.0.2'
b73128324 Prepare for v1.0.2 GA
d4e6f109a Revert erroneous SONAME bump
a39824a5d Merge branch 'release/v1.0.1'
8d1bfac46  Prepare for v1.0.1 GA
76a7b1000 Fix for integer/buffer overflow CVE-2021-32765
9eca1f36f Allow to override OPENSSL_PREFIX in Linux
2d9d77518 Don't leak memory if an invalid type is set (#906)
f5f31ff9b Added REDIS_NO_AUTO_FREE_REPLIES flag (#962)
5850a8ecd Ensure we curry any connect error to an async context.
b6f86f38c Fix README.md
667dbf536 Merge pull request #935 from kristjanvalur/pr5
9bf6c250e Merge pull request #939 from zmartzone/improve_pr_896_ssl_leak
959af9760 Merge pull request #949 from plan-do-break-fix/Typo-corrections
0743f57bb fix(docs): corrects typos in project README
5f4382247 improve SSL leak fix redis/hiredis#896
e06ecf7e4 Ignore timeout callback from a successful connect
dfa33e60b Change order independant push logic to not change behavior.
6204182aa Handle the case where an invalidation is sent second.
d6a0b192b Merge branch 'reader-updates'
410c24d2a Fix off-by-one error in seekNewline
bd7488d27 read: Validate line items prior to checking for object creation callbacks
5f9242a1f read: Remove obsolete comment on nested multi bulk depth limitation
83c145042 read: Add support for the RESP3 bignum type
c6646cb19 read: Ensure no invalid '\r' or '\n' in simple status/error strings
e43061156 read: Additional validation and test case for RESP3 double
c8adea402 redisReply: Fix parent type assertions during double, nil, bool creation
ff73f1f9e redisReply: Explicitly list nil and bool cases in freeReplyObject() switch.
0f9251884 test: Add test case for RESP3 set
33c06dd50 test: Add test case for RESP3 map
397fe2630 read: Use memchr() in seekNewline() instead of looping over entire string
81c48a982 test: Add test cases for RESP3 bool
51e693f4f read: Add additional RESP3 bool validation
790b4d3b4 test: Add test cases for RESP3 nil
d8899fbc1 read: Add additional RESP3 nil validation
96e8ea611 test: Add test cases for infinite and NaN doubles
f913e9b99 read: Fix double validation and infinity parsing
8039c7d26 test: Add test case for doubles
49539fd1a redisReply: Fix - set len in double objects
53a8144c8 Merge pull request #924 from cheese1/master
9390de006 http -> https
7d99b5635 Merge pull request #917 from Nordix/stack-alloc-dict-iter
4bba72103 Handle OOM during async command callback registration
920128a26 Stack allocate dict iterators
297ecbecb Tiny formatting changes + suppress implicit memcpy warning
f746a28e7 Removed 2 typecasts
940a04f4d Added fuzzer
e4a200040 Merge pull request #896 from ayeganov/bugfix/ssl_leak
aefef8987 Free SSL object when redisSSLConnect fails
e3f88ebcf Merge pull request #894 from jcohen02/fix/issue893
308ffcab8 Updating SSL connection example
297f6551d Merge pull request #889 from redis/wincert
e7dda9785 Formatting
f44945a0a Merge pull request #874 from masariello/position-independent-code
74e78498c Merge pull request #888 from michael-grunder/nil-push-invalidation
b9b9f446f Fix handling of NIL invalidation messages.
acc917548 Merge pull request #885 from gkorland/patch-1
b086f763e clean a warning, remvoe empty else block
b47fae4e7 Merge pull request #881 from timgates42/bugfix_typo_terminated
f989670e5 docs: Fix simple typo, termined -> terminated
773d6ea8a Copy error to redisAsyncContext on timeout
e35300a66 add pdb files to packages for MSVC builds
dde6916b4 Add d suffix to debug libraries so that can packaged together with optimized builds (Release, RelWithDebInfo, etc)
3b68b5018 Enable position-independent code
6693863f4 Add support for system CA certificate store on Windows
2a5a57b90 Remove whitespace
1b40ec509 fixed issue with unit test linking on windows with SSL
d7b1d21e8 Merge branch 'master' of github.com:redis/hiredis
fb0e6c0dd Merge pull request #870 from michael-grunder/cmake-c99
13a35bdb6 Explicitly set c99 in CMake
bea137ca9 Merge pull request #868 from michael-grunder/fix-sockaddr-typo
bd6f86eb6 Fix sockaddr typo
48696e7e5 Don't use non-installed win32.h helper in examples (#863)
faa1c4863 Merge tag 'v1.0.0'
5003906d6 Define a no op assert if we detect NDEBUG (#861)
ea063b7cc Use development specific versions in master
04a27f480 We can run SSL tests everywhere except mingw/Windows (#859)
8966a1fc2 Remove extra whitespace (#858)
34b7f7a0f Keep libev's code style (#857)
07c3618ff Add static library target and cpack support
REVERT: 00272d669 Rename sds calls so they don't conflict in Redis.

git-subtree-dir: deps/hiredis
git-subtree-split: f8de9a4bd433791890572f7b9147e685653ddef9
 2022-02-14 13:51:42 +02:00
YaacovHazan e6478cfd10 fix "Connect multiple replicas at the same time" test (#10294) 
In order to make sure no more commands processed, we wait that
the 'load handlers' will disconncet.

The test by mistake waited on the (last) slave instead of the master.
 2022-02-14 08:46:58 +02:00
Oran Agra b099889a3a Fix and improve module error reply statistics (#10278) 
This PR handles several aspects
1. Calls to RM_ReplyWithError from thread safe contexts don't violate thread safety.
2. Errors returning from RM_Call to the module aren't counted in the statistics (they
  might be handled silently by the module)
3. When a module propagates a reply it got from RM_Call to it's client, then the error
  statistics are counted.

This is done by:
1. When appending an error reply to the output buffer, we avoid updating the global
  error statistics, instead we cache that error in a deferred list in the client struct.
2. When creating a RedisModuleCallReply object, the deferred error list is moved from
  the client into that object.
3. when a module calls RM_ReplyWithCallReply we copy the deferred replies to the dest
  client (if that's a real client, then that's when the error statistics are updated to the server)

Note about RM_ReplyWithCallReply: if the original reply had an array with errors, and the module
replied with just a portion of the original reply, and not the entire reply, the errors are currently not
propagated and the errors stats will not get propagated.

Fix #10180
 2022-02-13 18:37:32 +02:00
Oran Agra 1193e96d02 Add workflow_dispatch filters for daily CI. (#10289) 
sometimes you just wanna run one test on one system (e.g. memefficiency
on macos), so you want all other tests to be skipped
 2022-02-13 17:43:19 +02:00
Binbin 62c8be28ee Regression test for sync psync crash (#10288) 
Added regression tests for #10020 / #10081 / #10243.
The above PRs fixed some crashes due to an asserting,
see function `clientHasPendingReplies` (introduced in #9166).

This commit added some tests to cover the above scenario.
These tests will all fail in #9166, althought fixed not,
there is value in adding these tests to cover and verify
the changes. And it also can cover #8868 (verify the logs).

Other changes: 
1. Reduces the wait time in `waitForBgsave` and `waitForBgrewriteaof`
from 1s to 50ms, which should reduce the time for some tests.
2. Improve the test infra to print context when `assert_match` fails.
3. Improve the test infra to print `$error` when `assert_error` fails.
```
Expected an error matching 'ERR*' but got 'OK' (context: type eval line 4 cmd {assert_error "ERR*" {r set a b}} proc ::test)
```
 2022-02-13 09:52:38 +02:00
yoav-steinberg 2eb9b19612 Fix Eval scripts defrag (broken 7.0 in RC1) (#10271) 
Remove scripts defragger since it was broken since #10126 (released in 7.0 RC1).
would crash the server if defragger starts in a server that contains eval scripts.

In #10126 the global `lua_script` dict became a dict to a custom `luaScript` struct with an internal `robj`
in it instead of a generic `sds` -> `robj` dict. This means we need custom code to defrag it and since scripts
should never really cause much fragmentation it makes more sense to simply remove the defrag code for scripts.
 2022-02-11 21:58:05 +02:00
sundbandOran Agra 5f0119ca91 Fix duplicate module options define (#10284) 
The bug is introduced by #9323. (released in 7.0 RC1)
The define of `REDISMODULE_OPTIONS_HANDLE_IO_ERRORS` and `REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED` have the same value.

This will result in skipping `signalModifiedKey()` after `RM_CloseKey()` if the module has set
`REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD` option.
The implication is missing WATCH and client side tracking invalidations.

Other changes:
- add `no-implicit-signal-modified` to the options in INFO modules

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-11 20:15:52 +02:00
chenyang8094 a2f2b6f5b1 Modify AOF preamble related logs, and change the RDB aux field (#10283) 
In multi-part aof,  We no longer have the concept of `RDB-preamble`, so the related logs should be removed.
However, in order to print compatible logs when loading old-style AOFs, we also have to keep the relevant code.
Additionally, when saving an RDB, change the RDB aux field from "aof-preamble" to "aof-base".
 2022-02-11 18:47:03 +02:00
Harkrishn Patro a5d17f0b6c Check target node is a primary during cluster setslot. (#10277) 2022-02-10 23:14:27 -08:00
Omer ShadmiandOran Agra 5bdd72bea7 Use binary representation for key values dumped to crash log (#10275) 
Use binary representation for key values dumped crash to log,
so that if they contain null chars they're still printed correctly.
Additionally limit their length to 128 chars

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-10 13:22:57 +02:00
Pengcheng Huangandyoav-steinberg 88795f5cba Fix the misleading description of the option repl-ping-slave-period (#4687) 
Co-authored-by: yoav-steinberg <yoav@monfort.co.il>
 2022-02-10 09:18:59 +02:00
Wen Hui 64e1e7e207 Add AUTH arity test (#10266) 
Add test for AUTH with too many arguments
 2022-02-09 22:09:20 +02:00
chenyang8094 ceeff6bf86 Remove unused code - leftover from script replication mechanisms (#10272) 
append for PR #9812
 2022-02-09 15:44:09 +02:00
Binbin beb94c901e Fix INFO SENTINEL memory leak (#10268) 
* Fix INFO SENTINEL memory leak

Introduced in #6891

* remove the copy-paste sentence
 2022-02-09 07:33:24 +02:00
mowenliunian 051cc3d2e6 fix grammar issue in a comment (#10269) 
Fixed some syntax errors in the comments
 2022-02-09 07:32:40 +02:00
Oran Agra aa9beaca77 Attempt to fix a rare crash in cluster tests. (#10265) 
The theory is that a replica gets disconnected from within REPLCONF ACK,
so when we go up the stack, we'll crash when attempting to access
c->cmd->flags
 2022-02-08 19:10:13 +02:00
gms 0990dec3f5 Fix SENTINEL DEBUG with wrong arguments (#10258) 
There are two issues in SENTINEL DEBUG:
1. The error message should mention SENTINEL DEBUG
2. Add missing reuturn in args parse.

```
redis> sentinel debug INFO-PERIOD aaa
(error) ERR Invalid argument 'aaa' for SENTINEL SET 'INFO-PERIOD'

redis> sentinel debug a b c d
(error) ERR Unknown option or number of arguments for SENTINEL SET 'a'
redis> ping
(error) ERR Unknown option or number of arguments for SENTINEL SET 'b'
```

Introduced in #9291. Also do some cleanups in the code.
 2022-02-08 18:45:47 +02:00
weiguo 34c288fe11 Remove duplicate header file include (#10264) 2022-02-08 16:49:47 +02:00
Viktor Söderqvist b571c9609d Remove assert and refuse delete expired on ro replicas (#10248) 
There's an assertion added recently to make sure that non-write commands don't use lookupKeyWrite,
It was initially meant to be used only on read-only replicas, but we thought it'll not have enough coverage,
so used it on the masters too.
We now realize that in some cases this can cause issues for modules, so we remove the assert.

Other than that, we also make sure not to force expireIfNeeded on read-only replicas.
even if they somehow run a write command.

See https://github.com/redis/redis/pull/9572#discussion_r800179373
 2022-02-08 13:19:27 +02:00
Wen HuiandOran Agra 2e1bc942aa Make INFO command variadic (#6891) 
This is an enhancement for INFO command, previously INFO only support one argument
for different info section , if user want to get more categories information, either perform
INFO all / default or calling INFO for multiple times.

**Description of the feature**

The goal of adding this feature is to let the user retrieve multiple categories via the INFO
command, and still avoid emitting the same section twice.

A use case for this is like Redis Sentinel, which periodically calling INFO command to refresh
info from monitored Master/Slaves, only Server and Replication part categories are used for
parsing information. If the INFO command can return just enough categories that client side
needs, it can save a lot of time for client side parsing it as well as network bandwidth.

**Implementation**
To share code between redis, sentinel, and other users of INFO (DEBUG and modules),
we have a new `genInfoSectionDict` function that returns a dict and some boolean flags
(e.g. `all`) to the caller (built from user input).
Sentinel is later purging unwanted sections from that, and then it is forwarded to the info `genRedisInfoString`.

**Usage Examples**
INFO Server Replication   
INFO CPU Memory
INFO default commandstats

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-08 13:14:42 +02:00
yoav-steinberg b76016a948 Consistent erros returned from EVAL scripts (#10218) 
This PR handles inconsistencies in errors returned from lua scripts.
Details of the problem can be found in #10165.

### Changes

- Remove double stack trace. It's enough that a stack trace is automatically added by the engine's error handler
  see https://github.com/redis/redis/blob/d0bc4fff18afdf9e5421cc88e23ffbb876ecaec3/src/function_lua.c#L472-L485
  and https://github.com/redis/redis/blob/d0bc4fff18afdf9e5421cc88e23ffbb876ecaec3/src/eval.c#L243-L255
- Make sure all errors a preceded with an error code. Passing a simple string to `luaPushError()` will prepend it
  with a generic `ERR` error code.
- Make sure lua error table doesn't include a RESP `-` error status. Lua stores redis error's as a lua table with a
  single `err` field and a string. When the string is translated back to RESP we add a `-` to it.
  See https://github.com/redis/redis/blob/d0bc4fff18afdf9e5421cc88e23ffbb876ecaec3/src/script_lua.c#L510-L517
  So there's no need to store it in the lua table.

### Before & After
```diff
--- <unnamed>
+++ <unnamed>
@@ -1,14 +1,14 @@
  1: config set maxmemory 1
  2: +OK
  3: eval "return redis.call('set','x','y')" 0
- 4: -ERR Error running script (call to 71e6319f97b0fe8bdfa1c5df3ce4489946dda479): @user_script:1: @user_script: 1: -OOM command not allowed when used memory > 'maxmemory'.
+ 4: -ERR Error running script (call to 71e6319f97b0fe8bdfa1c5df3ce4489946dda479): @user_script:1: OOM command not allowed when used memory > 'maxmemory'.
  5: eval "return redis.pcall('set','x','y')" 0
- 6: -@user_script: 1: -OOM command not allowed when used memory > 'maxmemory'.
+ 6: -OOM command not allowed when used memory > 'maxmemory'.
  7: eval "return redis.call('select',99)" 0
  8: -ERR Error running script (call to 4ad5abfc50bbccb484223905f9a16f09cd043ba8): @user_script:1: ERR DB index is out of range
  9: eval "return redis.pcall('select',99)" 0
 10: -ERR DB index is out of range
 11: eval_ro "return redis.call('set','x','y')" 0
-12: -ERR Error running script (call to 71e6319f97b0fe8bdfa1c5df3ce4489946dda479): @user_script:1: @user_script: 1: Write commands are not allowed from read-only scripts.
+12: -ERR Error running script (call to 71e6319f97b0fe8bdfa1c5df3ce4489946dda479): @user_script:1: ERR Write commands are not allowed from read-only scripts.
 13: eval_ro "return redis.pcall('set','x','y')" 0
-14: -@user_script: 1: Write commands are not allowed from read-only scripts.
+14: -ERR Write commands are not allowed from read-only scripts.
```
 2022-02-08 11:44:40 +02:00
guybe7 3c3e6cc1c7 X[AUTO]CLAIM should skip deleted entries (#10227) 
Fix #7021 #8924 #10198

# Intro
Before this commit X[AUTO]CLAIM used to transfer deleted entries from one
PEL to another, but reply with "nil" for every such entry (instead of the entry id).
The idea (for XCLAIM) was that the caller could see this "nil", realize the entry
no longer exists, and XACK it in order to remove it from PEL.
The main problem with that approach is that it assumes there's a correlation
between the index of the "id" arguments and the array indices, which there
isn't (in case some of the input IDs to XCLAIM never existed/read):

```
127.0.0.1:6379> XADD x 1 f1 v1
"1-0"
127.0.0.1:6379> XADD x 2 f1 v1
"2-0"
127.0.0.1:6379> XADD x 3 f1 v1
"3-0"
127.0.0.1:6379> XGROUP CREATE x grp 0
OK
127.0.0.1:6379> XREADGROUP GROUP grp Alice COUNT 2 STREAMS x >
1) 1) "x"
   2) 1) 1) "1-0"
         2) 1) "f1"
            2) "v1"
      2) 1) "2-0"
         2) 1) "f1"
            2) "v1"
127.0.0.1:6379> XDEL x 1 2
(integer) 2
127.0.0.1:6379> XCLAIM x grp Bob 0 0-99 1-0 1-99 2-0
1) (nil)
2) (nil)
```

# Changes
Now,  X[AUTO]CLAIM acts in the following way:
1. If one tries to claim a deleted entry, we delete it from the PEL we found it in
  (and the group PEL too). So de facto, such entry is not claimed, just cleared
  from PEL (since anyway it doesn't exist in the stream)
2. since we never claim deleted entries, X[AUTO]CLAIM will never return "nil"
  instead of an entry.
3. add a new element to XAUTOCLAIM's response (see below)

# Knowing which entries were cleared from the PEL
The caller may want to log any entries that were found in a PEL but deleted from
the stream itself (it would suggest that there might be a bug in the application:
trimming the stream while some entries were still no processed by the consumers)

## XCLAIM
the set {XCLAIM input ids} - {XCLAIM returned ids} contains all the entry ids that were
not claimed which means they were deleted (assuming the input contains only entries
from some PEL). The user doesn't need to XACK them because XCLAIM had already
deleted them from the source PEL.

## XAUTOCLAIM
XAUTOCLAIM has a new element added to its reply: it's an array of all the deleted
stream IDs it stumbled upon.

This is somewhat of a breaking change since X[AUTO]CLAIM used to be able to reply
with "nil" and now it can't... But since it was undocumented (and generally a bad idea
to rely on it, as explained above) the breakage is not that bad.
 2022-02-08 10:20:09 +02:00
Oran Agra 66be30f7fc Handle key-spec flags with modules (#10237) 
- add COMMAND GETKEYSANDFLAGS sub-command
- add RM_KeyAtPosWithFlags and GetCommandKeysWithFlags
- RM_KeyAtPos and RM_CreateCommand set flags requiring full access for keys
- RM_CreateCommand set VARIABLE_FLAGS
- expose `variable_flags` flag in COMMAND INFO key-specs
- getKeysFromCommandWithSpecs prefers key-specs over getkeys-api
- add tests for all of these
 2022-02-08 10:01:35 +02:00
ivanstosic-janea bb875603fb Fix protocol error caused by redis-benchmark (#10236) 
The protocol error was caused by the buggy `writeHandler` in `redis-benchmark.c`,
which didn't handle one of the cases, thereby repeating data, leading to protocol errors
when the values being sent are very long.

This PR fixes #10233, issue introduced by #7959
 2022-02-07 22:57:11 +02:00
Avital-Fine 9a0ab2fbbf Update function-delete.json - argument is library name, not function name (#10253) 2022-02-07 20:01:34 +02:00
Binbin 7f4cca11dc COMMAND DOCS avoid adding summary/since if they don't exist (#10252) 
If summary or since is empty, we used to return NULL in
COMMAND DOCS. Currently all redis commands will have these
two fields.

But not for module command, summary and since are optional
for RM_SetCommandInfo. With the change in #10043, if a module
command doesn't have the summary or since, redis-cli will
crash (see #10250).

In this commit, COMMAND DOCS avoid adding summary or since
when they are missing.
 2022-02-07 19:57:50 +02:00
Binbin b95beeb595 Fix redis-cli with sentinel crash due to SENTINEL DEBUG missing summary (#10250) 
Fix redis-cli with sentinel crash due to SENTINEL DEBUG missing summary

Because SENTINEL DEBUG missing summary in its json file,
with the change in #10043, the following assertion will fail.
```
[redis]# src/redis-cli -p 26379
redis-cli: redis-cli.c:678: cliInitCommandHelpEntry: Assertion `reply->type == 1' failed.
```

This commit add the summary and complexity for SENTINEL DEBUG,
which introduced in #9291, and also improved the help message.
 2022-02-07 13:27:33 +02:00
yoav-steinberg 9dfeda58ed acl check api for functions and eval (#10220) 
Changes:
1. Adds the `redis.acl_check_cmd()` api to lua scripts. It can be used to check if the
  current user has permissions to execute a given command. The new function receives
  the command to check as an argument exactly like `redis.call()` receives the command
  to execute as an argument.
2. In the PR I unified the code used to convert lua arguments to redis argv arguments from
  both the new `redis.acl_check_cmd()` API and the `redis.[p]call()` API. This cleans up
  potential duplicate code.
3. While doing the refactoring in 2 I noticed there's an optimization to reduce allocation calls
  when parsing lua arguments into an `argv` array in the `redis.[p]call()` implementation.
  These optimizations were introduced years ago in 48c49c4851
  and 4f686555ce. It is unclear why this was added.
  The original commit message claims a 4% performance increase which I couldn't recreate
  and might not be worth it even if it did recreate. This PR removes that optimization.
  Following are details of the benchmark I did that couldn't reveal any performance
  improvements due to this optimization:

```
benchmark 1: src/redis-benchmark -P 500 -n 10000000 eval 'return redis.call("ping")' 0
benchmark 2: src/redis-benchmark -P 500 -r 1000 -n 1000000 eval 'return redis.call("mset","k1__rand_int__","v1__rand_int__","k2__rand_int__","v2__rand_int__","k3__rand_int__","v3__rand_int__","k4__rand_int__","v4__rand_int__")' 0
benchmark 3: src/redis-benchmark -P 500 -r 1000 -n 100000 eval "for i=1,100,1 do redis.call('set','kk'..i,'vv'..__rand_int__) end return redis.call('get','kk5')" 0
benchmark 4: src/redis-benchmark -P 500 -r 1000 -n 1000000 eval 'return redis.call("mset","k1__rand_int__","v1__rand_int__","k2__rand_int__","v2__rand_int__","k3__rand_int__","v3__rand_int__","k4__rand_int__","v4__rand_int__xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")'
```
I ran the benchmark on this branch with and without commit 68b71680a4d3bb8f0509e06578a9f15d05b92a47
Results in requests per second:
cmd | without optimization | without optimization 2nd run | with original optimization | with original optimization 2nd run
-- | -- | -- | -- | --
1 | 461233.34 | 477395.31 | 471098.16 | 469946.91
2 | 34774.14 | 35469.8 | 35149.38 | 34464.93
3 | 6390.59 | 6281.41 | 6146.28 | 6464.12
4 | 28005.71 |   | 27965.77 |  

As you can see, different use cases showed identical or negligible performance differences.
So finally I decided to chuck the original optimization and simplify the code.
 2022-02-07 08:04:01 +02:00
Binbin c5e3d135ae create-cluster clean now will clean appendonlydir (#10223) 
In #9788, now we stores all persistent append-only files in
a dedicated directory. The name of the directory is determined
by the appenddirname configuration parameter in redis.conf.
Now each node have a separate folder.
Update create-cluster clean to clean this default directory.
 2022-02-07 07:58:15 +02:00
weiguo d6e9cde595 Supplement define guards to prevent multiple inclusion (#10246) 2022-02-06 20:13:34 -08:00
Oran Agra 98b3f52599 add test suite infra to test RESP3 attributes (#10247) 
So far we only tested attributes using readraw, not the
resp parser caches them, so that after getting the reply,
you can query them if you want.
 2022-02-07 00:10:05 +02:00
footpatch 91cc2059f5 Fix file descriptor leak in memtest_test_linux_anonymous_maps (#4241) 
when we fail opening `/proc`, we need to close the log file fd.
 2022-02-06 13:17:38 +02:00
Binbin 344e41c922 Fix PSYNC crash with wrong offset (#10243) 
`PSYNC replicationid str_offset` will crash the server.

The reason is in `masterTryPartialResynchronization`,
we will call `getLongLongFromObjectOrReply` check the
offset. With a wrong offset, it will add a reply and
then trigger a full SYNC and the client become a replica.

So crash in `c->bufpos == 0 && listLength(c->reply) == 0`.
In this commit, we check the psync_offset before entering
function `masterTryPartialResynchronization`, and return.

Regardless of that crash, accepting the sync, but also replying
with an error would have corrupt the replication stream.
 2022-02-06 13:13:56 +02:00
Wen Hui 6ebb679f06 Add tests for ACL command error cases (#10183) 2022-02-06 07:58:28 +02:00
Oran Agra c42b163332 fix file permissions for scripts in utils folder (#10241) 
make sure the scripts are executable
 2022-02-05 18:40:09 +02:00
Jason ElbaumandOran Agra 5b17909c4f redis-cli generates command help tables from the results of COMMAND (#10043) 
This is a followup to #9656 and implements the following step mentioned in that PR:

* When possible, extract all the help and completion tips from COMMAND DOCS (Redis 7.0 and up)
* If COMMAND DOCS fails, use the static help.h compiled into redis-cli.
* Supplement additional command names from COMMAND (pre-Redis 7.0)

The last step is needed to add module command and other non-standard commands.

This PR does not change the interactive hinting mechanism, which still uses only the param
strings to provide somewhat unreliable and inconsistent command hints (see #8084).
That task is left for a future PR. 

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-02-05 16:54:16 +02:00
Binbin e662f6835e Update FCALL/FCALL_RO summary and complexity (#10240) 
1. Update fcall.json and fcall_ro.json
2. Update command.c
3. Update help.h
 2022-02-05 10:35:07 +02:00
Viktor Söderqvist 0a82fe8447 Command info module API (#10108) 
Adds RM_SetCommandInfo, allowing modules to provide the following command info:

* summary
* complexity
* since
* history
* hints
* arity
* key specs
* args

This information affects the output of `COMMAND`, `COMMAND INFO` and `COMMAND DOCS`,
Cluster, ACL and is used to filter commands with the wrong number of arguments before
the call reaches the module code.

The recently added API functions for key specs (never released) are removed.

A minimalist example would look like so:
```c
    RedisModuleCommand *mycmd = RedisModule_GetCommand(ctx,"mymodule.mycommand");
    RedisModuleCommandInfo mycmd_info = {
        .version = REDISMODULE_COMMAND_INFO_VERSION,
        .arity = -5,
        .summary = "some description",
    };
    if (RedisModule_SetCommandInfo(mycmd, &mycmd_info) == REDISMODULE_ERR)
        return REDISMODULE_ERR;
````

Notes:
* All the provided information (including strings) is copied, not keeping references to the API input data.
* The version field is actually a static struct that contains the sizes of the the structs used in arrays,
  so we can extend these in the future and old version will still be able to take the part they can support.
 2022-02-04 21:09:36 +02:00
Binbin d7fcb3c5a1 Fix SENTINEL SET config rewrite test (#10232) 
Change the sentinel config file to a directory in SENTINEL SET test.
So it will now fail on the `rename` in `rewriteConfigOverwriteFile`.

The test used to set the sentinel config file permissions to `000` to
simulate failure. But it fails on centos7 / freebsd / alpine. (introduced in #10151)

Other changes:
1. More error messages after the config rewrite failure.
2. Modify arg name `force_all` in `rewriteConfig` to `force_write`. (was rename in #9304)
3. Fix a typo in debug quicklist-packed-threshold, then -> than. (#9357)
 2022-02-04 11:39:51 +02:00
Binbin d2fde2f655 Fix cluster tests failing due to subcommand names (#10231) 
Introduced in #10128
 2022-02-04 11:32:30 +02:00
Wen HuiandYossi Gottlieb 65ef543f8c Sentinel: return an error if configuration save fails (#10151) 
When performing `SENTINEL SET`, Sentinel updates the local configuration file. Before this commit, failure to update the file would still result with an `+OK` reply. Now, a `-ERR Failed to save config file` error will be returned.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
 2022-02-03 13:20:35 +02:00
Vo Trong PhucPhuc. Vo TrongOran AgraMeir Shpilraien
 		53c43fcc84 Add check min-slave-* feature when evaluating Lua scripts and Functions (#10160) 
Add check enough good slaves for write command when evaluating scripts.
This check is made before the script is executed, if we have function flags, and per redis command if we don't.

Co-authored-by: Phuc. Vo Trong <phucvt@vng.com.vn>
Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Meir Shpilraien (Spielrein) <meir@redis.com>
 2022-02-03 11:57:51 +02:00
Viktor Söderqvist f4ecc799c8 Add 'Available since' to module API function docs (#10229) 
The script which generates the markdown docs from module.c is updated to include
the version in which each module API function was introduced.

The script uses git tags to find this information. If git is not available or if we're not in
a git repo, the 'since' is silently skipped.

The line `**Available since:** (version)` is added after the function prototype

Rename to utils/generate-module-api-doc.rb
 2022-02-03 10:25:37 +02:00
Wen Hui c9e1602f90 Add test case to improve code coverage for Addslotsrange and Delslotsrange command (#10128) 
add more test cases for addslotsrange and delslotsrange
 2022-02-02 18:22:46 -08:00
Meir Shpilraien (Spielrein) 8eb19cc251 update function help message, changed DESC->DESCRIPTION. (#10228) 
update function help message, changed DESC->DESCRIPTION (doc was outdated)
 2022-02-02 15:50:41 +02:00
dependabot[bot] edc050cc57 Bump vmactions/freebsd-vm from 0.1.5 to 0.1.6 (#10219) 2022-02-02 10:39:34 +02:00
sundb 948285de39 Fix src/modules build issue on OS X 11 (#10224) 2022-02-01 21:39:10 +02:00
郭伟光andsundb 6b5b3ca414 forbid module to unload when it holds ongoing timer (#10187) 
This is done to avoid a crash when the timer fires after the module was unloaded.
Or memory leaks in case we wanted to just ignore the timer.
It'll cause the MODULE UNLOAD command to return with an error

Co-authored-by: sundb <sundbcn@gmail.com>
 2022-02-01 14:54:11 +02:00
Oran Agra d5915a167f 7.0-RC1
CI / build-32bit (push) Has been cancelled
Details
External Server Tests / test-external-standalone (push) Has been cancelled
Details
CI / build-libc-malloc (push) Has been cancelled
Details
CI / test-ubuntu-latest (push) Has been cancelled
Details
CI / build-centos7-jemalloc (push) Has been cancelled
Details
External Server Tests / test-external-cluster (push) Has been cancelled
Details
Spellcheck / Spellcheck (push) Has been cancelled
Details
CI / test-sanitizer-address (push) Has been cancelled
Details
CodeQL / Analyze (cpp) (push) Has been cancelled
Details
CI / build-debian-old (push) Has been cancelled
Details
External Server Tests / test-external-nodebug (push) Has been cancelled
Details
CI / build-macos-latest (push) Has been cancelled
Details
2022-01-31 12:50:24 +02:00
Meir Shpilraien (Spielrein) 6ca97da0fc Fix wrong version calculation on Redis Function tests. (#10217) 2022-01-31 12:49:57 +02:00
Madelyn Olson d0bc4fff18 Added history for cluster-slots changes for hostnames(#10216) 
* The first field was previously only an IP address, it can now be a hostname or NULL.
* There is now a fourth field, which has more information.
 2022-01-30 23:57:10 -08:00
Madelyn Olson 8b1cda7568 Change replica migration tests to use continous slots to improve speed (#10215) 2022-01-30 22:44:32 -08:00
Oran Agra d364ede59c Revent the attempt to fix cluster rebalance test (#10207) (#10212) 
It seems that fix didn't really solve the problem with ASAN,
and also introduced issues with other CI runs.

unrelated:
- make runtest-cluster able to take multiple --single arguments
 2022-01-31 01:47:58 +02:00
Oran Agra ef93125988 update help.h before release (#10210) 2022-01-30 22:53:13 +02:00
Ping Xie 8013af6f3d Fix cluster bus extensions backwards compatibility (#10206) 
Before this commit, notused1 was incorrectly resized resulting with a clusterMsg that is not backwards compatible as expected.
 2022-01-30 19:43:37 +02:00
Moti Cohen 52b2fbe970 Improve srand entropy (and fix Sentinel failures) (#10197) 
As Sentinel relies upon consensus algorithm, all sentinel instances,
randomize a time to initiate their next attempt to become the
leader of the group. But time after time, all raffled the same value.

The problem is in the line `srand(time(NULL)^getpid())` such that
all spinned up containers get same time (in seconds) and same pid
which is always 1. Added material `tv_usec` and verify that even
consecutive calls brings different values and makes the difference.
 2022-01-30 16:39:23 +02:00
Tobias Nießen 4e17154060 Fix typo in function_load local variable (#10209) 
Refs: https://github.com/redis/redis/pull/10141
 2022-01-30 14:38:58 +02:00
Harkrishn Patro a43b6922d1 Set default channel permission to resetchannels for 7.0 (#10181) 
For backwards compatibility in 6.x, channels default permission was set to `allchannels` however with 7.0,
we should modify it and the default value should be `resetchannels` for better security posture.
Also, with selectors in ACL, a client doesn't have to set channel rules everytime and by default
the value will be `resetchannels`.

Before this change
```
127.0.0.1:6379> acl list
1) "user default on nopass ~* &* +@all"
127.0.0.1:6379>  acl setuser hp on nopass +@all ~*
OK
127.0.0.1:6379> acl list
1) "user default on nopass ~* &* +@all"
2) "user hp on nopass ~* &* +@all"
127.0.0.1:6379>  acl setuser hp1 on nopass -@all (%R~sales*)
OK
127.0.0.1:6379> acl list
1) "user default on nopass ~* &* +@all"
2) "user hp on nopass ~* &* +@all"
3) "user hp1 on nopass &* -@all (%R~sales* &* -@all)"
```

After this change
```
127.0.0.1:6379> acl list
1) "user default on nopass ~* &* +@all"
127.0.0.1:6379> acl setuser hp on nopass +@all ~*
OK
127.0.0.1:6379> acl list
1) "user default on nopass ~* &* +@all"
2) "user hp on nopass ~* resetchannels +@all"
127.0.0.1:6379> acl setuser hp1 on nopass -@all (%R~sales*)
OK
127.0.0.1:6379> acl list
1) "user default on nopass ~* &* +@all"
2) "user hp on nopass ~* resetchannels +@all"
3) "user hp1 on nopass resetchannels -@all (%R~sales* resetchannels -@all)"
```
 2022-01-30 12:02:55 +02:00
guybe7 eedec155ac Add key-specs notes (#10193) 
Add optional `notes` to keyspecs.

Other changes:

1. Remove the "incomplete" flag from SORT and SORT_RO: it is misleading since "incomplete" means "this spec may not return all the keys it describes" but SORT and SORT_RO's specs (except the input key) do not return any keys at all.
So basically:
If a spec's begin_search is "unknown" you should not use it at all, you must use COMMAND KEYS;
if a spec itself is "incomplete", you can use it to get a partial list of keys, but if you want all of them you must use COMMAND GETKEYS;
otherwise, the spec will return all the keys

2. `getKeysUsingKeySpecs` handles incomplete specs internally
 2022-01-30 12:00:03 +02:00
Oran Agra be0d293354 fix cluster rebalance test race (#10207) 
Try to fix the rebalance cluster test that's failing with ASAN daily:

Looks like `redis-cli --cluster rebalance` gets `ERR Please use SETSLOT only with masters` in `clusterManagerMoveSlot()`.
it happens when `12-replica-migration-2.tcl` is run with ASAN in GH Actions.
in `Resharding all the master #0 slots away from it`

So the fix (assuming i got it right) is to call `redis-cli --cluster check` before `--cluster rebalance`.
p.s. it looks like a few other checks in these tests needed that wait, added them too.

Other changes:
* in instances.tcl, make sure to catch tcl test crashes and let the rest of the code proceed, so that if there was
  a redis crash, we'll find it and print it too.
* redis-cli, try to make sure it prints an error instead of silently exiting.

specifically about redis-cli:
1. clusterManagerMoveSlot used to print an error, only if the caller also asked for it (should be the other way around).
2. clusterManagerCommandReshard asked for an error, but didn't use it (probably tried to avoid the double print).
3. clusterManagerCommandRebalance didn't ask for the error, now it does.
4. making sure that other places in clusterManagerCommandRebalance print something before exiting with an error.
 2022-01-30 11:30:19 +02:00
Binbin 21135471a6 Fix some wrong commands arguments since (#10208) 
ZADD NX and XX was introduced in 3.0.2, not 6.2.0
ZADD GT and LT was introduced in 6.2.0, not 3.0.2

Add missing `COUNT ANY` history in georadius_ro
Add missing `SHUTDOWN [NOW] [FORCE] [ABORT]` since in shutdown.json
 2022-01-30 08:45:12 +02:00
Moti Cohen 79f089bdd9 Fixed Sentinel support for hostnames (#10146) 
Sentinel tries to resolve instances hostname to IP only during registration. 
It might be that the instance is unavailable during that time, such as 
leader crashed and failover took place. Yet, promoted replica must support:

 - Register leader, even if it fails to resolve its hostname during failover
 - Try later to resolve it, if instance is disconnected. Note that
   this condition also support ip-change of an instance.
 2022-01-29 21:00:29 +02:00
Ozan Tezcan 2d391e62bd Skip external server and codeql cron tests on forks (#10204) 
keep the push triggers for all repos, but run the scheduled ones only on redis/redis
 2022-01-29 19:02:35 +02:00
Oran Agra 75a950cb93 doc improvement about acl first-arg (#10199) 
We recently removed capabilities from the first-arg feature of ACL and added a warning.
but we didn't document it.
ref: #10147 and redis/redis-doc#1761
 2022-01-29 18:34:59 +02:00
Binbin 93d951556b commands arguments improvement about unix-time type (#10203) 
Change the name to unix-time-seconds or unix-time-milliseconds
to be consistent. Change the type from INTEGER to UNIX_TIME.

SET (EXAT and PXAT) was already ok.
and naming aside, both PXAT and EXAT everywhere used unit-time (for both milliseconds and seconds).
the only ones that where wrong are GETEX and XCLAIM (using "integer" for both seconds and milliseconds)
 2022-01-29 15:21:59 +02:00
weiguo 76784e4ece Use object size then pointer size when malloc (#10201) (#10202) 
By a happy coincidence, sizeof(sds *) is equal to sizeof(sds) here,
while it's logically consistent to use sizeof(sds) instead.
 2022-01-29 14:43:16 +02:00
Madelyn Olson 7a89e49859 Added missing documentation about the ACL GETUSER changes in acl-v2 (#10200) 2022-01-28 12:13:44 -08:00
Yossi Gottlieb 068190a802 Update generate-command-help.rb and help.h. (#10196) 
There are some inevitable changes between the old and new output, as a
result of the different `commands.json` semantics.
 2022-01-27 21:48:06 +02:00
BinbinMadelyn OlsonOran Agra
 		d616925835 Allow SET without GET arg on write-only ACL. Allow BITFIELD GET on read-only ACL (#10148) 
SET is a R+W command, because it can also do `GET` on the data.
SET without GET is a write-only command.
SET with GET is a read+write command.

In #9974, we added ACL to let users define write-only access.
So when the user uses SET with GET option, and the user doesn't
have the READ permission on the key, we need to reject it,
but we rather not reject users with write-only permissions from using
the SET command when they don't use GET.

In this commit, we add a `getkeys_proc` function to control key
flags in SET command. We also add a new key spec flag (VARIABLE_FLAGS)
means that some keys might have different flags depending on arguments.

We also handle BITFIELD command, add a `bitfieldGetKeys` function.
BITFIELD GET is a READ ONLY command.
BITFIELD SET or BITFIELD INCR are READ WRITE commands.

Other changes:
1. SET GET was added in 6.2, add the missing since in set.json
2. Added tests to cover the changes in acl-v2.tcl
3. Fix some typos in server.h and cleanups in acl-v2.tcl

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-26 21:03:21 +02:00
Oran Agra 795ea011ba Solve race in a BGSAVE test (#10190) 
This PR attempts to solve two problems that happen sometime in valgrind:
`ERR Background save already in progress`
and
`not bgsave not aborted`

the test used to populate the database with DEBUG, which didn't
increment the dirty counter, so couldn't trigger an automatic bgsave.
then it used a manual bgsave, and aborted it (when it got aborted it
populated the dirty counter), and then it tried to do another bgsave.
that other bgsave could have failed if the automatic one already
started.
 2022-01-26 19:46:02 +02:00
Oran Agra da48a81290 solve race in expiration test (#10192) 
Failed on a non-valgrind run. on this line:
```
assert_equal 0 [$slave exists k]
```
the condition in `keyIsExpired` is `now > when`.
so if the test is really fast, maybe it can get to EXISTS exactly 1000 milliseconds after the
expiration was set, and the key isn't yet gone)
 2022-01-26 19:45:31 +02:00
guybe7 c79389f032 Reply for command args should be an array, not a set (#10188) 2022-01-26 12:49:24 +02:00
Binbin 7fdcada67b Fix unused variable warning in subcommand.c (#10184) 
Forgot to handle it in #10135.
 2022-01-26 10:21:51 +02:00
Madelyn Olson f6b76e50ad Change expression to look for at least one limit exceeded (#10173) 
This is an attempt to fix some of the issues with the cluster mode tests we are seeing in the daily run.

The test is trying to incrementally adds a bunch of publish messages, expecting that eventually one
of them will overflow. The tests stops one of the processes, so it expects that just that one Redis node
will overflow. I think the test is flaky because under certain circumstances multiple links are getting
disconnected, not just the one that is stalled.
 2022-01-26 09:59:53 +02:00
Viktor Söderqvist 4491ee1805 redis-cli: Aligned RESP3 maps with multiline value in TTY (#10170) 
Before:

```
127.0.0.1:6379> command info get
1)  1) "get"
    2) (integer) 2
    3) 1~ readonly
       2~ fast
    4) (integer) 1
    5) (integer) 1
    6) (integer) 1
    7) 1~ @read
       2~ @string
       3~ @fast
    8) (empty set)
    9) 1~ 1# "flags" => 1~ RO
             2~ access
          2# "begin_search" => 1# "type" => "index"
             2# "spec" => 1# "index" => (integer) 1
          3# "find_keys" => 1# "type" => "range"
             2# "spec" => 1# "lastkey" => (integer) 0
                2# "keystep" => (integer) 1
                3# "limit" => (integer) 0
   10) (empty set)
```

After:

```
127.0.0.1:6379> command info get
1)  1) "get"
    2) (integer) 2
    3) 1~ readonly
       2~ fast
    4) (integer) 1
    5) (integer) 1
    6) (integer) 1
    7) 1~ @read
       2~ @string
       3~ @fast
    8) (empty set)
    9) 1~ 1# "flags" =>
             1~ RO
             2~ access
          2# "begin_search" =>
             1# "type" => "index"
             2# "spec" => 1# "index" => (integer) 1
          3# "find_keys" =>
             1# "type" => "range"
             2# "spec" =>
                1# "lastkey" => (integer) 0
                2# "keystep" => (integer) 1
                3# "limit" => (integer) 0
   10) (empty set)
```
 2022-01-25 17:15:57 +02:00
Meir Shpilraien (Spielrein) 5a38ccc253 Added engine stats to FUNCTION STATS command. (#10179) 
Added the following statistics (per engine) to FUNCTION STATS command:
* number of functions
* number of libraries

Output example:
```
> FUNCTION stats
1) "running_script"
2) (nil)
3) "engines"
4) 1) "LUA"
   2) 1) "libraries_count"
      2) (integer) 1
      3) "functions_count"
      4) (integer) 1
```

To collect the stats, added a new dictionary to libraries_ctx that contains
for each engine, the engine statistics representing the current libraries_ctx.
Update the stats on:
1. Link library to libraries_ctx
2. Unlink library from libraries_ctx
3. Flushing libraries_ctx
 2022-01-25 15:50:14 +02:00
yoav-steinberg d26453a3f8 Fix commands.c build issue on merge (#10172) 
Set commands.c's merge driver to binary so when it conflicts during a merge git will leave the local version unmodified.
This way our Makefile will rebuild it based on src/commands/*.json before trying to compile it.
Otherwise the file gets modified with merge conflict markers and gets the same timestamp as the *.json files,
so the Makefile doesn't attempt to rebuild it before compiling and we get a compilation error.
 2022-01-25 12:24:06 +02:00
chenyang8094 ac011ebd7f Delete the residual code related to aof rewrite buf (#10176) 2022-01-25 11:37:18 +02:00
Madelyn Olson 823da54361 Improve testing and update flags around commands without ACL keyspec flags (#10167) 
This PR aims to improve the flags associated with some commands and adds various tests around
these cases. Specifically, it's concerned with commands which declare keys but have no ACL
flags (think `EXISTS`), the user needs either read or write permission to access this type of key.

This change is primarily concerned around commands in three categories:

# General keyspace commands
These commands are agnostic to the underlying data outside of side channel attacks, so they are not
marked as ACCESS.
* TOUCH
* EXISTS
* TYPE
* OBJECT 'all subcommands'

Note that TOUCH is not a write command, it could be a side effect of either a read or a write command.

# Length and cardinality commands
These commands are marked as NOT marked as ACCESS since they don't return actual user strings,
just metadata.
* LLEN
* STRLEN
* SCARD
* HSTRLEN

# Container has member commands
These commands return information about the existence or metadata about the key. These commands
are NOT marked as ACCESS since the check of membership is used widely in write commands
e.g. the response of HSET. 
* SISMEMBER
* HEXISTS

# Intersection cardinality commands
These commands are marked as ACCESS since they process data to compute the result.
* PFCOUNT
* ZCOUNT
* ZINTERCARD
* SINTERCARD
 2022-01-25 09:55:30 +02:00
Madelyn Olson c275010fff Correctly handle minimum arity checks in scripts (#10171) 
Correctly handle variable arity checks in scripts
 2022-01-24 22:08:57 -08:00
Oran Agra 0343fe9fa5 re-fix EVAL timeout test (#10169) 
The edit i made to #10098 before merging was broken.
(INFO is forbidden in LOADING)
 2022-01-24 23:01:08 +02:00
chenyang8094andOran Agra fa60049648 Fix EVAL timeout test failed on freebsd (#10098) 
* Refactor EVAL timeout test
* since the test used r config set appendonly yes which generates a rewrite, it missed it's purpose
* Fix the bug that start_server returns before redis starts ready, which affects when multiple tests share the same dir.
* Elapsed time tracking no loner needed

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-24 22:31:35 +02:00
Binbin 495ac8b79a Fix outdated save key word in redis.conf (#10166) 
For some complex data types, server.dirty actually counts
the number of elements that have been changed.
And in FLUSHDB or FLUSHALL, we count the number of keys.

So the word "key" is not strictly correct and is outdated.
Some discussion can be seen at #8140.
 2022-01-24 22:02:42 +02:00
yoav-steinberg 7eadc5ee70 Support function flags in script EVAL via shebang header (#10126) 
In #10025 we added a mechanism for flagging certain properties for Redis Functions.
This lead us to think we'd like to "port" this mechanism to Redis Scripts (`EVAL`) as well. 

One good reason for this, other than the added functionality is because it addresses the
poor behavior we currently have in `EVAL` in case the script performs a (non DENY_OOM) write operation
during OOM state. See #8478 (And a previous attempt to handle it via #10093) for details.
Note that in Redis Functions **all** write operations (including DEL) will return an error during OOM state
unless the function is flagged as `allow-oom` in which case no OOM checking is performed at all.

This PR:
- Enables setting `EVAL` (and `SCRIPT LOAD`) script flags as defined in #10025.
- Provides a syntactical framework via [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) for
  additional script annotations and even engine selection (instead of just lua) for scripts.
- Provides backwards compatibility so scripts without the new annotations will behave as they did before.
- Appropriate tests.
- Changes `EVAL[SHA]/_RO` to be flagged as `STALE` commands. This makes it possible to flag individual
  scripts as `allow-stale` or not flag them as such. In backwards compatibility mode these commands will
  return the `MASTERDOWN` error as before.
- Changes `SCRIPT LOAD` to be flagged as a `STALE` command. This is mainly to make it logically
  compatible with the change to `EVAL` in the previous point. It enables loading a script on a stale server
  which is technically okay it doesn't relate directly to the server's dataset. Running the script does, but that
  won't work unless the script is explicitly marked as `allow-stale`.

Note that even though the LUA syntax doesn't support hash tag comments `.lua` files do support a shebang
tag on the top so they can be executed on Unix systems like any shell script. LUA's `luaL_loadfile` handles
this as part of the LUA library. In the case of `luaL_loadbuffer`, which is what Redis uses, I needed to fix the
input script in case of a shebang manually. I did this the same way `luaL_loadfile` does, by replacing the
first line with a single line feed character.
 2022-01-24 16:50:02 +02:00
Viktor SöderqvistandOran Agra 857dc5bacd Disable keyspec module API in 7.0 RC1 (#10135) 
The keyspec API is not yet released and there is a plan to change it
in #10108, which is going to be included in RC2. Therefore, we hide
it in RC1 to avoid introducing a breaking change in RC2.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-24 15:02:07 +02:00
chenyang8094 6dc3f09cb9 Fix AOFRW limit test occasional failures on slower machines (#10164) 2022-01-24 14:55:24 +02:00
ny0312andMadelyn Olson b40a9ba5fd Fix flaky cluster tests in 24-links.tcl (#10157) 
* Fix flaky cluster test "Disconnect link when send buffer limit reached"
* Fix flaky cluster test "Each node has two links with each peer"

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
 2022-01-23 17:28:32 -08:00
Binbin 7e5ded2ad0 Fix timing issue in sentinel CKQUORUM test (#10036) 
A test failure was reported in Daily CI (test-centos7-tls).
`CKQUORUM detects failover authorization cannot be reached`.

```
CKQUORUM detects failover authorization cannot be reached: FAILED:
Expected 'invalid command name "OK 4 usable Sentinels. Quorum and failover authorization can be reached"' to match '*NOQUORUM*'
```

It seems that current sentinel does not confirm that the other
sentinels are actually `down`, and then check the quorum.
It at least take 3 seconds on my machine, and we can see there
will be a timing issue with the hard code `after 5000`.

In this commit, we check the response of `SENTINEL SENTINELS mymaster`
to ensure that other sentinels are actually `down` in the view the
current sentinel. Solve the timing issue due to sentinel monitor mechanism.
 2022-01-23 13:54:50 +02:00
BinbinOran Agraguybe7
 		23325c135f sub-command support for ACL CAT and COMMAND LIST. redisCommand always stores fullname (#10127) 
Summary of changes:
1. Rename `redisCommand->name` to `redisCommand->declared_name`, it is a
  const char * for native commands and SDS for module commands.
2. Store the [sub]command fullname in `redisCommand->fullname` (sds).
3. List subcommands in `ACL CAT`
4. List subcommands in `COMMAND LIST`
5. `moduleUnregisterCommands` now will also free the module subcommands.
6. RM_GetCurrentCommandName returns full command name

Other changes:
1. Add `addReplyErrorArity` and `addReplyErrorExpireTime`
2. Remove `getFullCommandName` function that now is useless.
3. Some cleanups about `fullname` since now it is SDS.
4. Delete `populateSingleCommand` function from server.h that is useless.
5. Added tests to cover this change.
6. Add some module unload tests and fix the leaks
7. Make error messages uniform, make sure they always contain the full command
  name and that it's quoted.
7. Fixes some typos

see the history in #9504, fixes #10124

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: guybe7 <guy.benoish@redislabs.com>
 2022-01-23 10:05:06 +02:00
guybe7 a6fd2a46d1 Improved handling of subcommands (don't allow ACL on first-arg of a sub-command) (#10147) 
Recently we added extensive support for sub-commands in for redis 7.0,
this meant that the old ACL mechanism for
sub-commands wasn't needed, or actually was improved (to handle both include
and exclude control, like for commands), but only for real sub-commands.
The old mechanism in ACL was renamed to first-arg, and was able to match the
first argument of any command (including sub-commands).
We now realized that we might wanna completely delete that first-arg feature some
day, so the first step was not to give it new capabilities in 7.0 and it didn't have before.

Changes:
1. ACL: Block the first-arg mechanism on subcommands (we keep if in non-subcommands
  for backward compatibility)
2. COMMAND: When looking up a command, insist the command name doesn't contain
  extra words. Example: When a user issues `GET key` we want `lookupCommand` to return
  `getCommand` but when if COMMAND calls `lookupCommand` with `get|key` we want it to fail.

Other changes:
1. ACLSetUser: prevent a redundant command lookup
 2022-01-22 14:09:40 +02:00
Madelyn OlsonandOran Agra 55c81f2cd3 ACL V2 - Selectors and key based permissions (#9974) 
* Implemented selectors which provide multiple different sets of permissions to users
* Implemented key based permissions 
* Added a new ACL dry-run command to test permissions before execution
* Updated module APIs to support checking key based permissions

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-20 13:05:27 -08:00
guybe7 10bbeb6837 Add command tips to COMMAND DOCS (#10104) 
Adding command tips (see https://redis.io/topics/command-tips) to commands.

Breaking changes:
1. Removed the "random" and "sort_for_script" flags. They are now command tips.
(this isn't affecting redis behavior since #9812, but could affect some client applications
that's relying on COMMAND command flags)

Summary of changes:
1. add BLOCKING flag (new flag) for all commands that could block. The ACL category with
  the same name is now implicit.
2. move RANDOM flag to a `nondeterministic_output` tip
3. move SORT_FOR_SCRIPT flag to `nondeterministic_output_order` tip
3. add REQUEST_POLICY and RESPONSE_POLICY where appropriate as documented in the tips
4. deprecate (ignore) the `random` flag for RM_CreateCommand

Other notes:
1. Proxies need to send `RANDOMKEY` to all shards and then select one key randomly.
  The other option is to pick a random shard and transfer `RANDOMKEY `to it, but that scheme
  fails if this specific shard is empty
2. Remove CMD_RANDOM from `XACK` (i.e. XACK does not have RANDOM_OUTPUT)
   It was added in 9e4fb96ca1, I guess by mistake.
   Also from `(P)EXPIRETIME` (new command, was flagged "random" by mistake).
3. Add `nondeterministic_output` to `OBJECT ENCODING` (for the same reason `XTRIM` has it:
   the reply may differ depending on the internal representation in memory)
4. RANDOM on `HGETALL` was wrong (there due to a limitation of the old script sorting logic), now
  it's `nondeterministic_output_order`
5. Unrelated: Hide CMD_PROTECTED from COMMAND
 2022-01-20 11:32:11 +02:00
Meir Shpilraien (Spielrein) 9c60292250 Added functions support to redis-check-rdb (#10154) 
The PR added the missing verification for functions on redis-check-rdb.
The verification only verifies the rdb structure and does not try to load the functions code and
verify more advance checks (like compilation of the function code).
 2022-01-20 11:10:33 +02:00
perryitayandOran Agra c4b788230c Adding module api for processing commands during busy jobs and allow flagging the commands that should be handled at this status (#9963) 
Some modules might perform a long-running logic in different stages of Redis lifetime, for example:
* command execution
* RDB loading
* thread safe context

During this long-running logic Redis is not responsive.

This PR offers 
1. An API to process events while a busy command is running (`RM_Yield`)
2. A new flag (`ALLOW_BUSY`) to mark the commands that should be handled during busy
  jobs which can also be used by modules (`allow-busy`)
3. In slow commands and thread safe contexts, this flag will start rejecting commands with -BUSY only
  after `busy-reply-threshold`
4. During loading (`rdb_load` callback), it'll process events right away (not wait for `busy-reply-threshold`),
  but either way, the processing is throttled to the server hz rate.
5. Allow modules to Yield to redis background tasks, but not to client commands

* rename `script-time-limit` to `busy-reply-threshold` (an alias to the pre-7.0 `lua-time-limit`)

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-20 09:05:53 +02:00
Madelyn Olson 22172a4aa6 Fix double key declaration for renamenx and change flag to INSERT (#10137) 
* Fix double key declaration for renamenx
* Change the flag from UPDATE to INSERT, since it can not affect the data in the key
 2022-01-19 19:30:32 -08:00
Meir Shpilraien (Spielrein)andOran Agra c556c57e5e Added AOF rewrite support for functions. (#10141) 
Function PR was merged without AOF rw support because we thought this feature was going
to be removed on Redis 7.

Tests was added on aofrw.tcl
Other existing aofrw tests where slow due to unwanted rdb-key-save-delay

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-19 21:21:42 +02:00
David CARLIER 834fa5870c zmalloc_get_rss openbsd implementation (#10149) 
Add support for getting the RSS in OpenBSD
 2022-01-19 20:56:12 +02:00
Binbin 3720de78c9 Fix fcall_ro key spec flags in commands.c (#10145) 
In #10122, we modify the key spec flags to `RO` and `ACCESS`.
But forgot to call generate-command-code.py. Also formatted
it to follow the Python PEP8.
 2022-01-19 16:07:28 +02:00
Wen Hui 68a8d0b46d Add sentinel config set test case (#10114) 2022-01-19 11:57:51 +02:00
Ozan Tezcan 72e1b5de4d Fix replica count check in migration tests. (#10140) 
Tests were not using loop index as node id, checking replica count of the same node over and over.
 2022-01-19 11:36:24 +02:00
Ozan Tezcan 1af0a2c5ae Fix eventloop module test for valgrind (#10139) 
was eating too much memory, and taking too long with valgrind
 2022-01-19 09:13:51 +02:00
Oran Agra eef9c6b0ee New detailed key-spec flags (RO, RW, OW, RM, ACCESS, UPDATE, INSERT, DELETE) (#10122) 
The new ACL key based permissions in #9974 require the key-specs (#8324) to have more
explicit flags rather than just READ and WRITE. See discussion in #10040

This PR defines two groups of flags:
One about how redis internally handles the key (mutually-exclusive).
The other is about the logical operation done from the user's point of view (3 mutually exclusive
write flags, and one read flag, all optional).
In both groups, if we can't explicitly flag something as explicit read-only, delete-only, or
insert-only, we flag it as `RW` or `UPDATE`.
here's the definition from the code:
```
/* Key-spec flags *
 * -------------- */
/* The following refer what the command actually does with the value or metadata
 * of the key, and not necessarily the user data or how it affects it.
 * Each key-spec may must have exaclty one of these. Any operation that's not
 * distinctly deletion, overwrite or read-only would be marked as RW. */
#define CMD_KEY_RO (1ULL<<0)     /* Read-Only - Reads the value of the key, but
                                  * doesn't necessarily returns it. */
#define CMD_KEY_RW (1ULL<<1)     /* Read-Write - Modifies the data stored in the
                                  * value of the key or its metadata. */
#define CMD_KEY_OW (1ULL<<2)     /* Overwrite - Overwrites the data stored in
                                  * the value of the key. */
#define CMD_KEY_RM (1ULL<<3)     /* Deletes the key. */
/* The follwing refer to user data inside the value of the key, not the metadata
 * like LRU, type, cardinality. It refers to the logical operation on the user's
 * data (actual input strings / TTL), being used / returned / copied / changed,
 * It doesn't refer to modification or returning of metadata (like type, count,
 * presence of data). Any write that's not INSERT or DELETE, would be an UPADTE.
 * Each key-spec may have one of the writes with or without access, or none: */
#define CMD_KEY_ACCESS (1ULL<<4) /* Returns, copies or uses the user data from
                                  * the value of the key. */
#define CMD_KEY_UPDATE (1ULL<<5) /* Updates data to the value, new value may
                                  * depend on the old value. */
#define CMD_KEY_INSERT (1ULL<<6) /* Adds data to the value with no chance of,
                                  * modification or deletion of existing data. */
#define CMD_KEY_DELETE (1ULL<<7) /* Explicitly deletes some content
                                  * from the value of the key. */
```

Unrelated changes:
- generate-command-code.py is only compatible with python3 (modified the shabang)
- generate-command-code.py print file on json parsing error
- rename `shard_channel` key-spec flag to just `channel`.
- add INCOMPLETE flag in input spec of SORT and SORT_RO
 2022-01-18 16:00:00 +02:00
Wang Yuan d697daa7a5 Use const char pointer in redismodule.h as far as possible (#10064) 
When I used C++ to develop a redis module. i  used `string.data()` as the second parameter `ele`
of  `RedisModule_DigestAddStringBuffer`, but there is a warning, since we never change the `ele`,
i think we should use `const char` for it.

This PR adds const to just a handful of module APIs that required it, all not very widely used.
The implication is a breaking change in terms of compilation error that's easy to resolve, and no ABI impact.
The affected APIs are around Digest, Info injection, and Cluster bus messages.
 2022-01-18 15:55:20 +02:00
Ozan Tezcan 99ab4236af Add event loop support to the module API (#10001) 
Modules can now register sockets/pipe to the Redis main thread event loop and do network operations asynchronously. Previously, modules had to maintain an event loop and another thread for asynchronous network operations.

Also, if a module is calling API functions after doing some network operations, it had to synchronize its event loop thread's access with Redis main thread by locking the GIL, causing contention on the lock. After this commit, no synchronization is needed as module can operate in Redis main thread context. So, this commit may improve the performance for some use cases.

Added three functions to the module API:

* RedisModule_EventLoopAdd(int fd, int mask, RedisModuleEventLoopFunc func, void *user_data)
* RedisModule_EventLoopDel(int fd, int mask)
* RedisModule_EventLoopAddOneShot(RedisModuleEventLoopOneShotFunc func, void *user_data) - This function can be called from other threads to trigger callback on Redis main thread. Callback will be triggered only once. If Redis main thread is sleeping, this call will wake up the Redis main thread.
Event loop callbacks are called by Redis main thread after locking the GIL. Inside callbacks, modules can operate as if they are holding the GIL.

Added REDISMODULE_EVENT_EVENTLOOP event with two subevents:

* REDISMODULE_SUBEVENT_EVENTLOOP_BEFORE_SLEEP
* REDISMODULE_SUBEVENT_EVENTLOOP_AFTER_SLEEP

These events are for modules that want to participate in the before and after sleep action. e.g It might be useful to implement batching : Read data from the network, write all to a file in one go on BEFORE_SLEEP event.
 2022-01-18 13:10:07 +02:00
Yossi Gottlieb 25e6d4d459 Fix additional AOF filename issues. (#10110) 
This extends the previous fix (#10049) to address any form of
non-printable or whitespace character (including newlines, quotes,
non-printables, etc.)

Also, removes the limitation on appenddirname, to align with the way
filenames are handled elsewhere in Redis.
 2022-01-18 12:52:27 +02:00
Meir Shpilraien (Spielrein) 51f9bed3dd Fix FUNCTION LOAD ignores unknown parameter. (#10131) 
Following discussion on: https://github.com/redis/redis/issues/9899#issuecomment-1014689385
Raise error if unknows parameter is given to `FUNCTION LOAD`.

Before the fix:
```
127.0.0.1:6379> function load LUA lib2 foo bar "local function test1() return 5 end redis.register_function('test1', test1)"
OK
```

After the fix:
```
127.0.0.1:6379> function load LUA lib2 foo bar "local function test1() return 5 end redis.register_function('test1', test1)"
(error) ERR Unkowns option given: foo
```
 2022-01-18 10:29:52 +02:00
Oran Agra ae89958972 Set repl-diskless-sync to yes by default, add repl-diskless-sync-max-replicas (#10092) 
1. enable diskless replication by default
2. add a new config named repl-diskless-sync-max-replicas that enables
   replication to start before the full repl-diskless-sync-delay was
   reached.
3. put replica online sooner on the master (see below)
4. test suite uses repl-diskless-sync-delay of 0 to be faster
5. a few tests that use multiple replica on a pre-populated master, are
   now using the new repl-diskless-sync-max-replicas
6. fix possible timing issues in a few cluster tests (see below)

put replica online sooner on the master 
----------------------------------------------------
there were two tests that failed because they needed for the master to
realize that the replica is online, but the test code was actually only
waiting for the replica to realize it's online, and in diskless it could
have been before the master realized it.

changes include two things:
1. the tests wait on the right thing
2. issues in the master, putting the replica online in two steps.

the master used to put the replica as online in 2 steps. the first
step was to mark it as online, and the second step was to enable the
write event (only after getting ACK), but in fact the first step didn't
contains some of the tasks to put it online (like updating good slave
count, and sending the module event). this meant that if a test was
waiting to see that the replica is online form the point of view of the
master, and then confirm that the module got an event, or that the
master has enough good replicas, it could fail due to timing issues.

so now the full effect of putting the replica online, happens at once,
and only the part about enabling the writes is delayed till the ACK.

fix cluster tests 
--------------------
I added some code to wait for the replica to sync and avoid race
conditions.
later realized the sentinel and cluster tests where using the original 5
seconds delay, so changed it to 0.

this means the other changes are probably not needed, but i suppose
they're still better (avoid race conditions)
 2022-01-17 14:11:11 +02:00
zhaozhao.zzandOran Agra 90916f16a5 show subcommands latencystats (#10103) 
since `info commandstats` already shows sub-commands, we should do the same in `info latencystats`.
similarly, the LATENCY HISTOGRAM command now shows sub-commands (with their full name) when:
* asking for all commands
* asking for a specific container command
* asking for a specific sub-command)

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-17 12:32:32 +02:00
Binbin 26ef5132a6 Fix timing issue in PSETEX/PEXPIRE sub-second expire tests (#10121) 
These two tests have a high probability of failure
on MacOS. Or it takes many retries to succeed.
Keys often expire before we can access them.

So this time we try to avoid this by reducing the time
of the first `after`, or removeing the first `after`.

The results of doing `20/81` and `0/101` are:
- PEXPIRE (20/81): 1069/1949
- PEXPIREAT (20/81): 1093/1949

- PEXPIRE (0/101): 31936 / 31936
- PEXPIREAT (0/101): 31936 / 31936

The first number is the number of times that the
test succeeded without any retries.
The second number is the total number of executions.

And we can see that `0/101` doesn't even need an extra
retries. Also reduces the time required for testing.
So in the end we chose `0/100`, i.e. remove the first `after`.

As for `PEXPIREAT`, there is no failure, but we still changed
it together, using `0/201`, after 2W tests, none of them failed.
 2022-01-17 10:42:13 +02:00
chenyang8094 d7aeb2ce5f Use am instead of using server.aof_manifest directly to call getBaseAndIncrAppendOnlyFilesSize (#10123) 2022-01-17 09:02:09 +02:00
David CARLIER 50fa627b90 zmalloc_get_rss netbsd impl fix proposal. (#10116) 
Seems like the previous implementation was broken (always returning 0)

since kinfo_proc2 is used the KERN_PROC2 sysctl oid is more appropriate
and also the query's length was not necessarily accurate (6 here).
 2022-01-16 10:03:09 +02:00
sundb 32e7b46a17 Fix quicklist node not being recompressed correctly after inserting a new node before or after it (#10120) 
### Describe
Fix crash found by CI, Introduced by #9849.
When we do any operation on the quicklist, we should make sure that all nodes
of the quicklist should not be in the recompressed state.

### Issues
This PR fixes two issues with incorrect recompression.
1. The current quicklist node is full and the previous node isn't full,
    the current node is not recompressed correctly after inserting elements into the previous node.
2. The current quicklist node is full and the next node isn't full,
    the current node is not recompressed correctly after inserting elements into the next node.

### Test
Add two tests to cover incorrect compression issues.

### Other
Fix unittest test failure caused by assertion introduced by #9849.
 2022-01-16 08:54:40 +02:00
David CARLIER 7da7d2aa8e checkTcpBacklogSettings check for solaris based systems. (#10109) 2022-01-15 20:39:05 +02:00
Ali-Akber Saifee 98414aca7f Fix typo in doc for PUBSUB command (#10117) 
Replace "Pub/Sun" -> "Pub/Sub"
 2022-01-15 19:57:24 +02:00
Binbin 440d28091b Fix function no-cluster flag test (#10115) 
Fixes cluster test introduced in #10066.
```
Function no-cluster flag: ERR Error registering functions: @user_function: 1: wrong number of arguments to redis.register_function
```
 2022-01-15 09:13:53 +02:00
Meir Shpilraien (Spielrein) 4db4b43417 Function Flags support (no-writes, no-cluster, allow-state, allow-oom) (#10066) 
# Redis Functions Flags

Following the discussion on #10025 Added Functions Flags support.
The PR is divided to 2 sections:
* Add named argument support to `redis.register_function` API.
* Add support for function flags

## `redis.register_function` named argument support

The first part of the PR adds support for named argument on `redis.register_function`, example:
```
redis.register_function{
    function_name='f1',
    callback=function()
        return 'hello'
    end,
    description='some desc'
}
```

The positional arguments is also kept, which means that it still possible to write:
```
redis.register_function('f1', function() return 'hello' end)
```

But notice that it is no longer possible to pass the optional description argument on the positional
argument version. Positional argument was change to allow passing only the mandatory arguments
(function name and callback). To pass more arguments the user must use the named argument version.

As with positional arguments, the `function_name` and `callback` is mandatory and an error will be
raise if those are missing. Also, an error will be raise if an unknown argument name is given or the
arguments type is wrong.

Tests was added to verify the new syntax.

## Functions Flags

The second part of the PR is adding functions flags support.
Flags are given to Redis when the engine calls `functionLibCreateFunction`, supported flags are:

* `no-writes` - indicating the function perform no writes which means that it is OK to run it on:
   * read-only replica
   * Using FCALL_RO
   * If disk error detected
   
   It will not be possible to run a function in those situations unless the function turns on the `no-writes` flag

* `allow-oom` - indicate that its OK to run the function even if Redis is in OOM state, if the function will
  not turn on this flag it will not be possible to run it if OOM reached (even if the function declares `no-writes`
  and even if `fcall_ro` is used). If this flag is set, any command will be allow on OOM (even those that is
  marked with CMD_DENYOOM). The assumption is that this flag is for advance users that knows its
  meaning and understand what they are doing, and Redis trust them to not increase the memory usage.
  (e.g. it could be an INCR or a modification on an existing key, or a DEL command)

* `allow-state` - indicate that its OK to run the function on stale replica, in this case we will also make
  sure the function is only perform `stale` commands and raise an error if not.

* `no-cluster` - indicate to disallow running the function if cluster is enabled.

Default behaviure of functions (if no flags is given):
1. Allow functions to read and write
2. Do not run functions on OOM
3. Do not run functions on stale replica
4. Allow functions on cluster

### Lua API for functions flags

On Lua engine, it is possible to give functions flags as `flags` named argument:

```
redis.register_function{function_name='f1', callback=function() return 1 end, flags={'no-writes', 'allow-oom'}, description='description'}
```

The function flags argument must be a Lua table that contains all the requested flags, The following
will result in an error:
* Unknown flag
* Wrong flag type

Default behaviour is the same as if no flags are used.

Tests were added to verify all flags functionality

## Additional changes
* mark FCALL and FCALL_RO with CMD_STALE flag (unlike EVAL), so that they can run if the function was
  registered with the `allow-stale` flag.
* Verify `CMD_STALE` on `scriptCall` (`redis.call`), so it will not be possible to call commands from script while
  stale unless the command is marked with the `CMD_STALE` flags. so that even if the function is allowed while
  stale we do not allow it to bypass the `CMD_STALE` flag of commands.
* Flags section was added to `FUNCTION LIST` command to provide the set of flags for each function:
```
> FUNCTION list withcode
1)  1) "library_name"
    2) "test"
    3) "engine"
    4) "LUA"
    5) "description"
    6) (nil)
    7) "functions"
    8) 1) 1) "name"
          2) "f1"
          3) "description"
          4) (nil)
          5) "flags"
          6) (empty array)
    9) "library_code"
   10) "redis.register_function{function_name='f1', callback=function() return 1 end}"
```
* Added API to get Redis version from within a script, The redis version can be provided using:
   1. `redis.REDIS_VERSION` - string representation of the redis version in the format of MAJOR.MINOR.PATH
   2. `redis.REDIS_VERSION_NUM` - number representation of the redis version in the format of `0x00MMmmpp`
      (`MM` - major, `mm` - minor,  `pp` - patch). The number version can be used to check if version is greater or less 
      another version. The string version can be used to return to the user or print as logs.

   This new API is provided to eval scripts and functions, it also possible to use this API during functions loading phase.
 2022-01-14 14:02:02 +02:00
Peter Hickey 38a5116728 Fix typo in deps/README.md (#10113) 2022-01-14 13:54:22 +02:00
Binbin 56a802057e Fix kill aof rewrite child test (#10107) 
The dbs doesn't have any keys, `rdb-key-save-delay`
config has no effect that cause the rewrite to complete.

It was introduced in #10015.
 2022-01-13 12:38:41 +02:00
Ozan Tezcan f41cc87088 Added RM_MonotonicMicroseconds() API to provide monotonic time function (#10101) 
Added RM_MonotonicMicroseconds(). Modules can use monotonic timestamp counter for measurements.
 2022-01-13 11:36:03 +02:00
chenyang8094andOran Agra e9bff7978a Always create base AOF file when redis start from empty. (#10102) 
Force create a BASE file (use a foreground `rewriteAppendOnlyFile`) when redis starts from an
empty data set and  `appendonly` is  yes.

The reasoning is that normally, after redis is running for some time, and the AOF has gone though
a few rewrites, there's always a base rdb file. and the scenario where the base file is missing, is
kinda rare (happens only at empty startup), so this change normalizes it.
But more importantly, there are or could be some complex modules that are started with some
configuration, when they create persistence they write that configuration to RDB AUX fields, so
that can can always know with which configuration the persistence file they're loading was
created (could be critical). there is (was) one scenario in which they could load their persisted data,
and that configuration was missing, and this change fixes it.

Add a new module event: REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_AOF_START, similar to
REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START which is async.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-13 08:49:26 +02:00
Binbin 20c33fe6a8 Show subcommand full name in error log / ACL LOG (#10105) 
Use `getFullCommandName` to get the full name of the command.
It can also get the full name of the subcommand, like "script|help".

Before:
```
> SCRIPT HELP
(error) NOPERM this user has no permissions to run the 'help' command or its subcommand

> ACL LOG
    7) "object"
    8) "help"
```

After:
```
> SCRIPT HELP
(error) NOPERM this user has no permissions to run the 'script|help' command

> ACL LOG
    7) "object"
    8) "script|help"
```

Fix #10094
 2022-01-12 20:05:14 +02:00
Binbin e22146b07a Add script tests to cover keys with expiration time set (#10096) 
This commit adds some tests that the test cases will
access the keys with expiration time set in the script call.
There was no test case for this part before. See #10080

Also there is a test will cover #1525. we block the time so
that the key can not expire in the middle of the script execution.

Other changes:
1. Delete `evalTimeSnapshot` and just use `scriptTimeSnapshot` in it's place.
2. Some cleanups to scripting.tcl.
3. better names for tests that run in a loop to make them distinctable
 2022-01-11 22:43:18 +02:00
Ozan Tezcan 6790d848c5 Reuse temporary client objects for blocked clients by module (#9940) 
Added a pool for temporary client objects to reuse in module operations.
By reusing temporary clients, we are avoiding expensive createClient()/freeClient()
calls and improving performance of RM_BlockClient() and  RM_GetThreadSafeContext() calls. 

This commit contains two optimizations: 

1 - RM_BlockClient() and RM_GetThreadSafeContext() calls create temporary clients and they are freed in
RM_UnblockClient() and RM_FreeThreadSafeContext() calls respectively. Creating/destroying client object
takes quite time. To avoid that, added a pool of temporary clients. Pool expands when more clients are needed.
Also, added a cron function to shrink the pool and free unused clients after some time. Pool starts with zero
clients in it. It does not have max size and can grow unbounded as we need it. We will keep minimum of 8
temporary clients in the pool once created. Keeping small amount of clients to avoid client allocation costs
if temporary clients are required after some idle period.

2 - After unblocking a client (RM_UnblockClient()), one byte is written to pipe to wake up Redis main thread.
If there are many clients that will be unblocked, each operation requires one write() call which is quite expensive.
Changed code to avoid subsequent calls if possible. 

There are a few more places that need temporary client objects (e.g RM_Call()). These are now using the same
temporary client pool to make things more centralized.
 2022-01-11 19:00:56 +02:00
Oran Agra 3204a03574 Move doc metadata from COMMAND to COMMAND DOCS (#10056) 
Syntax:
`COMMAND DOCS [<command name> ...]`

Background:
Apparently old version of hiredis (and thus also redis-cli) can't
support more than 7 levels of multi-bulk nesting.

The solution is to move all the doc related metadata from COMMAND to a
new COMMAND DOCS sub-command.

The new DOCS sub-command returns a map of commands (not an array like in COMMAND),
And the same goes for the `subcommands` field inside it (also contains a map)

Besides that, the remaining new fields of COMMAND (hints, key-specs, and
sub-commands), are placed in the outer array rather than a nested map.
this was done mainly for consistency with the old format.

Other changes:
---
* Allow COMMAND INFO with no arguments, which returns all commands, so that we can some day deprecated
  the plain COMMAND (no args)

* Reduce the amount of deferred replies from both COMMAND and COMMAND
  DOCS, especially in the inner loops, since these create many small
  reply objects, which lead to many small write syscalls and many small
  TCP packets.
  To make this easier, when populating the command table, we count the
  history, args, and hints so we later know their size in advance.
  Additionally, the movablekeys flag was moved into the flags register.
* Update generate-commands-json.py to take the data from both command, it
  now executes redis-cli directly, instead of taking input from stdin.
* Sub-commands in both COMMAND (and COMMAND INFO), and also COMMAND DOCS,
  show their full name. i.e. CONFIG 
*   GET will be shown as `config|get` rather than just `get`.
  This will be visible both when asking for `COMMAND INFO config` and COMMAND INFO config|get`, but is
  especially important for the later.
  i.e. imagine someone doing `COMMAND INFO slowlog|get config|get` not being able to distinguish between the two
  items in the array response.
 2022-01-11 17:16:16 +02:00
guybe7 5009b43dc6 Module events: Fail RM_SubscribeToServerEvent if event is too new (#9987) 
We must fail RM_SubscribeToServerEvent in case a module, that
was compiled with a new redismodule.h, tries to subscribe to an
event that doesn't exist on an old redis-server
 2022-01-11 14:33:39 +02:00
Binbin 39feee8e3a LPOP/RPOP with count against non existing list return null array (#10095) 
It used to return `$-1` in RESP2, now we will return `*-1`.
This is a bug in redis 6.2 when COUNT was added, the `COUNT`
option was introduced in #8179. Fix #10089.

the documentation of [LPOP](https://redis.io/commands/lpop) says
```
When called without the count argument:
Bulk string reply: the value of the first element, or nil when key does not exist.

When called with the count argument:
Array reply: list of popped elements, or nil when key does not exist.
```
 2022-01-11 14:26:13 +02:00
小令童鞋 1e25bdf780 fix redis crached by using eval with access to volatile keys (#10080) 
This is a recent regression from the Redis Functions commits
 2022-01-11 07:56:31 +02:00
Madelyn Olson d0949b7c5c Fix timing issue with cluster hostname test (#10086) 2022-01-10 16:21:05 -08:00
chenyang8094 bd46a2abf4 Support whitespace characters in appendfilename, and ban them in appenddirname (#10049) 
1. Ban whitespace characters in `appenddirname`   
2. Handle the case where `appendfilename` contains spaces (for backwards compatibility)
 2022-01-10 09:09:39 +02:00
Binbin a1ae260e8a Make sure replicas don't write their own replies to the replication link (#10081) 
The following steps will crash redis-server:
```
[root]# cat crash
PSYNC replicationid -1
SLOWLOG GET
GET key
[root]# nc 127.0.0.1 6379 < crash
```

This one following #10020 and the crash was reported in #10076.

Other changes about the output info:
1. Cmd with a full name by using `getFullCommandName`, now it will print the right
   subcommand name like `slowlog|get`.
2. Print the full client info by using `catClientInfoString`, the info is also valuable.:
 2022-01-10 08:21:16 +02:00
Wen Hui d7479107c7 update codes according to new json file (#7425) 
readonly/readwrite only set client flags for slave in cluster mode,
so it should be ok for setting ok-stale and ok-loading command flag
 2022-01-10 08:00:04 +02:00
Madelyn Olson e8e02f900c Changed latency histogram output to omit trailing 0s and periods (#10075) 
Changed latency percentile output to omit trailing 0s and periods
 2022-01-09 17:04:18 -08:00
Binbin a84c964d37 Fix crash when error [sub]command name contains | (#10082) 
The following error commands will crash redis-server:
```
> get|
Error: Server closed the connection
> get|set
Error: Server closed the connection
> get|other
```

The reason is in #9504, we use `lookupCommandBySds` for find the
container command. And it split the command (argv[0]) with `|`.
If we input something like `get|other`, after the split, `get`
will become a valid command name, pass the `ERR unknown command`
check, and finally crash in `addReplySubcommandSyntaxError`

In this case we do not need to split the command name with `|`
and just look in the commands dict to find if `argv[0]` is a
container command.

So this commit introduce a new function call `isContainerCommandBySds`
that it will return true if a command name is a container command.

Also with the old code, there is a incorrect error message:
```
> config|get set
(error) ERR Unknown subcommand or wrong number of arguments for 'set'. Try CONFIG|GET HELP.
```

The crash was reported in #10070.
 2022-01-09 13:06:51 +02:00
Itamar Haber 75c50a1563 Fixes 'since' for GEORADIUS[BYMEMBER]_RO (#10034) 2022-01-09 11:13:32 +02:00
YEONCHEOL JANG b9669829c8 Fixes typo for redis.conf (#10072) 
Fixes typo for redis.conf
 2022-01-09 08:08:55 +02:00
guybe7 7cd6a64d2f lpGetInteger returns int64_t, avoid overflow (#10068) 
Fix #9410

Crucial for the ms and sequence deltas, but I changed all
calls, just in case (e.g. "flags")

Before this commit:
`ms_delta` and `seq_delta` could have overflown, causing `currid` to be wrong,
which in turn would cause `streamTrim` to trim the entire rax node (see new test)
 2022-01-07 15:31:05 +02:00
Viktor Söderqvist e88f6acb94 Build commands.c in Makefile (#10039) 
With this rule, the script to generate commands.c from JSON runs whenever commands.o is built if any of commands/*.json are modified. Without such rule, it's easy to forget to run the script when updating the JSON files.

It's a follow-up on #9656 and #9951.
 2022-01-06 17:59:37 +02:00
Meir Shpilraien (Spielrein) 885f6b5ceb Redis Function Libraries (#10004) 
# Redis Function Libraries

This PR implements Redis Functions Libraries as describe on: https://github.com/redis/redis/issues/9906.

Libraries purpose is to provide a better code sharing between functions by allowing to create multiple
functions in a single command. Functions that were created together can safely share code between
each other without worrying about compatibility issues and versioning.

Creating a new library is done using 'FUNCTION LOAD' command (full API is described below)

This PR introduces a new struct called libraryInfo, libraryInfo holds information about a library:
* name - name of the library
* engine - engine used to create the library
* code - library code
* description - library description
* functions - the functions exposed by the library

When Redis gets the `FUNCTION LOAD` command it creates a new empty libraryInfo.
Redis passes the `CODE` to the relevant engine alongside the empty libraryInfo.
As a result, the engine will create one or more functions by calling 'libraryCreateFunction'.
The new funcion will be added to the newly created libraryInfo. So far Everything is happening
locally on the libraryInfo so it is easy to abort the operation (in case of an error) by simply
freeing the libraryInfo. After the library info is fully constructed we start the joining phase by
which we will join the new library to the other libraries currently exist on Redis.
The joining phase make sure there is no function collision and add the library to the
librariesCtx (renamed from functionCtx). LibrariesCtx is used all around the code in the exact
same way as functionCtx was used (with respect to RDB loading, replicatio, ...).
The only difference is that apart from function dictionary (maps function name to functionInfo
object), the librariesCtx contains also a libraries dictionary that maps library name to libraryInfo object.

## New API
### FUNCTION LOAD
`FUNCTION LOAD <ENGINE> <LIBRARY NAME> [REPLACE] [DESCRIPTION <DESCRIPTION>] <CODE>`
Create a new library with the given parameters:
* ENGINE - REPLACE Engine name to use to create the library.
* LIBRARY NAME - The new library name.
* REPLACE - If the library already exists, replace it.
* DESCRIPTION - Library description.
* CODE - Library code.

Return "OK" on success, or error on the following cases:
* Library name already taken and REPLACE was not used
* Name collision with another existing library (even if replace was uses)
* Library registration failed by the engine (usually compilation error)

## Changed API
### FUNCTION LIST
`FUNCTION LIST [LIBRARYNAME <LIBRARY NAME PATTERN>] [WITHCODE]`
Command was modified to also allow getting libraries code (so `FUNCTION INFO` command is no longer
needed and removed). In addition the command gets an option argument, `LIBRARYNAME` allows you to
only get libraries that match the given `LIBRARYNAME` pattern. By default, it returns all libraries.

### INFO MEMORY
Added number of libraries to `INFO MEMORY`

### Commands flags
`DENYOOM` flag was set on `FUNCTION LOAD` and `FUNCTION RESTORE`. We consider those commands
as commands that add new data to the dateset (functions are data) and so we want to disallows
to run those commands on OOM.

## Removed API
* FUNCTION CREATE - Decided on https://github.com/redis/redis/issues/9906
* FUNCTION INFO - Decided on https://github.com/redis/redis/issues/9899

## Lua engine changes
When the Lua engine gets the code given on `FUNCTION LOAD` command, it immediately runs it, we call
this run the loading run. Loading run is not a usual script run, it is not possible to invoke any
Redis command from within the load run.
Instead there is a new API provided by `library` object. The new API's: 
* `redis.log` - behave the same as `redis.log`
* `redis.register_function` - register a new function to the library

The loading run purpose is to register functions using the new `redis.register_function` API.
Any attempt to use any other API will result in an error. In addition, the load run is has a time
limit of 500ms, error is raise on timeout and the entire operation is aborted.

### `redis.register_function`
`redis.register_function(<function_name>, <callback>, [<description>])`
This new API allows users to register a new function that will be linked to the newly created library.
This API can only be called during the load run (see definition above). Any attempt to use it outside
of the load run will result in an error.
The parameters pass to the API are:
* function_name - Function name (must be a Lua string)
* callback - Lua function object that will be called when the function is invokes using fcall/fcall_ro
* description - Function description, optional (must be a Lua string).

### Example
The following example creates a library called `lib` with 2 functions, `f1` and `f1`, returns 1 and 2 respectively:
```
local function f1(keys, args)
    return 1
end

local function f2(keys, args)
    return 2
end

redis.register_function('f1', f1)
redis.register_function('f2', f2)
```

Notice: Unlike `eval`, functions inside a library get the KEYS and ARGV as arguments to the
functions and not as global.

### Technical Details

On the load run we only want the user to be able to call a white list on API's. This way, in
the future, if new API's will be added, the new API's will not be available to the load run
unless specifically added to this white list. We put the while list on the `library` object and
make sure the `library` object is only available to the load run by using [lua_setfenv](https://www.lua.org/manual/5.1/manual.html#lua_setfenv) API. This API allows us to set
the `globals` of a function (and all the function it creates). Before starting the load run we
create a new fresh Lua table (call it `g`) that only contains the `library` API (we make sure
to set global protection on this table just like the general global protection already exists
today), then we use [lua_setfenv](https://www.lua.org/manual/5.1/manual.html#lua_setfenv)
to set `g` as the global table of the load run. After the load run finished we update `g`
metatable and set `__index` and `__newindex` functions to be `_G` (Lua default globals),
we also pop out the `library` object as we do not need it anymore.
This way, any function that was created on the load run (and will be invoke using `fcall`) will
see the default globals as it expected to see them and will not have the `library` API anymore.

An important outcome of this new approach is that now we can achieve a distinct global table
for each library (it is not yet like that but it is very easy to achieve it now). In the future we can
decide to remove global protection because global on different libraries will not collide or we
can chose to give different API to different libraries base on some configuration or input.

Notice that this technique was meant to prevent errors and was not meant to prevent malicious
user from exploit it. For example, the load run can still save the `library` object on some local
variable and then using in `fcall` context. To prevent such a malicious use, the C code also make
sure it is running in the right context and if not raise an error.
 2022-01-06 13:39:38 +02:00
Ozan Tezcan 568c2e039b Set errno to EEXIST in redisFork() if child process exists (#10059) 
Callers of redisFork() are logging `strerror(errno)` on failure.
`errno` is not set when there is already a child process, causing printing
current value of errno which was set before `redisFork()` call. 

Setting errno to EEXIST on this failure to provide more meaningful error message.
 2022-01-06 09:54:21 +02:00
filipe oliveiraandOran Agra 5dd15443ac Added INFO LATENCYSTATS section: latency by percentile distribution/latency by cumulative distribution of latencies (#9462) 
# Short description

The Redis extended latency stats track per command latencies and enables:
- exporting the per-command percentile distribution via the `INFO LATENCYSTATS` command.
  **( percentile distribution is not mergeable between cluster nodes ).**
- exporting the per-command cumulative latency distributions via the `LATENCY HISTOGRAM` command.
  Using the cumulative distribution of latencies we can merge several stats from different cluster nodes
  to calculate aggregate metrics .

By default, the extended latency monitoring is enabled since the overhead of keeping track of the
command latency is very small.
 
If you don't want to track extended latency metrics, you can easily disable it at runtime using the command:
 - `CONFIG SET latency-tracking no`

By default, the exported latency percentiles are the p50, p99, and p999.
You can alter them at runtime using the command:
- `CONFIG SET latency-tracking-info-percentiles "0.0 50.0 100.0"`


## Some details:
- The total size per histogram should sit around 40 KiB. We only allocate those 40KiB when a command
  was called for the first time.
- With regards to the WRITE overhead As seen below, there is no measurable overhead on the achievable
  ops/sec or full latency spectrum on the client. Including also the measured redis-benchmark for unstable
  vs this branch. 
- We track from 1 nanosecond to 1 second ( everything above 1 second is considered +Inf )

## `INFO LATENCYSTATS` exposition format

   - Format: `latency_percentiles_usec_<CMDNAME>:p0=XX,p50....` 

## `LATENCY HISTOGRAM [command ...]` exposition format

Return a cumulative distribution of latencies in the format of a histogram for the specified command names.

The histogram is composed of a map of time buckets:
- Each representing a latency range, between 1 nanosecond and roughly 1 second.
- Each bucket covers twice the previous bucket's range.
- Empty buckets are not printed.
- Everything above 1 sec is considered +Inf.
- At max there will be log2(1000000000)=30 buckets

We reply a map for each command in the format:
`<command name> : { `calls`: <total command calls> , `histogram` : { <bucket 1> : latency , < bucket 2> : latency, ...  } }`

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-05 14:01:05 +02:00
sundb 4d3c4cfac7 Show the elapsed time of single test and speed up some tests (#10058) 
Following #10038.

This PR introduces two changes.
1. Show the elapsed time of a single test in the test output, in order to have a more
detailed understanding of the changes in test run time.

2. Speedup two tests related to `key-load-delay` configuration.
other tests do not seem to be affected by #10003.
 2022-01-05 13:49:01 +02:00
Binbin 9538088751 Fix typos in aof.c / redis.conf (#10057) 2022-01-05 12:06:02 +02:00
Ozan Tezcan d1b5b63872 Fix typo in multi test (#10054) 2022-01-05 10:16:04 +02:00
Binbin b7f9e9ae39 Add tests for blocking XREAD[GROUP] when the stream ran dry (#10035) 
The purpose of this commit is to add some tests to
cover #5299, which was fixed in #5300 but without tests.

This commit should close #5306 and #5299.
 2022-01-04 21:48:49 +02:00
Yuta Hongo 8deb9a4f1e Stringify JSON key of --json option result (#10046) 
About RESP3 an ordered collection of key-value pairs, keys and value can
be any other RESP3 type, but a key should be string in JSON spec.
 2022-01-04 17:24:29 +02:00
yoav-steinberg 65a7635793 redis-cli --replica reads dummy empty rdb instead of full snapshot (#10044) 
This makes redis-cli --replica much faster and reduces COW/fork risks on server side.
This commit also improves the RDB filtering via REPLCONF rdb-filter-only to support no "include" specifiers at all.
 2022-01-04 17:09:22 +02:00
Matthieu MORELandMOREL Matthieu d5a3b3f5ec Setup dependabot for github-actions and codespell (#9857) 
This sets up  dependabot to check weekly updates for pip and github-actions dependencies.
If it finds an update it will create a PR to update the dependency. More information can be found here

It includes the update of:

* vmactions/freebsd-vm from 0.1.4 to 0.1.5
* codespell from 2.0.0 to 2.1.0

Also includes spelling fixes found by the latest version of codespell.
Includes a dedicated .codespell folder so dependabot can read a requirements.txt file and every files dedicated to codespell can be grouped in the same place

Co-Authored-By: Matthieu MOREL <mmorel-35@users.noreply.github.com>
Co-Authored-By: MOREL Matthieu <matthieu.morel@cnp.fr>
 2022-01-04 16:19:28 +02:00
Binbin c57e41c029 Print error messages in monitor/pubsub when errors occurs (#10050) 
In monitor/pubsub mode, if the server closes the connection,
for example, use `CLIENT KILL`, redis-cli will exit directly
without printing any error messages.

This commit ensures that redis-cli will try to print the
error messages before exiting. Also there is a minor cleanup
for restart, see the example below.

before:
```
127.0.0.1:6379> monitor
OK
[root@ redis]#

127.0.0.1:6379> subscribe channel
Reading messages... (press Ctrl-C to quit)
1) "subscribe"
2) "channel"
3) (integer) 1
[root@ redis]#

127.0.0.1:6379> restart
127.0.0.1:6379> get keyUse 'restart' only in Lua debugging mode.
(nil)
```

after:
```
127.0.0.1:6379> monitor
OK
Error: Server closed the connection
[root@ redis]#

127.0.0.1:6379> subscribe channel
Reading messages... (press Ctrl-C to quit)
1) "subscribe"
2) "channel"
3) (integer) 1
Error: Server closed the connection
[root@ redis]#

127.0.0.1:6379> restart
Use 'restart' only in Lua debugging mode.
```
 2022-01-04 14:45:10 +02:00
王辉andYossi Gottlieb 747b08bee0 Fix C11_ATOMIC detection on GNU Make 4.3 (#10033) 
Older version of GNU Make (<4.3) required quoting of number signs (#) to
avoid them being treated as a comment. Newer versions will treat this
quote as a literal.

This issue and a proposed solution is discussed here:
https://lists.gnu.org/archive/html/info-gnu/2020-01/msg00004.html

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
 2022-01-04 14:05:00 +02:00
guybe7andOran Agra ac84b1cd82 Ban snapshot-creating commands and other admin commands from transactions (#10015) 
Creating fork (or even a foreground SAVE) during a transaction breaks the atomicity of the transaction.
In addition to that, it could mess up the propagated transaction to the AOF file.

This change blocks SAVE, PSYNC, SYNC and SHUTDOWN from being executed inside MULTI-EXEC.
It does that by adding a command flag, so that modules can flag their commands with that flag too.

Besides it changes BGSAVE, BGREWRITEAOF, and CONFIG SET appendonly, to turn the
scheduled flag instead of forking righ taway.

Other changes:
* expose `protected`, `no-async-loading`, and `no_multi` flags in COMMAND command
* add a test to validate propagation of FLUSHALL inside a transaction.
* add a test to validate how CONFIG SET that errors reacts in a transaction

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-04 13:37:47 +02:00
zhaozhao.zz 2e1979a21e use startEvictionTimeProc() in config set maxmemory (#10019) 
This would mean that the effects of `CONFIG SET maxmemory` may not be visible once the command returns.
That could anyway happen since incremental eviction was added in redis 6.2 (see #7653)

We do this to fix one of the propagation bugs about eviction see #9890 and #10014.
 2022-01-04 13:08:10 +02:00
chenyang8094andOran Agra 87789fae0b Implement Multi Part AOF mechanism to avoid AOFRW overheads. (#9788) 
Implement Multi-Part AOF mechanism to avoid overheads during AOFRW.
Introducing a folder with multiple AOF files tracked by a manifest file.

The main issues with the the original AOFRW mechanism are:
* buffering of commands that are processed during rewrite (consuming a lot of RAM)
* freezes of the main process when the AOFRW completes to drain the remaining part of the buffer and fsync it.
* double disk IO for the data that arrives during AOFRW (had to be written to both the old and new AOF files)

The main modifications of this PR:
1. Remove the AOF rewrite buffer and related code.
2. Divide the AOF into multiple files, they are classified as two types, one is the the `BASE` type,
  it represents the full amount of data (Maybe AOF or RDB format) after each AOFRW, there is only
  one `BASE` file at most. The second is `INCR` type, may have more than one. They represent the
  incremental commands since the last AOFRW.
3. Use a AOF manifest file to record and manage these AOF files mentioned above.
4. The original configuration of `appendfilename` will be the base part of the new file name, for example:
  `appendonly.aof.1.base.rdb` and `appendonly.aof.2.incr.aof`
5. Add manifest-related TCL tests, and modified some existing tests that depend on the `appendfilename`
6. Remove the `aof_rewrite_buffer_length` field in info.
7. Add `aof-disable-auto-gc` configuration. By default we're automatically deleting HISTORY type AOFs.
  It also gives users the opportunity to preserve the history AOFs. just for testing use now.
8. Add AOFRW limiting measure. When the AOFRW failures reaches the threshold (3 times now),
  we will delay the execution of the next AOFRW by 1 minute. If the next AOFRW also fails, it will be
  delayed by 2 minutes. The next is 4, 8, 16, the maximum delay is 60 minutes (1 hour). During the limit
  period, we can still use the 'bgrewriteaof' command to execute AOFRW immediately.
9. Support upgrade (load) data from old version redis.
10. Add `appenddirname` configuration, as the directory name of the append only files. All AOF files and
  manifest file will be placed in this directory.
11. Only the last AOF file (BASE or INCR) can be truncated. Otherwise redis will exit even if
  `aof-load-truncated` is enabled.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-03 19:14:13 +02:00
Meir Shpilraien (Spielrein) 78a62c0124 Fix OOM error not raised of functions (#10048) 
OOM Error did not raise on functions due to a bug.
Added test to verify the fix.
 2022-01-03 19:04:29 +02:00
Madelyn Olson 5460c10047 Implement clusterbus message extensions and cluster hostname support (#9530) 
Implement the ability for cluster nodes to advertise their location with extension messages.
 2022-01-02 19:48:29 -08:00
Harkrishn PatroHarkrishn PatroMadelyn Olson
 		9f8885760b Sharded pubsub implementation (#8621) 
This commit implements a sharded pubsub implementation based off of shard channels.

Co-authored-by: Harkrishn Patro <harkrisp@amazon.com>
Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
 2022-01-02 16:54:47 -08:00
Binbin b8ba942ac2 Add DUMP RESTORE tests for redis-cli -x and -X options (#10041) 
This commit adds DUMP RESTORES tests for the -x and -X options.
I wanted to add it in #9980 which introduce the -X option, but
back then i failed due to some errors (related to redis-cli call).
 2022-01-02 13:58:22 +02:00
yoav-steinberg 2ff3fc1790 Make sure replicas don't write their own replies to the replication link (#10020) 
Since #9166 we have an assertion here to make sure replica clients don't write anything to their buffer.
But in reality a replica may attempt write data to it's buffer simply by sending a command on the replication link.
This command in most cases will be rejected since #8868 but it'll still generate an error.
Actually the only valid command to send on a replication link is 'REPCONF ACK` which generates no response.

We want to keep the design so that replicas can send commands but we need to avoid any situation where we start
putting data in their response buffers, especially since they aren't used anymore. This PR makes sure to disconnect
a rogue client which generated a write on the replication link that cause something to be written to the response buffer.

To recreate the bug this fixes simply connect via telnet to a redis server and write sync\r\n wait for the the payload to
be written and then write any command (valid or invalid), such as ping\r\n on the telnet connection. It'll crash the server.
 2022-01-02 11:18:43 +02:00
Joey from AWS 09c668f220 Report slot to keys map size in MEMORY STATS in cluster mode (#10017) 
Report slot to keys map size in MEMORY STATS in cluster mode
Report dictMetadataSize in MEMORY USAGE command as well
 2022-01-02 10:39:59 +02:00
Viktor SöderqvistandOran Agra 45a155bd0f Wait for replicas when shutting down (#9872) 
To avoid data loss, this commit adds a grace period for lagging replicas to
catch up the replication offset.

Done:

* Wait for replicas when shutdown is triggered by SIGTERM and SIGINT.

* Wait for replicas when shutdown is triggered by the SHUTDOWN command. A new
  blocked client type BLOCKED_SHUTDOWN is introduced, allowing multiple clients
  to call SHUTDOWN in parallel.
  Note that they don't expect a response unless an error happens and shutdown is aborted.

* Log warning for each replica lagging behind when finishing shutdown.

* CLIENT_PAUSE_WRITE while waiting for replicas.

* Configurable grace period 'shutdown-timeout' in seconds (default 10).

* New flags for the SHUTDOWN command:

    - NOW disables the grace period for lagging replicas.

    - FORCE ignores errors writing the RDB or AOF files which would normally
      prevent a shutdown.

    - ABORT cancels ongoing shutdown. Can't be combined with other flags.

* New field in the output of the INFO command: 'shutdown_in_milliseconds'. The
  value is the remaining maximum time to wait for lagging replicas before
  finishing the shutdown. This field is present in the Server section **only**
  during shutdown.

Not directly related:

* When shutting down, if there is an AOF saving child, it is killed **even** if AOF
  is disabled. This can happen if BGREWRITEAOF is used when AOF is off.

* Client pause now has end time and type (WRITE or ALL) per purpose. The
  different pause purposes are *CLIENT PAUSE command*, *failover* and
  *shutdown*. If clients are unpaused for one purpose, it doesn't affect client
  pause for other purposes. For example, the CLIENT UNPAUSE command doesn't
  affect client pause initiated by the failover or shutdown procedures. A completed
  failover or a failed shutdown doesn't unpause clients paused by the CLIENT
  PAUSE command.

Notes:

* DEBUG RESTART doesn't wait for replicas.

* We already have a warning logged when a replica disconnects. This means that
  if any replica connection is lost during the shutdown, it is either logged as
  disconnected or as lagging at the time of exit.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-02 09:50:15 +02:00
yoav-steinbergandOran Agra 1bf6d6f11e Generate RDB with Functions only via redis-cli --functions-rdb (#9968) 
This is needed in order to ease the deployment of functions for ephemeral cases, where user
needs to spin up a server with functions pre-loaded.

#### Details:

* Added `--functions-rdb` option to _redis-cli_.
* Functions only rdb via `REPLCONF rdb-filter-only functions`. This is a placeholder for a space
  separated inclusion filter for the RDB. In the future can be `REPLCONF rdb-filter-only
  "functions db:3 key-patten:user*"` and a complementing `rdb-filter-exclude` `REPLCONF`
  can also be added.
* Handle "slave requirements" specification to RDB saving code so we can use the same RDB
  when different slaves express the same requirements (like functions-only) and not share the
  RDB when their requirements differ. This is currently just a flags `int`, but can be extended to
  a more complex structure with various filter fields.
* make sure to support filters only in diskless replication mode (not to override the persistence file),
  we do that by forcing diskless (even if disabled by config)

other changes:
* some refactoring in rdb.c (extract portion of a big function to a sub-function)
* rdb_key_save_delay used in AOFRW too
* sendChildInfo takes the number of updated keys (incremental, rather than absolute)

Co-authored-by: Oran Agra <oran@redislabs.com>
 2022-01-02 09:39:01 +02:00
sundb 888e92eb57 Fix a valgrind test failure due to slowly shutdown (#10038) 
This pr is mainly to solve the problem that redis process cannot be exited normally, due to changes in #10003.
When a test uses the `key-load-delay` config to delay loading, but does not reset it at the end of the test, will lead to server wait for the loading to reach the event
loop (once in 2mb) before actually shutting down.
 2022-01-01 17:45:13 +02:00
Madelyn Olson 39ced1afa1 Constrain error message to just node-id (#10022) 2021-12-31 09:32:33 -08:00
sundb 73951abe7b Fix when the master connection is disconnected, replication retry read indefinitely (#10032) 
Now if redis is still loading when we receive sigterm, we will wait for the loading to reach the event
loop (once in 2mb) before actually shutting down. See #10003.

This change caused valgrind CI to fail.
See https://github.com/redis/redis/runs/4662901673?check_suite_focus=true

This pr is mainly to solve the problem that redis process cannot be exited normally.
When the master is disconnected, if repl is processing diskless loading and using `connRead` to read data from master,
it may enter an infinite retry state, which does not handle `connRead` returning 0(master connection disconnected).
 2021-12-31 14:08:04 +02:00
Viktor Söderqvist e4b3a257ee Modules: Mark all APIs non-experimental (#9983) 
These exist for quite some time, and are no longer experimental
 2021-12-30 12:17:22 +02:00
Binbin 4836ae32c7 redis-cli: Add -X option and extend --cluster call take arg from stdin (#9980) 
There are two changes in this commit:

1. Add -X option to redis-cli.
Currently `-x` can only be used to provide the last argument,
so you can do `redis-cli dump keyname > key.dump`,
and then do `redis-cli -x restore keyname 0 < key.dump`.

But what if you want to add the replace argument (which comes last?).
oran suggested adding such usage:
`redis-cli -X <tag> restore keyname <tag> replace < key.dump`

i.e. you're able to provide a string in the arguments that's gonna be
substituted with the content from stdin.

Note that the tag name should not conflict with others non-replaced args.
And the -x and -X options are conflicting.

Some usages:
```
[root]# echo mypasswd | src/redis-cli -X passwd_tag mset username myname password passwd_tag                                                   OK
[root]# echo username > username.txt
[root]# head -c -1 username.txt | src/redis-cli -X name_tag mget name_tag password
1) "myname"
2) "mypasswd\n"
```

2. Handle the combination of both `-x` and `--cluster` or `-X` and `--cluster`
Extend the broadcast option to receive the last arg or <tag> arg from the stdin.

Now we can use `redis-cli -x --cluster call <host>:<port> cmd`,
or `redis-cli -X <tag> --cluster call <host>:<port> cmd <tag>`.
(support part of #9899)
 2021-12-30 12:10:04 +02:00
Viktor Söderqvist 5006eab552 Module API doc formatting from #9656 (#10026) 
Add blank before lists, so that they will be rendered as lists.

Commands RM_GetCommand and RM_CreateSubcommand were added in #9656.
 2021-12-30 11:34:41 +02:00
Itamar Haber aec8c57701 Add missing metadata to the commands SSOT files. (#10016) 
Add missing information about commands, mainly from reviewing redis-doc and removing
the metadata from it (https://github.com/redis/redis-doc/pull/1722)

* Reintroduces CLUSTER S****S (supported by Redis) but missing from the JSON / docs (related? #9675).
  Note that without that json file, the command won't work (breaking change)
* Adds the `replicas` argument (exists in Redis) to `CLIENT KILL`.
* Adds `history` entries to several commands based on redis-doc's man pages.
* Adds `since` to applicable command arguments based on `history` (this basically makes
  some of `history` redundant - perhaps at a later stage).
* Uses proper semantic versioning in all version references.
* Also removes `geoencodeCommand` and `geodecodeCommand` header
  declarations per b96af595a5.
 2021-12-29 21:57:40 +02:00
Ozan Tezcan b0c06e904a Fixed typo in test tag (for needs:debug) (#10021) 2021-12-28 16:23:02 +02:00
perryitay 43229e4f10 Safe and organized exit when receiving sigterm while loading (#10003) 
on the signal handler we are enabling server.shutdown_asap flag instead of just doing `exit()`,
and then catch it on the whileBlockedCron() where we prepare for shutdown correctly.

this is a more ore organized and safe termination, the old approach was missing these for example:
1. removal of the pidfile
2. shutdown event to modules
 2021-12-28 13:25:56 +02:00
guybe7 266d95066d Remove incomplete fix of a broader problem (#10013) 
Preventing COFIG SET maxmemory from propagating is just the tip of the iceberg.
Module that performs a write operation in a notification can cause any
command to be propagated, based on server.dirty

We need to come up with a better solution.
 2021-12-28 10:19:58 +02:00
David CARLIERandYossi Gottlieb 2c5573e894 Check somaxconn system settings on macOS, FreeBSD and OpenBSD. (#9972) 
Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
 2021-12-28 09:20:10 +02:00
chenyang8094andOran Agra af0b50f83a Tests: don't rely on the response of MEMORY USAGE when mem_allocator is not jemalloc (#10010) 
It turns out that libc malloc can return an allocation of a different size on requests of the same size.
this means that matching MEMORY USAGE of one key to another copy of the same data can fail.

Solution:
Keep running the test that calls MEMORY USAGE, but ignore the response.
We do that by introducing a new utility function to get the memory usage, which always returns 1
when the allocator is not jemalloc.

Other changes:
Some formatting for datatype2.tcl

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-12-27 21:37:21 +02:00
Itamar Haber f810510bb2 Adds utils/gen-commands-json.py (#9958) 
Following #9656, this script generates a "commands.json" file from the output
of the new COMMAND. The output of this script is used in redis/redis-doc#1714
and by redis/redis-io#259. This also converts a couple of rogue dashes (in 
'key-specs' and 'multiple-token' flags) to underscores (continues #9959).
 2021-12-27 19:31:13 +02:00
chenyang8094andOran Agra 317464a386 Fix failing test due to recent change in transaction propagation (#10006) 
PR #9890 may have introduced a problem.
There are tests that use MULTI-EXEC to make sure two BGSAVE / BGREWRITEAOF are executed together.
But now it's not valid to run run commands that create a snapshot inside a transaction (gonna be blocked soon)
This PR modifies the test not to rely on MULTI-EXEC.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-12-27 15:18:17 +02:00
guybe7 0f15e025e6 Fix race in propagation test (#10012) 
There's a race between testing DBSIZE and the thread starting.
If the thread hadn't started by the time we checked DBISZE, no
keys will have been evicted.
The correct way is to check the evicted_keys stat.
 2021-12-27 12:31:24 +02:00
Binbin e84ccc3f56 santize dump payload: fix carsh when zset with NAN score (#10002) 
`zslInsert` with a NAN score will crash the server.
This one found by the `corrupt-dump-fuzzer`.
 2021-12-26 11:40:11 +02:00
Meir Shpilraien (Spielrein) 365cbf46a7 Add FUNCTION DUMP and RESTORE. (#9938) 
Follow the conclusions to support Functions in redis cluster (#9899)

Added 2 new FUNCTION sub-commands:
1. `FUNCTION DUMP` - dump a binary payload representation of all the functions.
2. `FUNCTION RESTORE <PAYLOAD> [FLUSH|APPEND|REPLACE]` - give the binary payload extracted
   using `FUNCTION DUMP`, restore all the functions on the given payload. Restore policy can be given to
   control how to handle existing functions (default is APPEND):
   * FLUSH: delete all existing functions.
   * APPEND: appends the restored functions to the existing functions. On collision, abort.
   * REPLACE: appends the restored functions to the existing functions. On collision,
     replace the old function with the new function.

Modify `redis-cli --cluster add-node` to use `FUNCTION DUMP` to get existing functions from
one of the nodes in the cluster, and `FUNCTION RESTORE` to load the same set of functions
to the new node. `redis-cli` will execute this step before sending the `CLUSTER MEET` command
to the new node. If `FUNCTION DUMP` returns an error, assume the current Redis version do not
support functions and skip `FUNCTION RESTORE`. If `FUNCTION RESTORE` fails, abort and do not send
the `CLUSTER MEET` command. If the new node already contains functions (before the `FUNCTION RESTORE`
is sent), abort and do not add the node to the cluster. Test was added to verify
`redis-cli --cluster add-node` works as expected.
 2021-12-26 09:03:37 +02:00
Meir Shpilraien (Spielrein) 08ff606b0b Changed fuction name to be case insensitive. (#9984) 
Use case insensitive string comparison for function names (like we do for commands and configs)
In addition, add verification that the functions only use the following characters: [a-zA-Z0-9_]
 2021-12-26 08:37:24 +02:00
Yuta Hongo 63f606d3e3 redis-cli: Add OUTPUT_JSON format type (#9954) 
Introduce `redis-cli --json` option.
CSV doesn't support Map type, then parsing SLOWLOG or HMSET with multiple args are not helpful.

By default the `--json` implies RESP3, which makes it much more useful, and a `-2`
option was added to force RESP2.
When `HELLO 3` fails, it prints a warning message (which can be silenced with `-2`).
If a user passed `-3` explicitly, the non-interactive mode will also exit with error without
running the command, while in interactive session it'll keep running after printing the warning.

JSON output would be helpful to parse Redis replies with other tools like jq.

```
redis-cli --json slowlog get | jq

[
  [
    1,
    1639677545,
    322362,
    [
      "HMSET",
      "dummy-key",
      "field1",
      "123,456,789... (152 more bytes)",
      "field2",
      "111,222,333... (140 more bytes)",
      "field3",
      "... (349 more arguments)"
    ],
    "127.0.0.1:49312",
    ""
  ]
]

```
 2021-12-23 10:21:18 +02:00
guybe7 7ac213079c Sort out mess around propagation and MULTI/EXEC (#9890) 
The mess:
Some parts use alsoPropagate for late propagation, others using an immediate one (propagate()),
causing edge cases, ugly/hacky code, and the tendency for bugs

The basic idea is that all commands are propagated via alsoPropagate (i.e. added to a list) and the
top-most call() is responsible for going over that list and actually propagating them (and wrapping
them in MULTI/EXEC if there's more than one command). This is done in the new function,
propagatePendingCommands.

Callers to propagatePendingCommands:
1. top-most call() (we want all nested call()s to add to the also_propagate array and just the top-most
   one to propagate them) - via `afterCommand`
2. handleClientsBlockedOnKeys: it is out of call() context and it may propagate stuff - via `afterCommand`. 
3. handleClientsBlockedOnKeys edge case: if the looked-up key is already expired, we will propagate the
   expire but will not unblock any client so `afterCommand` isn't called. in that case, we have to propagate
   the deletion explicitly.
4. cron stuff: active-expire and eviction may also propagate stuff
5. modules: the module API allows to propagate stuff from just about anywhere (timers, keyspace notifications,
   threads). I could have tried to catch all the out-of-call-context places but it seemed easier to handle it in one
   place: when we free the context. in the spirit of what was done in call(), only the top-most freeing of a module
   context may cause propagation.
6. modules: when using a thread-safe ctx it's not clear when/if the ctx will be freed. we do know that the module
   must lock the GIL before calling RM_Replicate/RM_Call so we propagate the pending commands when
   releasing the GIL.

A "known limitation", which were actually a bug, was fixed because of this commit (see propagate.tcl):
   When using a mix of RM_Call with `!` and RM_Replicate, the command would propagate out-of-order:
   first all the commands from RM_Call, and then the ones from RM_Replicate

Another thing worth mentioning is that if, in the past, a client would issue a MULTI/EXEC with just one
write command the server would blindly propagate the MULTI/EXEC too, even though it's redundant.
not anymore.

This commit renames propagate() to propagateNow() in order to cause conflicts in pending PRs.
propagatePendingCommands is the only caller of propagateNow, which is now a static, internal helper function.

Optimizations:
1. alsoPropagate will not add stuff to also_propagate if there's no AOF and replicas
2. alsoPropagate reallocs also_propagagte exponentially, to save calls to memmove

Bugfixes:
1. CONFIG SET can create evictions, sending notifications which can cause to dirty++ with modules.
   we need to prevent it from propagating to AOF/replicas
2. We need to set current_client in RM_Call. buggy scenario:
   - CONFIG SET maxmemory, eviction notifications, module hook calls RM_Call
   - assertion in lookupKey crashes, because current_client has CONFIG SET, which isn't CMD_WRITE
3. minor: in eviction, call propagateDeletion after notification, like active-expire and all commands
   (we always send a notification before propagating the command)
 2021-12-23 00:03:48 +02:00
Oran Agra b7567394e1 resolve replication test timing sensitivity - 2nd attempt (#9988) 
issue started failing after #9878 was merged (made an exiting test more sensitive)
looks like #9982 didn't help, tested this one and it seems to work better.

this commit does two things:
1. reduce the extra delay i added earlier and instead add more keys, the effect no duration
   of replication is the same, but the intervals in which the server is responsive to the tcl client is higher.
2. improve the test infra to print context when assert_error fails.
 2021-12-22 23:37:12 +02:00
Oran Agra e33e0295bb resolve replication test timing sensitivity (#9982) 
issue started failing after #9878 was merged (made an exiting test more sensitive)
 2021-12-22 16:05:53 +02:00
Oran Agra 41e6e05dee Allow most CONFIG SET during loading, block some commands in async-loading (#9878) 
## background
Till now CONFIG SET was blocked during loading.
(In the not so distant past, GET was disallowed too)

We recently (not released yet) added an async-loading mode, see #9323,
and during that time it'll serve CONFIG SET and any other command.
And now we realized (#9770) that some configs, and commands are dangerous
during async-loading.

## changes
* Allow most CONFIG SET during loading (both on async-loading and normal loading)
* Allow CONFIG REWRITE and CONFIG RESETSTAT during loading
* Block a few config during loading (`appendonly`, `repl-diskless-load`, and `dir`)
* Block a few commands during loading (list below)

## the blocked commands:
* SAVE - obviously we don't wanna start a foregreound save during loading 8-)
* BGSAVE - we don't mind to schedule one, but we don't wanna fork now
* BGREWRITEAOF - we don't mind to schedule one, but we don't wanna fork now
* MODULE - we obviously don't wanna unload a module during replication / rdb loading
  (MODULE HELP and MODULE LIST are not blocked)
* SYNC / PSYNC - we're in the middle of RDB loading from master, must not allow sync
  requests now.
* REPLICAOF / SLAVEOF - we're in the middle of replicating, maybe it makes sense to let
  the user abort it, but he couldn't do that so far, i don't wanna take any risk of bugs due to odd state.
* CLUSTER - only allow [HELP, SLOTS, NODES, INFO, MYID, LINKS, KEYSLOT, COUNTKEYSINSLOT,
  GETKEYSINSLOT, RESET, REPLICAS, COUNT_FAILURE_REPORTS], for others, preserve the status quo

## other fixes
* processEventsWhileBlocked had an issue when being nested, this could happen with a busy script
  during async loading (new), but also in a busy script during AOF loading (old). this lead to a crash in
  the scenario described in #6988
 2021-12-22 14:11:16 +02:00
zhugezy ad55fbaabb Shorten timeouts of CLIENT PAUSE to avoid hanging when tests fail. (#9975) 
If a test fails at `wait_for_blocked_clients_count` after the `PAUSE` command,
It won't send `UNPAUSE` to server, leading to the server hanging until timeout,
which is bad and hard to debug sometimes when developing.
This PR tries to fix this.

Timeout in `CLIENT PAUSE` shortened from 1e5 seconds(extremely long) to 50~100 seconds.
 2021-12-22 12:06:29 +02:00
Hongcai Ren b28dbef59d There is mismach between function sdssplitlen() comments and implementation (#4909) 
when count is 0, return NULL
 2021-12-22 10:00:21 +02:00
Meir Shpilraien (Spielrein) 3bcf108416 Change FUNCTION CREATE, DELETE and FLUSH to be WRITE commands instead of MAY_REPLICATE. (#9953) 
The issue with MAY_REPLICATE is that all automatic mechanisms to handle
write commands will not work. This require have a special treatment for:
* Not allow those commands to be executed on RO replica.
* Allow those commands to be executed on RO replica from primary connection.
* Allow those commands to be executed on the RO replica from AOF.

By setting those commands as WRITE commands we are getting all those properties from Redis.
Test was added to verify that those properties work as expected.

In addition, rearrange when and where functions are flushed. Before this PR functions were
flushed manually on `rdbLoadRio` and cleaned manually on failure. This contradicts the
assumptions that functions are data and need to be created/deleted alongside with the
data. A side effect of this, for example, `debug reload noflush` did not flush the data but
did flush the functions, `debug loadaof` flush the data but not the functions.
This PR move functions deletion into `emptyDb`. `emptyDb` (renamed to `emptyData`) will
now accept an additional flag, `NOFUNCTIONS` which specifically indicate that we do not
want to flush the functions (on all other cases, functions will be flushed). Used the new flag
on FLUSHALL and FLUSHDB only! Tests were added to `debug reload` and `debug loadaof`
to verify that functions behave the same as the data.

Notice that because now functions will be deleted along side with the data we can not allow
`CLUSTER RESET` to be called from within a function (it will cause the function to be released
while running), this PR adds `NO_SCRIPT` flag to `CLUSTER RESET`  so it will not be possible
to be called from within a function. The other cluster commands are allowed from within a
function (there are use-cases that uses `GETKEYSINSLOT` to iterate over all the keys on a
given slot). Tests was added to verify `CLUSTER RESET` is denied from within a script.

Another small change on this PR is that `RDBFLAGS_ALLOW_DUP` is also applicable on functions.
When loading functions, if this flag is set, we will replace old functions with new ones on collisions.
 2021-12-21 16:13:29 +02:00
yancz2000 23b6173486 Add --version support for redis-check-rdb (#9971) 
Let's you see which version of redis this tool is part of (similarly to redis-cli and redis-benchmark)
 2021-12-21 14:11:18 +02:00
zhugezyandOran Agra 1b0968df46 Remove EVAL script verbatim replication, propagation, and deterministic execution logic (#9812) 
# Background

The main goal of this PR is to remove relevant logics on Lua script verbatim replication,
only keeping effects replication logic, which has been set as default since Redis 5.0.
As a result, Lua in Redis 7.0 would be acting the same as Redis 6.0 with default
configuration from users' point of view.

There are lots of reasons to remove verbatim replication.
Antirez has listed some of the benefits in Issue #5292:

>1. No longer need to explain to users side effects into scripts.
    They can do whatever they want.
>2. No need for a cache about scripts that we sent or not to the slaves.
>3. No need to sort the output of certain commands inside scripts
    (SMEMBERS and others): this both simplifies and gains speed.
>4. No need to store scripts inside the RDB file in order to startup correctly.
>5. No problems about evicting keys during the script execution.

When looking back at Redis 5.0, antirez and core team decided to set the config
`lua-replicate-commands yes` by default instead of removing verbatim replication
directly, in case some bad situations happened. 3 years later now before Redis 7.0,
it's time to remove it formally.

# Changes

- configuration for lua-replicate-commands removed
  - created config file stub for backward compatibility
- Replication script cache removed
  - this is useless under script effects replication
  - relevant statistics also removed
- script persistence in RDB files is also removed
- Propagation of SCRIPT LOAD and SCRIPT FLUSH to replica / AOF removed
- Deterministic execution logic in scripts removed (i.e. don't run write commands
  after random ones, and sorting output of commands with random order)
  - the flags indicating which commands have non-deterministic results are kept as hints to clients.
- `redis.replicate_commands()` & `redis.set_repl()` changed
  - now `redis.replicate_commands()` does nothing and return an 1
  - ...and then `redis.set_repl()` can be issued before `redis.replicate_commands()` now
- Relevant TCL cases adjusted
- DEBUG lua-always-replicate-commands removed

# Other changes
- Fix a recent bug comparing CLIENT_ID_AOF to original_client->flags instead of id. (introduced in #9780)

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-12-21 08:32:42 +02:00
Binbin febc3f63b2 Fix recent daily CI test failures (#9966) 
Recent PRs have introduced some failures, this commit
try to fix these CI failures. Here are the changes:

1. Enable debug-command in sentinel test.
```
Master reboot in very short time: ERR DEBUG command not allowed. If the
enable-debug-command option is set to "local", you can run it from a
local connection, otherwise you need to set this option in the
configuration file, and then restart the server.
```

2. Enable protected-config in sentinel test.
```
SDOWN is triggered by misconfigured instance replying with errors: ERR
CONFIG SET failed (possibly related to argument 'dir') - can't set
protected config
```

3. Enable debug-command in cluster test.
```
Verify slaves consistency: ERR DEBUG command not allowed. If the
enable-debug-command option is set to "local", you can run it from a
local connection, otherwise you need to set this option in the
configuration file, and then restart the server.
```

4. quicklist fill should be signed int.
The reason for the modification is to eliminate the warning.
Modify `int fill: QL_FILL_BITS` to `signed int fill: QL_FILL_BITS`

The first three were introduced at #9920 (same issue).
And the last one was introduced at #9962.
 2021-12-20 12:31:13 +02:00
Binbin 0fb1aa0645 Fix some nonsense came from LGTM (#9962) 
1. Local variable 's' hides a parameter of the same name.
```
int anetTcpAccept(char *err, int s, char *ip, size_t ip_len, int *port) {
    if ((fd = anetGenericAccept(err,s,(struct sockaddr*)&sa,&salen)) == ANET_ERR){}
}
```
Change the parameter name from `s` to `serversock`,
also unified with the header file definition.

2. Comparison is always false because i <= 48.
```
for (i = 0; i < DICT_STATS_VECTLEN-1; i++) {  // i < 49
    (i == DICT_STATS_VECTLEN-1)?">= ":"",  // i == 49
}
```
`i == DICT_STATS_VECTLEN-1` always result false, it is a dead code.

3. Empty block without comment.
`else if (!strcasecmp(opt,"ch")) {}`, add a comment to avoid warnings.

4. Bit field fill of type int should have explicitly unsigned integral, explicitly signed integral, or enumeration type.
Modify `int fill: QL_FILL_BITS;` to `unsigned int fill: QL_FILL_BITS;`

5. The result of this call to reconnectingRedisCommand is not checked for null, but 80% of calls to reconnectingRedisCommand check for null.
Just a cleanup job like others.
 2021-12-19 17:52:23 +02:00
Oran Agra 6add1b7217 Add external test that runs without debug command (#9964) 
- add needs:debug flag for some tests
- disable "save" in external tests (speedup?)
- use debug_digest proc instead of debug command directly so it can be skipped
- use OBJECT ENCODING instead of DEBUG OBJECT to get encoding
- add a proc for OBJECT REFCOUNT so it can be skipped
- move a bunch of tests in latency_monitor tests to happen later so that latency monitor has some values in it
- add missing close_replication_stream calls
- make sure to close the temp client if DEBUG LOG fails
 2021-12-19 17:41:51 +02:00
YaacovHazan ae2f5b7b2e Protected configs and sensitive commands (#9920) 
Block sensitive configs and commands by default.

* `enable-protected-configs` - block modification of configs with the new `PROTECTED_CONFIG` flag.
   Currently we add this flag to `dbfilename`, and `dir` configs,
   all of which are non-mutable configs that can set a file redis will write to.
* `enable-debug-command` - block the `DEBUG` command
* `enable-module-command` - block the `MODULE` command

These have a default value set to `no`, so that these features are not
exposed by default to client connections, and can only be set by modifying the config file.

Users can change each of these to either `yes` (allow all access), or `local` (allow access from
local TCP connections and unix domain connections)

Note that this is a **breaking change** (specifically the part about MODULE command being disabled by default).
I.e. we don't consider DEBUG command being blocked as an issue (people shouldn't have been using it),
and the few configs we protected are unlikely to have been set at runtime anyway.
On the other hand, it's likely to assume some users who use modules, load them from the config file anyway.
Note that's the whole point of this PR, for redis to be more secure by default and reduce the attack surface on
innocent users, so secure defaults will necessarily mean a breaking change.
 2021-12-19 10:46:16 +02:00
guybe7 5df070ba39 COMMAND: Use underscores instead of hyphens in attributes (#9959) 
some languages can build a json-like object by parsing a textual json,
but it works poorly when attributes contain hyphens

example in JS:
```
let j = JSON.parse(json)
j['key-name'] <- works
j.key-name <= illegal syntax
```
 2021-12-18 09:00:42 +02:00
ny0312 792afb4432 Introduce memory management on cluster link buffers (#9774) 
Introduce memory management on cluster link buffers:
 * Introduce a new `cluster-link-sendbuf-limit` config that caps memory usage of cluster bus link send buffers.
 * Introduce a new `CLUSTER LINKS` command that displays current TCP links to/from peers.
 * Introduce a new `mem_cluster_links` field under `INFO` command output, which displays the overall memory usage by all current cluster links.
 * Introduce a new `total_cluster_links_buffer_limit_exceeded` field under `CLUSTER INFO` command output, which displays the accumulated count of cluster links freed due to `cluster-link-sendbuf-limit`.
 2021-12-16 21:56:59 -08:00
Meir Shpilraien (Spielrein) 687210f155 Add FUNCTION FLUSH command to flush all functions (#9936) 
Added `FUNCTION FLUSH` command. The new sub-command allows delete all the functions.
An optional `[SYNC|ASYNC]` argument can be given to control whether or not to flush the
functions synchronously or asynchronously. if not given the default flush mode is chosen by
`lazyfree-lazy-user-flush` configuration values.

Add the missing `functions.tcl` test to the list of tests that are executed in test_helper.tcl,
and call FUNCTION FLUSH in between servers in external mode
 2021-12-16 17:58:25 +02:00
guybe7andOran Agra ffbe36fc3e Command table: Sorted subcommands (#9951) 
Sort the sub-commands so that every time we execute the script it generates the exact same results.
This will case less merge conflicts if two PRs edit different json files.

also:
* make the script agnostic to where it is executed (more flexible).
* add documentation about commands.c and the json files in the readme.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-12-16 12:54:40 +02:00
yoav-steinberg 70ff26b454 Multiparam config get. (#9914) 
Support doing `CONFIG GET <x> <y> <z>`, each of them can also be
a pattern with wildcards.

This avoids duplicates in the result by looping over the configs and for
each once checking all the patterns, once a match is found for a pattern
we move on to the next config.
 2021-12-16 09:01:13 +02:00
ranshidandMadelyn Olson 28b5a6537d Throw error on too long unix domain socket file path (#9826) 
* Fix too long unix domain socket file path

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
 2021-12-15 21:38:45 -08:00
guybe7andItamar Haber 867816003e Auto-generate the command table from JSON files (#9656) 
Delete the hardcoded command table and replace it with an auto-generated table, based
on a JSON file that describes the commands (each command must have a JSON file).

These JSON files are the SSOT of everything there is to know about Redis commands,
and it is reflected fully in COMMAND INFO.

These JSON files are used to generate commands.c (using a python script), which is then
committed to the repo and compiled.

The purpose is:
* Clients and proxies will be able to get much more info from redis, instead of relying on hard coded logic.
* drop the dependency between Redis-user and the commands.json in redis-doc.
* delete help.h and have redis-cli learn everything it needs to know just by issuing COMMAND (will be
  done in a separate PR)
* redis.io should stop using commands.json and learn everything from Redis (ultimately one of the release
  artifacts should be a large JSON, containing all the information about all of the commands, which will be
  generated from COMMAND's reply)
* the byproduct of this is:
  * module commands will be able to provide that info and possibly be more of a first-class citizens
  * in theory, one may be able to generate a redis client library for a strictly typed language, by using this info.

### Interface changes

#### COMMAND INFO's reply change (and arg-less COMMAND)

Before this commit the reply at index 7 contained the key-specs list
and reply at index 8 contained the sub-commands list (Both unreleased).
Now, reply at index 7 is a map of:
- summary - short command description
- since - debut version
- group - command group
- complexity - complexity string
- doc-flags - flags used for documentation (e.g. "deprecated")
- deprecated-since - if deprecated, from which version?
- replaced-by - if deprecated, which command replaced it?
- history - a list of (version, what-changed) tuples
- hints - a list of strings, meant to provide hints for clients/proxies. see https://github.com/redis/redis/issues/9876
- arguments - an array of arguments. each element is a map, with the possibility of nesting (sub-arguments)
- key-specs - an array of keys specs (already in unstable, just changed location)
- subcommands - a list of sub-commands (already in unstable, just changed location)
- reply-schema - will be added in the future (see https://github.com/redis/redis/issues/9845)

more details on these can be found in https://github.com/redis/redis-doc/pull/1697

only the first three fields are mandatory 

#### API changes (unreleased API obviously)

now they take RedisModuleCommand opaque pointer instead of looking up the command by name

- RM_CreateSubcommand
- RM_AddCommandKeySpec
- RM_SetCommandKeySpecBeginSearchIndex
- RM_SetCommandKeySpecBeginSearchKeyword
- RM_SetCommandKeySpecFindKeysRange
- RM_SetCommandKeySpecFindKeysKeynum

Currently, we did not add module API to provide additional information about their commands because
we couldn't agree on how the API should look like, see https://github.com/redis/redis/issues/9944.

### Somehow related changes
1. Literals should be in uppercase while placeholder in lowercase. Now all the GEO* command
   will be documented with M|KM|FT|MI and can take both lowercase and uppercase

### Unrelated changes
1. Bugfix: no_madaory_keys was absent in COMMAND's reply
2. expose CMD_MODULE as "module" via COMMAND
3. have a dedicated uint64 for ACL categories (instead of having them in the same uint64 as command flags)

Co-authored-by: Itamar Haber <itamar@garantiadata.com>
 2021-12-15 21:23:15 +02:00
Meir Shpilraien (Spielrein) fbfdf513d2 Use server.dirty++ instead if forceCommandPropagation on FUNCTION commands (#9945) 
Functions are considered data, so changing a function should be counted
as a data change and should affect the persistence policy. For example
if we want to persist an RDB each minute if there was a single change,
functions should be counted as such change. Using `forceCommandPropagation`
will not give us the desired effect and so we must switch to `server.dirty++`
 2021-12-15 19:17:10 +02:00
丽媛自己动 95f943add6 in line 3749 resetServerSaveParams will set the param to null,so no need (#9943) 
to do here
 2021-12-15 16:32:14 +02:00
Wen Hui a09bc5045b Error message improvement for CONFIG SET command (#9924) 
When CONFIG SET fails, print the name of the config that failed.
This is helpful since config set is now variadic.

however, there are cases where several configs have the same apply
function, and we can't be sure which one of them caused the failure.
 2021-12-15 09:46:32 +02:00
yoav-steinberg c7dc17fc0f Fix possible int overflow when hashing an sds. (#9916) 
This caused a crash when adding elements larger than 2GB to a set (same goes for hash keys). See #8455.

Details:
* The fix makes the dict hash functions receive a `size_t` instead of an `int`. In practice the dict hash functions
  call siphash which receives a `size_t` and the callers to the hash function pass a `size_t` to it so the fix is trivial.
* The issue was recreated by attempting to add a >2gb value to a set. Appropriate tests were added where I create
  a set with large elements and check basic functionality on it (SADD, SCARD, SPOP, etc...).
* When I added the tests I also refactored a bit all the tests code which is run under the `--large-memory` flag.
  This removed code duplication for the test framework's `write_big_bulk` and `write_big_bulk` code and also takes
  care of not allocating the test frameworks helper huge string used by these tests when not run under `--large-memory`.
* I also added the _violoations.tcl_ unit tests to be part of the entire test suite and leaned up non relevant list related
  tests that were in there. This was done in this PR because most of the _violations_ tests are "large memory" tests.
 2021-12-13 21:16:25 +02:00
Madelyn Olson c40d23b89f Redact ACL SETUSER arguments if the user has spaces (#9935) 2021-12-13 08:39:04 -08:00
Binbin b93ccee451 Fix timing issue in strem blocking tests (#9927) 
A test failure was reported in Daily CI (FreeBSD).
`XREAD: XADD + DEL should not awake client`

```
*** [err]: XREAD: XADD + DEL should not awake client in tests/unit/type/stream.tcl
Expected [lindex  0 0] eq {s1} (context: type eval line 11 cmd {assert {[lindex $res 0 0] eq {s1}}} proc ::test)
```

It seems that `r` is executed before `rd` enters the blocking
state. And ended up getting a empty reply by timeout.

We use `wait_for_blocked_clients_count` to wait for the
blocking client to be ready and avoid this situation.
Also fixed other test cases that may have the same issue.
 2021-12-10 20:35:51 +02:00
Wen Hui 5b351d073f Fix the server log when client use Multiparam Config Set (#9919) 
When clients enter multiply parameters in the CONFIG SET command,
the server side log displays the wrong config parameters
 2021-12-09 16:37:05 +02:00
sundb 7f0fae947a Santize dump payload: fix crash when stream with duplicate consumes (#9918) 
When rdb creates a consumer without determining whether it exists in advance,
it may return NULL and crash if it encounters corrupt data with duplicate consumers.
 2021-12-08 18:11:57 +02:00
yoav-steinberg 07b1326073 Hide hidden configs from config get patterns. (#9888) 
Added `HIDDEN_CONFIG` to hide debug / dev / testing configs from CONFIG GET
when it is used with a wildcard.
These are not documented in redis.conf so now CONFIG GET only works when they
are explicitly specified.

The current configs are: 
```
key-load-delay
loading-process-events-interval-bytes
rdb-key-save-delay
use-exit-on-panic
watchdog-period
```
 2021-12-08 12:44:10 +02:00
Binbin f0373f720f Fix outdated protected-mode documentation in sentinel.conf (#9896) 
In 666b343, we modified the default value of protected-mode
from yes to no.

However, this change is not mentioned in sentinel.conf.
Looking at the sentinel.conf alone, it is easy to make
people think that in sentinel mode, we hava truned on
the protected-mode.
 2021-12-08 11:25:56 +02:00
Binbin a7726cdf51 Fix SENTINEL subcommands's arity (#9909) 
For `SENTINEL SET`, we can use in these ways:
1. SENTINEL SET mymaster quorum 3
2. SENTINEL SET mymaster quorum 5 parallel-syncs 1

For `SENTINEL SIMULATE-FAILURE`, although it is only used for testing:
1. SENTINEL SIMULATE-FAILURE CRASH-AFTER-ELECTION
2. SENTINEL SIMULATE-FAILURE CRASH-AFTER-ELECTION CRASH-AFTER-PROMOTION
 2021-12-08 08:59:02 +02:00
leishiaoandxiaolei 08ed44d722 improvement of a test in unit/pause.tcl (#9868) 
Co-authored-by: xiaolei <xiaolei@91jkys.com>
 2021-12-07 17:41:11 -08:00
丽媛自己动 0b26e389d8 Update documentation of _dictReset (#9908) 2021-12-07 11:14:18 -08:00
yoav-steinberg 1736fa4d22 Don't write oom score adj to proc unless we're managing it. (#9904) 
When disabling redis oom-score-adj managment we restore the
base value read before enabling oom-score-adj management.

This fixes an issue introduced in #9748 where updating
`oom-score-adj-values` while `oom-score-adj` was set to `no`
would write the base oom score adj value read on startup to `/proc`.
This is a bug since while `oom-score-adj` is disabled we should
never write to proc and let external processes manage it.

Added appropriate tests.
 2021-12-07 16:05:51 +02:00
Binbin b947049f85 Fix timing issue in logging.tcl with FreeBSD (#9910) 
A test failure was reported in Daily CI.
`Crash report generated on SIGABRT` with FreeBSD.

```
*** [err]: Crash report generated on SIGABRT in tests/integration/logging.tcl
Expected [string match *crashed by signal* ### Starting...(logs) in tests/integration/logging.tcl]
```

It look like `tail -1000` was executed too early, before it
printed out all the crash logs. We can give it a few more
chances by using `wait_for_log_messages`.

Other changes:
1. In `Server is able to generate a stack trace on selected systems`,
use `wait_for_log_messages`to reduce the lines of code. And if it
fails, there are more detailed logs that can be printed.

2. In `Crash report generated on DEBUG SEGFAULT`, we also use
`wait_for_log_messages` to avoid possible timing issues.
 2021-12-07 12:02:58 +02:00
Madelyn Olson 36ca545286 Fix spelling of sanitization (#9901) 2021-12-06 10:14:13 +02:00
sundb 1808618f5d Santize dump payload: fix invalid listpack entry start with EOF (#9889) 
When an invalid listpack entry starts with EOF, we will skip it when we verify it in the loop.
 2021-12-04 16:43:08 +02:00
Oran Agra 64f6159646 Merge Redis Functions PR (#9780) 
# Redis Function
This PR added the Redis Functions capabilities that were suggested on #8693.
The PR also introduce a big refactoring to the current Lua implementation
(i.e `scripting.c`). The main purpose of the refactoring is to have better
code sharing between the Lua implementation that exists today on Redis
(`scripting.c`) and the new Lua engine that is introduced on this PR.
The refactoring includes code movements and file name changes as well as some
logic changes that need to be carefully reviewed. To make the review easier,
the PR was split into multiple commits. Each commit is deeply described later on
but the main concept is that some commits are just moving code around without
making any logical changes, those commits are less likely to cause any issues
or regressions and can be reviewed fast. Other commits, which perform code and
logic changes, need to be reviewed carefully, but those commits were created
after the code movements so it's pretty easy to see what was changed. To sum up,
it is highly recommended to review this PR commit by commit as it will be easier
to see the changes, it is also recommended to read each commit description
(written below) to understand what was changed on the commit and whether or not
it's just a huge code movement or a logic changes.

## Terminology
Currently, the terminology in Redis is not clearly defined. Scripts refer to Lua
scripts and eval also refers only to Lua. Introducing Redis Function requires
redefining those terms to be able to clearly understand what is been discussed
on each context.
* eval - legacy Lua script implementation.
* Function - new scripting implementation (currently implemented in Lua but in
  the future, it might be other languages like javascript).
* Engine - the component that is responsible for executing functions.
* Script - Function or legacy Lua (executed with `eval` or `evalsha`)

## Refactoring New Structure
Today, the entire scripting logic is located on `scripting.c`. This logic can
be split into 3 main groups:
1. Script management - responsible for storing the scripts that were sent to
   Redis and retrieving them when they need to be run (base on the script sha
   on the current implementation).
2. Script invocation - invoke the script given on `eval` or `evalsha` command
   (this part includes finding the relevant script, preparing the arguments, ..)
3. Interact back with Redis (command invocation)

Those 3 groups are tightly coupled on `scripting.c`. Redis Functions also need
to use those groups logics, for example,  to interact back with Redis or to
execute Lua code. The refactoring attempts to split those 3 groups and define
APIs so that we can reuse the code both on legacy Lua scripts and Redis Functions.

In order to do so we define the following units:
1. script.c: responsible for interaction with Redis from within a script.
2. script_lua.c: responsible to execute Lua code, uses `script.c` to interact
   with Redis from within the Lua code.
3. function_lua.c: contains the Lua engine implementation, uses `script_lua.c`
   to execute the Lua code.
4. functions.c: Contains Redis Functions implementation (`FUNCTION` command,),
   uses `functions_lua.c` if the function it wants to invoke needs the Lua
   engine.
4. eval.c: the original `scripting.c` contains the Lua legacy implementation and
   was refactored to use `script_lua.c` to invoke the Lua code.

## Commits breakdown
Notice: Some small commits are omitted from this list as they are small and
insignificant (for example build fixes)

### First commit - code movements
This commit rename `scripting.c` -> `eval.c` and introduce the new `script_lua.c`
unit. The commit moves relevant code from `eval.c` (`scripting.c`) to
`script_lua.c`, the purpose of moving the code is so that later we will be able
to re-use the code on the Lua engine (`function_lua.c`). The commit only moves
the code without modifying even a single line, so there is a very low risk of
breaking anything and it also makes it much easier to see the changes on the
following commits.
Because the commit does not change the code (only moves it), it does not compile.
But we do not care about it as the only purpose here is to make the review
processes simpler.

### Second commit - move legacy Lua variables into `eval.c`
Today, all Lua-related variables are located on the server struct. The commit
attempt to identify those variable and take them out from the server struct,
leaving only script related variables (variables that later need to be used
also by engines)
The following variable where renamed and left on the server struct:
   * lua_caller 			-> script_caller
   * lua_time_limit 		-> script_time_limit
   * lua_timedout 		-> script_timedout
   * lua_oom 			-> script_oom
   * lua_disable_deny_script 	-> script_disable_deny_script
   * in_eval			-> in_script

The following variables where moved to lctx under eval.c
   * lua
   * lua_client
   * lua_cur_script
   * lua_scripts
   * lua_scripts_mem
   * lua_replicate_commands
   * lua_write_dirty
   * lua_random_dirty
   * lua_multi_emitted
   * lua_repl
   * lua_kill
   * lua_time_start
   * lua_time_snapshot

This commit is in a low risk of introducing any issues and it is just moving
variables around and not changing any logic.

### Third commit - introducing script unit
This commit introduces the `script.c` unit. Its purpose (as described above) is
to provide an API for scripts to interact with Redis. Interaction includes
mostly executing commands, but also other functionalities. The interaction is
done using a `ScriptRunCtx` object that needs to be created by the user and
initialized using `scriptPrepareForRun`. A detailed list of functionalities
expose by the unit:
1. Calling commands (including all the validation checks such as
   acl, cluster, read only run, ...)
2. Set Resp
3. Set Replication method (AOF/REPLICATION/NONE)
4. Call Redis back on long-running scripts to allow Redis to reply to clients
   and perform script kill

The commit introduces the new unit and uses it on eval commands to interact with
Redis.

### Fourth commit - Moved functionality of invoke Lua code to `script_lua.c`
This commit moves the logic of invoking the Lua code into `script_lua.c` so
later it can be used also by Lua engine (`function_lua.c`). The code is located
on `callFunction` function and assumes the Lua function already located on the
top of the Lua stack. This commit also change `eval.c` to use the new
functionality to invoke Lua code.

### Fith commit - Added Redis Functions unit (`functions.c`) and Lua engine
(`function_lua.c`)
Added Redis Functions unit under `functions.c`, included:
1. FUNCTION command:
     * FUNCTION CREATE
     * FUNCTION CALL
     * FUNCTION DELETE
     * FUNCTION KILL
     * FUNCTION INFO
     * FUNCTION STATS
2. Register engines

In addition, this commit introduces the first engine that uses the Redis
Functions capabilities, the Lua engine (`function_lua.c`)

## API Changes
### `lua-time-limit`
configuration was renamed to `script-time-limit` (keep `lua-time-limit` as alias
for backward compatibility).

### Error log changes
When integrating with Redis from within a Lua script, the `Lua` term was removed
from all the error messages and instead we write only `script`. For example:
`Wrong number of args calling Redis command From Lua script` -> `Wrong number
of args calling Redis command From script`

### `info memory` changes:
Before stating all the changes made to memory stats we will try to explain the
reason behind them and what we want to see on those metrics:
* memory metrics should show both totals (for all scripting frameworks), as well
  as a breakdown per framework / vm.
* The totals metrics should have "human" metrics while the breakdown shouldn't.
* We did try to maintain backward compatibility in some way, that said we did
  make some repurpose to existing metrics where it looks reasonable.
* We separate between memory used by the script framework (part of redis's
  used_memory), and memory used by the VM (not part of redis's used_memory)

A full breakdown of `info memory` changes:
* `used_memory_lua` and `used_memory_lua_human` was deprecated,
  `used_memory_vm_eval` has the same meaning as `used_memory_lua`
* `used_memory_scripts` was renamed to `used_memory_scripts_eval`
* `used_memory_scripts` and `used_memory_scripts_human` were repurposed and now
  return the total memory used by functions and eval (not including vm memory,
  only code cache, and structs).
* `used_memory_vm_function` was added and represents the total memory used by
  functions vm's
* `used_memory_functions` was added and represents the total memory by functions
  (not including vm memory, only code cache, and structs)
* `used_memory_vm_total` and `used_memory_vm_total_human` was added and
  represents the total memory used by vm's (functions and eval combined)

### `functions.caches`
`functions.caches` field was added to `memory stats`, representing the memory
used by engines that are not functions (this memory includes data structures
like dictionaries, arrays, ...)

## New API
### FUNCTION CREATE

Usage: FUNCTION CREATE `ENGINE` `NAME` `[REPLACE]` `[DESC <DESCRIPTION>]` `<CODE>`

* `ENGINE` - The name of the engine to use to create the script.
* `NAME` - the name of the function that can be used later to call the function
  using `FUNCTION CALL` command.
* `REPLACE` - if given, replace the given function with the existing function
  (if exists).
* `DESCRIPTION` - optional argument describing the function and what it does
* `CODE` - function code.

The command will return `OK` if created successfully or error in the following
cases:
* The given engine name does not exist
* The function name is already taken and `REPLACE` was not used.
* The given function failed on the compilation.

### FCALL and FCALL_RO

Usage: FCALL/FCALL_RO `NAME` `NUM_KEYS key1 key2` … ` arg1 arg2`

Call and execute the function specified by `NAME`. The function will receive
all arguments given after `NUM_KEYS`. The return value from the function will
be returned to the user as a result.

* `NAME` - Name of the function to run.
* The rest is as today with EVALSHA command.

The command will return an error in the following cases:
* `NAME` does not exist
* The function itself returned an error.

The `FCALL_RO` is equivalent to `EVAL_RO` and allows only read-only commands to
be invoked from the script.

### FUNCTION DELETE

Usage: FUNCTION DELETE `NAME`

Delete a function identified by `NAME`. Return `OK` on success or error on one
of the following:
* The given function does not exist

### FUNCTION INFO

Usage: FUNCTION INFO `NAME` [WITHCODE]

Return information about a function by function name:
* Function name
* Engine name
* Description
* Raw code (only if WITHCODE argument is given)

### FUNCTION LIST

Usage: FUNCTION LIST

Return general information about all the functions:
* Function name
* Engine name
* Description

### FUNCTION STATS

Usage: FUNCTION STATS

Return information about the current running function:
* Function name
* Command that was used to invoke the function
* Duration in MS that the function is already running

If no function is currently running, this section is just a RESP nil.

Additionally, return a list of all the available engines.

### FUNCTION KILL

Usage: `FUNCTION KILL`

Kill the currently executing function. The command will fail if the function
already initiated a write command.

## Notes
Note: Function creation/deletion is replicated to AOF but AOFRW is not
implemented sense its going to be removed: #9794
 2021-12-02 21:41:58 +02:00
meir@redislabs.com cbd463175f Redis Functions - Added redis function unit and Lua engine 
Redis function unit is located inside functions.c
and contains Redis Function implementation:
1. FUNCTION commands:
  * FUNCTION CREATE
  * FCALL
  * FCALL_RO
  * FUNCTION DELETE
  * FUNCTION KILL
  * FUNCTION INFO
2. Register engine

In addition, this commit introduce the first engine
that uses the Redis Function capabilities, the
Lua engine.
 2021-12-02 19:35:52 +02:00
Binbin e57a4db5d7 Fix CONFIG SET test failures in MacOS/FreeBSD (#9881) 
After the introduction of `Multiparam config set` in #9748,
there are two tests cases failed.

```
[exception]: Executing test client: ERR Config set failed - Failed to set current oom_score_adj. Check server logs..
ERR Config set failed - Failed to set current oom_score_adj. Check server logs.
```

`CONFIG sanity` test failed on the `config set oom-score-adj-values`
which is a "special" config that does not catch no-op changes.
And then it will update `oom-score-adj` which not supported in
MacOs. We solve it by adding `oom-score*` to the `skip_configs` list.

```
*** [err]: CONFIG SET rollback on apply error in tests/unit/introspection.tcl
Expected an error but nothing was caught
```

`CONFIG SET rollback on apply error` test failed on the
`config set port $used_port`. In theory, it should throw the
error `Unable to listen on this port*`. But it failed on MacOs.
We solve it by adding `-myaddr 127.0.0.1` to the socket call.
 2021-12-02 18:18:18 +02:00
Binbin e3c0ea1cb4 Fix a harmless bug when using monitor in redis-cli with wrong reply (#9875) 
When we use monitor in redis-cli but encounter an error reply,
we will get stuck until we press Ctrl-C to quit.

This is a harmless bug. It might be useful if we add parameters
to monitor in the future, suck as monitoring only selected db.

before:
```
127.0.0.1:6379> monitor wrong
(error) ERR wrong number of arguments for 'monitor' command or subcommand
^C(9.98s)
127.0.0.1:6379>
```

after:
```
127.0.0.1:6379> monitor wrong
(error) ERR wrong number of arguments for 'monitor' command or subcommand
127.0.0.1:6379>
```
 2021-12-02 10:41:50 +02:00
meir@redislabs.com f21dc38a6e Redis Functions - Moved invoke Lua code functionality to script_lua.c 
The functionality was moved to script_lua.c under
callFunction function. Its purpose is to call the Lua
function already located on the top of the Lua stack.

Used the new function on eval.c to invoke Lua code.
The function will also be used to invoke Lua
code on the Lua engine.
 2021-12-01 23:57:06 +02:00
meir@redislabs.com fc731bc67f Redis Functions - Introduce script unit. 
Script unit is a new unit located on script.c.
Its purpose is to provides an API for functions (and eval)
to interact with Redis. Interaction includes mostly
executing commands, but also functionalities like calling
Redis back on long scripts or check if the script was killed.

The interaction is done using a scriptRunCtx object that
need to be created by the user and initialized using scriptPrepareForRun.

Detailed list of functionalities expose by the unit:
1. Calling commands (including all the validation checks such as
   acl, cluster, read only run, ...)
2. Set Resp
3. Set Replication method (AOF/REPLICATION/NONE)
4. Call Redis back to on long running scripts to allow Redis reply
   to clients and perform script kill

The commit introduce the new unit and uses it on eval commands to
interact with Redis.
 2021-12-01 23:54:23 +02:00
meir@redislabs.com e0cd580aef Redis Functions - Move Lua related variable into luaCtx struct 
The following variable was renamed:
1. lua_caller 			-> script_caller
2. lua_time_limit 		-> script_time_limit
3. lua_timedout 		-> script_timedout
4. lua_oom 			-> script_oom
5. lua_disable_deny_script 	-> script_disable_deny_script
6. in_eval			-> in_script

The following variables was moved to lctx under eval.c
1.  lua
2.  lua_client
3.  lua_cur_script
4.  lua_scripts
5.  lua_scripts_mem
6.  lua_replicate_commands
7.  lua_write_dirty
8.  lua_random_dirty
9.  lua_multi_emitted
10. lua_repl
11. lua_kill
12. lua_time_start
13. lua_time_snapshot

This commit is in a low risk of introducing any issues and it
is just moving varibales around and not changing any logic.
 2021-12-01 23:31:08 +02:00
meir@redislabs.com 22aab1ce94 Redis Functions - Move code to make review process easier. 
This commit is only move code around without changing it.
The reason behind this is to make review process easier
by allowing the reviewer to simply ignore all code movements.

changes:
1. rename scripting.c to eval.c
2. introduce and new file, script_lua.c, and move parts of Lua
   functionality to this new file. script_lua.c will eventually
   contains the shared code between legacy lua and lua engine.

This commit does not compiled on purpose. Its only purpose is to move
code and rename files.
 2021-12-01 23:30:59 +02:00
yoav-steinberg 0e5b813ef9 Multiparam config set (#9748) 
We can now do: `config set maxmemory 10m repl-backlog-size 5m`

## Basic algorithm to support "transaction like" config sets:

1. Backup all relevant current values (via get).
2. Run "verify" and "set" on everything, if we fail run "restore".
3. Run "apply" on everything (optional optimization: skip functions already run). If we fail run "restore".
4. Return success.

### restore
1. Run set on everything in backup. If we fail log it and continue (this puts us in an undefined
   state but we decided it's better than the alternative of panicking). This indicates either a bug
   or some unsupported external state.
2. Run apply on everything in backup (optimization: skip functions already run). If we fail log
   it (see comment above).
3. Return error.

## Implementation/design changes:
* Apply function are idempotent (have no effect if they are run more than once for the same config).
* No indication in set functions if we're reading the config or running from the `CONFIG SET` command
   (removed `update` argument).
* Set function should set some config variable and assume an (optional) apply function will use that
   later to apply. If we know this setting can be safely applied immediately and can always be reverted
   and doesn't depend on any other configuration we can apply immediately from within the set function
   (and not store the setting anywhere). This is the case of this `dir` config, for example, which has no
   apply function. No apply function is need also in the case that setting the variable in the `server` struct
   is all that needs to be done to make the configuration take effect. Note that the original concept of `update_fn`,
   which received the old and new values was removed and replaced by the optional apply function.
* Apply functions use settings written to the `server` struct and don't receive any inputs.
* I take care that for the generic (non-special) configs if there's no change I avoid calling the setter (possible
   optimization: avoid calling the apply function as well).
* Passing the same config parameter more than once to `config set` will fail. You can't do `config set my-setting
   value1 my-setting value2`.

Note that getting `save` in the context of the conf file parsing to work here as before was a pain.
The conf file supports an aggregate `save` definition, where each `save` line is added to the server's
save params. This is unlike any other line in the config file where each line overwrites any previous
configuration. Since we now support passing multiple save params in a single line (see top comments
about `save` in https://github.com/redis/redis/pull/9644) we should deprecate the aggregate nature of
this config line and perhaps reduce this ugly code in the future.
 2021-12-01 10:15:11 +02:00
Itamar Haber 21aa1d4b91 Adds auto-seq-only-generation via XADD ... <ms>-* (#9217) 
Adds the ability to autogenerate the sequence part of the millisecond-only explicit ID specified for `XADD`. This is useful in case added entries have an externally-provided timestamp without sub-millisecond resolution.
 2021-11-30 19:56:39 +02:00
Wen Hui 2afa41f628 Sentinel master reboot fix (#9438) 
Add master-reboot-down-after-period as a configurable parameter, to make it possible to trigger a failover from a master that is responding with `-LOADING` for a long time after being restarted.
 2021-11-30 18:46:15 +02:00
丽媛自己动 af072c26bc modify misleading note in comment (#9865) 
now rdbSaveInfo used in both way, so i think we should update previous notes, in case of misleading
 2021-11-30 14:10:49 +02:00
Viktor Söderqvist bdf531e38d Extend output of DEBUG HELP for POPULATE (#9869) 2021-11-30 12:36:52 +02:00
Meir Shpilraien (Spielrein) b8e82d205b Swap '\r\n' with spaces when returning a big number reply from Lua script. (#9870) 
The issue can only happened with a bad Lua script that claims to return
a big number while actually return data which is not a big number (contains
chars that are not digits). Such thing will not cause an issue unless the big
number value contains `\r\n` and then it messes the resp3 structure. The fix
changes all the appearances of '\r\n' with spaces.

Such an issue can also happened on simple string or error replies but those
already handle it the same way this PR does (replace `\r\n` with spaces).

Other replies type are not vulnerable to this issue because they are not
counting on free text that is terminated with `\r\n` (either it contains the
bulk length like string reply or they are typed reply that can not inject free
text like boolean or number).

The issue only exists on unstable branch, big number reply on Lua script
was not yet added to any official release.
 2021-11-30 12:27:05 +02:00
BinbinandOfir Luzon 3119a3aeb5 Fix CLIENT KILL kill all clients with id 0 (#9853) 
* Fix CLIENT KILL kill all clients with id 0 or with skipme
CLIENT KILL with ID argument should only kill the client with the provided ID. In old code, 
CLIENT KILL with id 0 will kill all the connected clients.

Co-authored-by: Ofir Luzon <ofirluzon@gmail.com>
 2021-11-29 13:35:36 -08:00
yoav-steinberg 2386e54182 fix deprecation of _BSD_SOURCE feature test macro (#9861) 
caused a build warning in linenoise since glibc 2.20
 2021-11-29 16:09:20 +02:00
OfirMos 9f9c78578f fixed mem leak on rdb load error (#9860) 
a rare case of short read that can happen when breaking the master-replica
connection on diskless load mode,
 2021-11-29 12:09:08 +02:00
Binbin 980bb3ae19 Add REDIS_CFLAGS='-Werror' to CI tests (#9828) 
Update CI so that warnings cause build failures.

Also fix a warning in `test-sanitizer-address`:
```
In function ‘strncpy’,
   inlined from ‘clusterUpdateMyselfIp’ at cluster.c:545:13:

/usr/include/x86_64-linux-gnu/bits/string_fortified.h:106:10:
error: ‘__builtin_strncpy’ specified bound 46 equals destination size [-Werror=stringop-truncation]

  106 |   return __builtin___strncpy_chk (__dest, __src, __len, __bos (__dest));
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
```
 2021-11-29 10:30:35 +02:00
leishiaoandxiaolei d56ded89c5 improvement of a blocking xread test (#9859) 
This test relies on that `XREAD BLOCK 20000 STREAMS s1{t} s2{t} s3{t} $ $ $`
is executed by redis before `XADD s2{t} * new abcd1234`. A ` wait_for_blocked_client`
is needed between the two to ensure the order, otherwise `XADD s2{t} * new abcd1234`
might be executed first due to network delay causing a test failure.

Co-authored-by: xiaolei <xiaolei@91jkys.com>
 2021-11-29 09:57:21 +02:00
sundb 494ee2f1fc Fix abnormal compression due to out-of-control recompress (#9849) 
This pr is following #9779 .

## Describe of feature
Now when we turn on the `list-compress-depth` configuration, the list will compress
the ziplist between `[list-compress-depth, -list-compress-depth]`.
When we need to use the compressed data, we will first decompress it, then use it,
and finally compress it again.
It's controlled by `quicklistNode->recompress`, which is designed to avoid the need to
re-traverse the entire quicklist for compression after each decompression, we only need
to recompress the quicklsitNode being used.
In order to ensure the correctness of recompressing, we should normally let
quicklistDecompressNodeForUse and quicklistCompress appear in pairs, otherwise,
it may lead to the head and tail being compressed or the middle ziplist not being
compressed correctly, which is exactly the problem this pr needs to solve.

## Solution
1. Reset `quicklistIter` after insert and replace.
    The quicklist node will be compressed in `quicklistInsertAfter`, `quicklistInsertBefore`,
   `quicklistReplaceAtIndex`, so we can safely reset the quicklistIter to avoid it being used again
2. `quicklistIndex` will return an iterator that can be used to recompress the current node after use.
    
## Test
1. In the `Stress Tester for #3343-Similar Errors` test, when the server crashes or when
   `valgrind` or `asan` error is detected, print violating commands.
2. Add a crash test due to wrongly recompressing after `lrem`.
3. Remove `insert before with 0 elements` and `insert after with 0 elements`,
   Now we forbid any operation on an NULL quicklistIter.
 2021-11-29 07:57:01 +02:00
Binbin 8759c1e14b Improve stability in some blocking command tests (#9856) 
In order to test the situation where multiple clients are
blocked, we set up multiple clients to execute some blocking
commands. These tests depend on the order of command processing.

Those tests are based on the wrong assumption that the command
send first will be executed by the server first, which is obviously
wrong in some network delyas.

This commit ensures orderly execution of commands by waiting
and judging the number of blocked clients each time.

Fix #9850
 2021-11-28 15:37:35 +02:00
Meir Shpilraien (Spielrein) 6b0b04f1b2 Clean Lua stack before parsing call reply to avoid crash on a call with many arguments (#9809) 
This commit 0f8b634cd (CVE-2021-32626 released in 6.2.6, 6.0.16, 5.0.14)
fixes an invalid memory write issue by using `lua_checkstack` API to make
sure the Lua stack is not overflow. This fix was added on 3 places:
1. `luaReplyToRedisReply`
2. `ldbRedis`
3. `redisProtocolToLuaType`

On the first 2 functions, `lua_checkstack` is handled gracefully while the
last is handled with an assert and a statement that this situation can
not happened (only with misbehave module):

> the Redis reply might be deep enough to explode the LUA stack (notice
that currently there is no such command in Redis that returns such a nested
reply, but modules might do it)

The issue that was discovered is that user arguments is also considered part
of the stack, and so the following script (for example) make the assertion reachable:
```
local a = {}
for i=1,7999 do
    a[i] = 1
end 
return redis.call("lpush", "l", unpack(a))
```

This is a regression because such a script would have worked before and now
its crashing Redis. The solution is to clear the function arguments from the Lua
stack which makes the original assumption true and the assertion unreachable.
 2021-11-28 11:59:39 +02:00
Meir Shpilraien (Spielrein) a8c1253b6f Fix Lua C API violation on lua msgpack lib. (#9832) 
msgpack lib missed using lua_checkstack and so on rare
cases overflow the stack by at most 2 elements. This is a
violation of the Lua C API. Notice that Lua allocates
additional 5 more elements on top of lua->stack_last
so Redis does not access an invalid memory. But it is an
API violation and we should avoid it.

This PR also added a new Lua compilation option. The new
option can be enable using environment variable called
LUA_DEBUG. If set to `yes` (by default `no`), Lua will be
compiled without optimizations and with debug symbols (`-O0 -g`).
In addition, in this new mode, Lua will be compiled with the
`-DLUA_USE_APICHECK` flag that enables extended Lua C API
validations.

In addition, set LUA_DEBUG=yes on daily valgrind flow so we
will be able to catch Lua C API violations in the future.
 2021-11-28 11:33:09 +02:00
Viktor Söderqvist acf3495eb8 Sort out the mess around writable replicas and lookupKeyRead/Write (#9572) 
Writable replicas now no longer use the values of expired keys. Expired keys are
deleted when lookupKeyWrite() is used, even on a writable replica. Previously,
writable replicas could use the value of an expired key in write commands such
as INCR, SUNIONSTORE, etc..

This commit also sorts out the mess around the functions lookupKeyRead() and
lookupKeyWrite() so they now indicate what we intend to do with the key and
are not affected by the command calling them.

Multi-key commands like SUNIONSTORE, ZUNIONSTORE, COPY and SORT with the
store option now use lookupKeyRead() for the keys they're reading from (which will
not allow reading from logically expired keys).

This commit also fixes a bug where PFCOUNT could return a value of an
expired key.

Test modules commands have their readonly and write flags updated to correctly
reflect their lookups for reading or writing. Modules are not required to
correctly reflect this in their command flags, but this change is made for
consistency since the tests serve as usage examples.

Fixes #6842. Fixes #7475.
 2021-11-28 11:26:28 +02:00
sundb 4d8700786e Fix COMMAND GETKEYS on LCS (#9852) 
Remove lcsGetKeys to clean up the remaining STRALGO after #9733.
i.e. it still used a getkeys_proc which was still looking for the KEYS or STRINGS arguments
 2021-11-28 09:02:38 +02:00
Huang Zhw 596635fa0c Cleanup: fix typo and remove some obsoleting definitions. (#9851) 2021-11-27 14:47:51 -08:00
uriyageandOran Agra fa48fb2d0f Do not watch keys for dirty client (#9829) 
Currently, the watching clients are marked as dirty when a watched
key is touched, but we continue watching the keys for no reason.
Then, when the same key is touched again, we iterate again on the
watching clients list and mark all clients as dirty again.
Only when the exec/unwatch command is issued will the client be
removed from the key->watching_clients list. The same applies when
a dirty client calls the WATCH command. The key will be added to be
watched by the client even if it has no effect.

In the field, no performance degradation was observed as a result of the
current implementation; it is merely a cleanup with possible memory and
performance gains in some situations.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-11-25 15:23:15 +02:00
Pavel MelkozerovandPavel Melkozerov 9630ded313 fix fob bad log messages in rdbSave (#9842) (#9843) 
logs message prints wrong file is failed to open temporary file
logs have error occurred in getcwd (uses same errno to report error)

Co-authored-by: Pavel Melkozerov <pavel.melkozerov@nokia.com>
 2021-11-24 16:01:39 +02:00
sundbandOran Agra 4512905961 Replace ziplist with listpack in quicklist (#9740) 
Part three of implementing #8702, following #8887 and #9366 .

## Description of the feature
1. Replace the ziplist container of quicklist with listpack.
2. Convert existing quicklist ziplists on RDB loading time. an O(n) operation.

## Interface changes
1. New `list-max-listpack-size` config is an alias for `list-max-ziplist-size`.
2. Replace `debug ziplist` command with `debug listpack`.

## Internal changes
1. Add `lpMerge` to merge two listpacks . (same as `ziplistMerge`)
2. Add `lpRepr` to print info of listpack which is used in debugCommand and `quicklistRepr`. (same as `ziplistRepr`)
3. Replace `QUICKLIST_NODE_CONTAINER_ZIPLIST` with `QUICKLIST_NODE_CONTAINER_PACKED`(following #9357 ).
    It represent that a quicklistNode is a packed node, as opposed to a plain node.
4. Remove `createZiplistObject` method, which is never used.
5. Calculate listpack entry size using overhead overestimation in `quicklistAllowInsert`.
    We prefer an overestimation, which would at worse lead to a few bytes below the lowest limit of 4k.

## Improvements
1. Calling `lpShrinkToFit` after converting Ziplist to listpack, which was missed at #9366.
2. Optimize `quicklistAppendPlainNode` to avoid memcpy data.

## Bugfix
1. Fix crash in `quicklistRepr` when ziplist is compressed, introduced from #9366.

## Test
1. Add unittest for `lpMerge`.
2. Modify the old quicklist ziplist corrupt dump test.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-11-24 13:34:13 +02:00
Binbin fb4f7be22c Wait for asyn_loading to stop in short read test (#9841) 
In #9323, when `repl-diskless-load` is enabled and set to `swapdb`,
if the master replication ID hasn't changed, we can load data-set
asynchronously, and serving read commands during the full resync.

In `diskless loading short read` test, after a loading successfully,
we will wait for the loading to stop and continue the for loop.

After the introduction of `async_loading`, we also need to check it.
Otherwise the next loop will start too soon, may trigger a timing issue.
 2021-11-24 12:46:43 +02:00
Binbin 9273d09dd4 Add tests to cover EXPIRE overflow fix (#9839) 
In #8287, some overflow checks have been added. But when
`when *= 1000` overflows, it will become a positive number.
And the check not able to catch it. The key will be added with
a short expiration time and will deleted a few seconds later.

In #9601, will check the overflow after `*=` and return an
error first, and avoiding this situation.

In this commit, added some tests to cover those code paths.
Found it in #9825, and close it.
 2021-11-24 09:39:23 +02:00
Oran Agra a3a014294f fix invalid read on corrupt ziplist (#9831) 
If the last bytes in ziplist are corrupt and we decode from tail to head,
we may reach slightly outside the ziplist.
 2021-11-23 14:56:52 +02:00
guybe7 b161cff5f9 QUIT is a command, HOST: and POST are not (#9798) 
Some people complain that QUIT is missing from help/command table.
Not appearing in COMMAND command, command stats, ACL, etc.
and instead, there's a hack in processCommand with a comment that looks outdated.
Note that it is [documented](https://redis.io/commands/quit)

At the same time, HOST: and POST are there in the command table although these are not real commands.
They would appear in the COMMAND command, and even in commandstats.

Other changes:
1. Initialize the static logged_time static var in securityWarningCommand
2. add `no-auth` flag to RESET so it can always be executed.
 2021-11-23 10:38:25 +02:00
Oran Agra f07dedf73f Fix invalid access in lpFind on corrupted listpack (#9819) 
Issue found by corrupt-dump-fuzzer test with ASAN.
The problem was that lpSkip and lpGetWithSize could read the next listpack entry without validating that it's in range.
Similarly even the memcmp in lpFind could do that and possibly crash on segfault and now they'll crash on assert first.

The naive fix of using lpAssertValidEntry every time, resulted in 30% degradation in the lpFind benchmark of the unit test.
The final fix with the condition at the bottom has no performance implications.
 2021-11-22 15:30:00 +02:00
Oran Agra f00a8ad93c fix string escaping in corrupt-dump test to support TCL8.5 (#9824) 
TCL8.5 can't handle cases where part of the string is escaped and part of it isn't,
if there's a single char that needs escaping, we need to escape the whole string.
 2021-11-22 12:30:06 +02:00
BinbinandOran Agra 698b577413 Fix timing issue in sub-second expires test (#9821) 
The `PEXPIRE/PSETEX/PEXPIREAT can set sub-second expires` test is
a very time sensitive test, it used to occasionally fail on MacOS.

It will perform there internal tests in a loop, as long as one
fails, it will try to excute again in the next loop.

oranagra suggested that we can split it into three individual tests,
so that if one fails, we do not need to retry the others. And maybe
it will increase the chances of success dramatically.

Each is executed 500 times, and the number of retries is collected:
```
PSETEX, total: 500, sum: 745, min: 0, max: 13, avg: 1.49

PEXPIRE, total: 500, sum: 575, min: 0, max: 16, avg: 1.15

PEXPIREAT, total: 500, sum: 0, min: 0, max: 0, avg: 0.0

ALL(old_way), total: 500, sum: 8090, min: 0, max: 138, avg: 16.18
```

And we can see the threshold is very low.
Splitting the test also makes the code better to maintain.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-11-22 08:39:27 +02:00
Oran Agra 183b90a625 Fix false positive leak reported by GCC ASAN (#9816) 
Leak found by the corrupt-dump-fuzzer when using GCC ASAN, which seems
to falsely report leaks on pointers kept only on the stack when calling exit.
Instead we now use _exit on panic / assert to skip these leak checks.

Additionally, check for sanitizer warnings in the corrupt-dump-fuzzer between iterations,
so that when something is found we know which test to relate it too (and it prints reproduction command list)
 2021-11-21 18:47:10 +02:00
Ozan Tezcan a68b71ac02 Don't use accurate option with ASAN unit tests (#9818) 
specifically the ziplist and listpack unit tests and benchmarks run for too long with address sanitizer and --accurate
 2021-11-21 18:27:54 +02:00
Yossi Gottlieb fd0ca74763 Fix occasional RM_Call() crashes. (#9805) 
With dynamically growing argc (#9528), it is necessary to initialize
argv_len. Normally createClient() handles that, but in the case of a
module shared_client, this needs to be done explicitly.

This also addresses an issue with rewriteClientCommandArgument() which
doesn't properly handle the case where the new element extends beyond
argc but not beyond argv_len.
 2021-11-21 15:54:14 +02:00
Oran Agra 1417648469 Prevent LCS from allocating temp memory over proto-max-bulk-len (#9817) 
LCS can allocate immense amount of memory (sizes of two inputs multiplied by each other).
In the past this caused some possible security issues due to overflows, which we solved
and also added use of `trymalloc` to return "Insufficient memory" instead of OOM panic zmalloc.

But in case overcommit is enabled, it could be that we won't get the OOM panic, and zmalloc
will succeed, and then we can get OOM killed by the kernel.

The solution here is to prevent LCS from allocating transient memory that's bigger than
`proto-max-bulk-len` config.
This config is not directly related to transient memory, but using a hard coded value ad well as
introducing a specific config seems wrong.

This comes to solve an error in the corrupt-dump-fuzzer test that started in the daily CI see #9799
 2021-11-21 14:30:20 +02:00
Oran Agra d4e7ffb38c Improve active defrag in jemalloc 5.2 (#9778) 
Background:
Following the upgrade to jemalloc 5.2, there was a test that used to be flaky and
started failing consistently (on 32bit), so we disabled it ​(see #9645).

This is a test that i introduced in #7289 when i attempted to solve a rare stagnation
problem, and it later turned out i failed to solve it, ans what's more i added a test that
caused it to be not so rare, and as i mentioned, now in jemalloc 5.2 it became consistent on 32bit.

Stagnation can happen when all the slabs of the bin are equally utilized, so the decision
to move an allocation from a relatively empty slab to a relatively full one, will never
happen, and in that test all the slabs are at 50% utilization, so the defragger could just
keep scanning the keyspace and not move anything.

What this PR changes:
* First, finally in jemalloc 5.2 we have the count of non-full slabs, so when we compare
  the utilization of the current slab, we can compare it to the average utilization of the non-full
  slabs in our bin, instead of the total average of our bin. this takes the full slabs out of the game,
  since they're not candidates for migration (neither source nor target).
* Secondly, We add some 12% (100/8) to the decision to defrag an allocation, this is the part
  that aims to avoid stagnation, and it's especially important since the above mentioned change
  can get us closer to stagnation.
* Thirdly, since jemalloc 5.2 adds sharded bins, we take into account all shards (something
  that's missing from the original PR that merged it), this isn't expected to make any difference
  since anyway there should be just one shard.

How this was benchmarked.
What i did was run the memefficiency test unit with `--verbose` and compare the defragger hits
and misses the tests reported.
At first, when i took into consideration only the non-full slabs, it got a lot worse (i got into
stagnation, or just got a lot of misses and a lot of hits), but when i added the 10% i got back
to results that were slightly better than the ones of the jemalloc 5.1 branch. i.e. full defragmentation
was achieved with fewer hits (relocations), and fewer misses (keyspace scans).
 2021-11-21 13:35:39 +02:00
Madelyn Olson 362b3b02e6 Initialize manual failover replica target (#9814) 2021-11-19 19:56:44 -08:00
RuiliF b567d3694c fix memory leak in example module hellocluster (#9813) 
Missing RedisModule_FreeCallReply
 2021-11-20 00:48:08 +02:00
Yossi Gottlieb 366d5101d3 Tests: add a few missing needs:debug tags. (#9806) 2021-11-18 23:01:56 +02:00
Wen Hui 75d50e5de9 Fix module doc - missing function due to blank line (#9807) 2021-11-18 21:07:53 +02:00
perryitay 0c10f0e1c0 Fix crashes when list-compress-depth is used. (#9779) 
Recently we started using list-compress-depth in tests (was completely untested till now).
Turns this triggered test failures with the external mode, since the tests left the setting enabled
and then it was used in other tests (specifically the fuzzer named "Stress tester for #3343-alike bugs").

This PR fixes the issue of the `recompress` flag being left set by mistake, which caused the code to
later to compress the head or tail nodes (which should never be compressed)

The solution is to reset the recompress flag when it should have been (when it was decided not to compress).

Additionally we're adding some assertions and improve the tests so in order to catch other similar bugs.
 2021-11-18 18:09:30 +02:00
Binbin 020092e08f Add --dump-logs to ci tests and daily tests (#9803) 
So that we can print server log on test failure.
 2021-11-18 16:04:01 +02:00
Eduardo Semprebon 1a255e3150 Reject PING with MASTERDOWN when replica-serve-stale-data=no (#9757) 
Currently PING returns different status when server is not serving data,
for example when `LOADING` or `BUSY`.
But same was not true for `MASTERDOWN`
This commit makes PING reply with `MASTERDOWN` when
replica-serve-stale-data=no and link is MASTER is down.
 2021-11-18 10:53:17 +02:00
guybe7 af7489886d Obliterate STRALGO! add LCS (which only works on keys) (#9799) 
Drop the STRALGO command, now LCS is a command of its own and it only works on keys (not input strings).
The motivation is that STRALGO's syntax was really messed-up...
- assumes all (future) string algorithms will take similar arguments
- mixes command that takes keys and one that doesn't in the same command.
- make it nearly impossible to expose the right key spec in COMMAND INFO (issues cluster clients)
- hard for cluster clients to determine the key names (firstkey, lastkey, etc)
- hard for ACL / flags (is it a read command?)

This is a breaking change.
 2021-11-18 10:47:49 +02:00
Binbin 91e77a0cfb Fixes ZPOPMIN/ZPOPMAX wrong replies when count is 0 with non-zset (#9711) 
Moves ZPOP ... 0 fast exit path after type check to reply with
WRONGTYPE. In the past it will return an empty array.

Also now count is not allowed to be negative.

see #9680

before:
```
127.0.0.1:6379> set zset str
OK
127.0.0.1:6379> zpopmin zset 0
(empty array)
127.0.0.1:6379> zpopmin zset -1
(empty array)
```

after:
```
127.0.0.1:6379> set zset str
OK
127.0.0.1:6379> zpopmin zset 0
(error) WRONGTYPE Operation against a key holding the wrong kind of value
127.0.0.1:6379> zpopmin zset -1
(error) ERR value is out of range, must be positive
```
 2021-11-18 10:13:16 +02:00
Madelyn Olson 32215e7889 Unpause clients earlier during manual cluster failover (#9676) 
Unpause clients after manual failover ends instead of the timed offset
 2021-11-17 21:28:13 -08:00
Oran Agra 45144fc79f Daily tests flags inconsistency (#9792) 
Add --accurate to unit tests (new feature recently added)
Add --no-latency to valgrind run (was present only for modules)
add --no-latency to macos and freebsd runs (was not present for modules)
add --timeout to freebsd (same one we have for valgrind)
 2021-11-16 21:00:13 +02:00
Axlgrep 5f8ee3815c optimze the judgment logic of use_pattern in scanGenericCommand() (#9789) 
in `scan 0 match ""` case,  pat is empty sds(patlen is 0), I don't think should access the
first character directly in this case(even though the first character is ' \0 '), for the
code readability, I switch the two positions of judgment logic.
 2021-11-16 17:03:24 +02:00
harleyliao 992a610fe8 set aof rewrite status err, when fork fail (#5606) 
when aof rewrite is failed by fork(), It'll be indicated by aof_last_bgrewrite_status
INFO field, same as when the fork child fails later on.
 2021-11-16 13:59:03 +02:00
sundb 985430b4fc Change lzf to handle values larger than UINT32_MAX (#9776) 
Redis supports inserting data over 4GB into string (and recently for lists too, see #9357),
But LZF compression used in RDB files (see `rdbcompression` config), and in quicklist
(see `list-compress-depth` config) does not support compress/decompress data over
UINT32_MAX, which will result in corrupting the rdb after compression.

Internal changes:
1. Modify the `unsigned int` parameter of `lzf_compress/lzf_decompress` to `size_t`.
2. Modify the variable types in `lzf_compress` involving offsets and lengths to `size_t`.
3. Set LZF_USE_OFFSETS to 0.
    When LZF_USE_OFFSETS is 1, lzf store offset into `LZF_HSLOT`(32bit). 
    Even in 64-bit, `LZF_USE_OFFSETS` defaults to 1, because lzf assumes that it only
    compresses and decompresses data smaller than UINT32_MAX.
    But now we need to make lzf support 64-bit, turning on `LZF_USE_OFFSETS` will make
    it impossible to store 64-bit offsets or pointers.
    BTW, disable LZF_USE_OFFSETS also brings a few performance improvements.

Tests:
1. Add test for compress/decompress string large than UINT32_MAX.
2. Add unittest for compress/decompress quicklistNode.
 2021-11-16 13:12:25 +02:00
sundb e725d737fb Add --large-memory flag for REDIS_TEST to enable tests that consume more than 100mb (#9784) 
This is a preparation step in order to add a new test in quicklist.c see #9776
 2021-11-16 08:55:10 +02:00
guoxiang1996 aba70df48f insufficient size for cached client flags in call() (#9783) 
The client flags is a 64 bit integer, but the temporary cached value on the stack of call() is 32 bit.
luckily this doesn't lead to any bugs since the only flags used against this variables are below 32 bit.
 2021-11-16 08:21:23 +02:00
yoav-steinberg e968d9ac58 Connection leak in external tests. (#9777) 
Two issues:
1. In many tests we simply forgot to close the connections we created, which doesn't matter for normal tests where the server is killed, but creates a leak on external server tests.
2. When calling `start_server` on external test we create a fresh connection instead of really starting a new server, but never clean it at the end.
 2021-11-15 11:07:43 +02:00
Binbin 174eedce44 Tune expire test threshold. (#9775) 
I have seen this CI failure twice on MacOS:

*** [err]: PEXPIRE/PSETEX/PEXPIREAT can set sub-second expires in tests/unit/expire.tcl
Expected 'somevalue {} somevalue {} somevalue {}' to equal or match '{} {} {} {} somevalue {}'

I did some loop test in my own daily CI, the results show that is
not particularly stable. Change the threshold from 30 to 50.
 2021-11-13 07:55:48 +02:00
Ozan Tezcan bcb7961f12 Enable running daily CI from forks (#9771) 
Was impossible to run the daily CI from private forks due to "redis/redis" repo check.
Let's disable that check for manual triggers.
 2021-11-11 14:39:20 +02:00
Ozan Tezcan b91d8b289b Add sanitizer support and clean up sanitizer findings (#9601) 
- Added sanitizer support. `address`, `undefined` and `thread` sanitizers are available.  
- To build Redis with desired sanitizer : `make SANITIZER=undefined`
- There were some sanitizer findings, cleaned up codebase
- Added tests with address and undefined behavior sanitizers to daily CI.
- Added tests with address sanitizer to the per-PR CI (smoke out mem leaks sooner).

Basically, there are three types of issues : 

**1- Unaligned load/store** : Most probably, this issue may cause a crash on a platform that
does not support unaligned access. Redis does unaligned access only on supported platforms.

**2- Signed integer overflow.** Although, signed overflow issue can be problematic time to time
and change how compiler generates code, current findings mostly about signed shift or simple
addition overflow. For most platforms Redis can be compiled for, this wouldn't cause any issue
as far as I can tell (checked generated code on godbolt.org).

 **3 -Minor leak** (redis-cli), **use-after-free**(just before calling exit());

UB means nothing guaranteed and risky to reason about program behavior but I don't think any
of the fixes here worth backporting. As sanitizers are now part of the CI, preventing new issues
will be the real benefit.
 2021-11-11 13:51:33 +02:00
yoav-steinberg cd6b3d558b Archive external redis log in external tests (#9765) 
On test failure store the external redis server logs as CI artifacts so we can review them.

Write test name to server log for external server tests.
This is attempted and silently failed in case external server doesn't support it.
Note that in non-external server mode we use a more robust method of writing to the log which doesn't depend on the
server actually running/working. This isn't possible for externl servers and required for some complex tests which are
skipped in external mode anyway.

Cleanup: remove dup code.
 2021-11-11 13:04:02 +02:00
Vadim PushtaevandVadim Pushtaev f069d09ee0 Use utils/create-cluster from any dir (#9766) 
Co-authored-by: Vadim Pushtaev <v.pushtaev@corp.mail.ru>
 2021-11-11 08:33:09 +02:00
Oran Agra 0927a0dd24 Try solving test timeout on freebsd CI (#9768) 
First, avoid using --accurate on the freebsd CI, we only care about
systematic issues there due to being different platform, but not
accuracy

Secondly, when looking at the test which timed out it seems silly and
outdated:
- it used KEYS to attempt to trigger lazy expiry, but KEYS doesn't do
  that anymore.
- it used some hard coded sleeps rather than waiting for things to
  happen and exiting ASAP
 2021-11-10 19:39:26 +02:00
Oran Agra 978eadbad4 Increase test timeout in valgrind runs (#9767) 
We saw some tests sporadically time out on valgrind (namely the ones
from #9323).

Increasing valgrind timeout from 20 mins to 40 mins in CI.
And fixing an outdated help message.
 2021-11-10 19:38:58 +02:00
YaacovHazan 03406fcb6c fix short timeout in replication short read tests (#9763) 
In both tests, "diskless loading short read" and "diskless loading short read with module",
the timeout of waiting for the replica to respond to a short read and log it, is too short.

Also, add --dump-logs in runtest-moduleapi for valgrind runs.
 2021-11-09 22:37:18 +02:00
Jim Brunner cd0f710be9 fix minor memory leak in rewriteConfigSds (#9762) 2021-11-09 10:35:22 -08:00
Eduardo Semprebon c22d3684ba Fix diskless load handling on broken EOF marker (#9752) 
During diskless replication, the check for broken EOF mark is misplaced
and should be earlier. Now we do not swap db, we do proper cleanup and
correctly raise module events on this kind of failure.

This issue existed prior to #9323, but before, the side effect was not restoring
backup and not raising the correct module events on this failure.
 2021-11-09 11:46:10 +02:00
Wen Hui 2ce29e032b Sentinel tls memory leak (#9753) 
There was a memory leak when tls is used in Sentinels.
The memory leak is noticed when some of the replicas are offline.
 2021-11-08 22:23:31 +02:00
Yossi Gottlieb a1aba4bf75 Fix EINTR test failures. (#9751) 
* Clean up EINTR handling so EINTR will not change connection state to begin with.
* On TLS, catch EINTR and return it as-is before going through OpenSSL error handling (which seems to not distinguish it from EAGAIN).
 2021-11-08 16:09:33 +02:00
Huang Zhw 48d870aed1 Move config from clusterCron to config update (#9580) 2021-11-07 18:56:03 -08:00
chendianqiangchendianqiangOran Agra
 		a527c3e814 Test suite - user server socket to optimize port detection (#9663) 
Optimized port detection for tcl, use 'socket -server' instead of 'socket' to rule out port on TIME_WAIT

Co-authored-by: chendianqiang <chendianqiang@meituan.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-11-07 13:53:57 +02:00
yoav-steinbergandBjorn Svensson 79ac57561f Refactor config.c for generic setter interface (#9644) 
This refactors all `CONFIG SET`s and conf file loading arguments go through
the generic config handling interface.

Refactoring changes:
- All config params go through the `standardConfig` interface (some stuff which
  is only related to the config file and not the `CONFIG` command still has special
  handling for rewrite/config file parsing, `loadmodule`, for example.) .
- Added `MULTI_ARG_CONFIG` flag for configs to signify they receive a variable
  number of arguments instead of a single argument. This is used to break up space
  separated arguments to `CONFIG SET` so the generic setter interface can pass
  multiple arguments to the setter function. When parsing the config file we also break
  up anything after the config name into multiple arguments to the setter function.

Interface changes:
- A side effect of the above interface is that the `bind` argument in the config file can
  be empty (no argument at all) this is treated the same as passing an single empty
  string argument (same as `save` already used to work).
- Support rewrite and setting `watchdog-period` from config file (was only supported
  by the CONFIG command till now).
- Another side effect is that the `save T X` config argument now supports multiple
  Time-Changes pairs in a single line like its `CONFIG SET` counterpart. So in the
  config file you can either do:
  ```
  save 3600 1
  save 600 10
  ```
  or do
  ```
  save 3600 1 600 10
  ```

Co-authored-by: Bjorn Svensson <bjorn.a.svensson@est.tech>
 2021-11-07 13:40:08 +02:00
Binbin ddb508c0a2 Fixing import of sys/errno (#9739) 2021-11-05 07:50:25 +02:00
Eduardo SemprebonEduardo SemprebonOran Agra
 		91d0c758e5 Replica keep serving data during repl-diskless-load=swapdb for better availability (#9323) 
For diskless replication in swapdb mode, considering we already spend replica memory
having a backup of current db to restore in case of failure, we can have the following benefits
by instead swapping database only in case we succeeded in transferring db from master:

- Avoid `LOADING` response during failed and successful synchronization for cases where the
  replica is already up and running with data.
- Faster total time of diskless replication, because now we're moving from Transfer + Flush + Load
  time to Transfer + Load only. Flushing the tempDb is done asynchronously after swapping.
- This could be implemented also for disk replication with similar benefits if consumers are willing
  to spend the extra memory usage.

General notes:
- The concept of `backupDb` becomes `tempDb` for clarity.
- Async loading mode will only kick in if the replica is syncing from a master that has the same
  repl-id the one it had before. i.e. the data it's getting belongs to a different time of the same timeline. 
- New property in INFO: `async_loading` to differentiate from the blocking loading
- Slot to Key mapping is now a field of `redisDb` as it's more natural to access it from both server.db
  and the tempDb that is passed around.
- Because this is affecting replicas only, we assume that if they are not readonly and write commands
  during replication, they are lost after SYNC same way as before, but we're still denying CONFIG SET
  here anyways to avoid complications.

Considerations for review:
- We have many cases where server.loading flag is used and even though I tried my best, there may
  be cases where async_loading should be checked as well and cases where it shouldn't (would require
  very good understanding of whole code)
- Several places that had different behavior depending on the loading flag where actually meant to just
  handle commands coming from the AOF client differently than ones coming from real clients, changed
  to check CLIENT_ID_AOF instead.

**Additional for Release Notes**
- Bugfix - server.dirty was not incremented for any kind of diskless replication, as effect it wouldn't
  contribute on triggering next database SAVE
- New flag for RM_GetContextFlags module API: REDISMODULE_CTX_FLAGS_ASYNC_LOADING
- Deprecated RedisModuleEvent_ReplBackup. Starting from Redis 7.0, we don't fire this event.
  Instead, we have the new RedisModuleEvent_ReplAsyncLoad holding 3 sub-events: STARTED,
  ABORTED and COMPLETED.
- New module flag REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD for RedisModule_SetModuleOptions
  to allow modules to declare they support the diskless replication with async loading (when absent, we fall
  back to disk-based loading).

Co-authored-by: Eduardo Semprebon <edus@saxobank.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-11-04 10:46:50 +02:00
Itamar Haber 06dd202a05 Fixes LPOP/RPOP wrong replies when count is 0 (#9692) 
Introduced in #8179, this fixes the command's replies in the 0 count edge case.
[BREAKING] changes the reply type when count is 0 to an empty array (instead of nil)
Moves LPOP ... 0 fast exit path after type check to reply with WRONGTYPE
 2021-11-04 09:43:08 +02:00
menwenandOran Agra ccf8a651f3 Retry when a blocked connection system call is interrupted by a signal (#9629) 
When repl-diskless-load is enabled, the connection is set to the blocking state.
The connection may be interrupted by a signal during a system call.
This would have resulted in a disconnection and possibly a reconnection loop.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-11-04 09:09:28 +02:00
Oran AgraandMadelyn Olson d04f306931 Fix race condition in cluster test 22-replica-in-sync (#9721) 
there was a chance that by the time the assertion is executed,
the replica already manages to reconnect.

now we make sure the replica is unable to re-connect to the master.

additionally, we wait for some gossip from the disconnected replica,
to see that it doesn't mess things up.

unrelated: fix a typo when trying to exhaust the backlog, one that
didn't have any harmful implications

Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
 2021-11-04 08:44:18 +02:00
perryitaysundbOran Agra
 		f27083a4a8 Add support for list type to store elements larger than 4GB (#9357) 
Redis lists are stored in quicklist, which is currently a linked list of ziplists.
Ziplists are limited to storing elements no larger than 4GB, so when bigger
items are added they're getting truncated.
This PR changes quicklists so that they're capable of storing large items
in quicklist nodes that are plain string buffers rather than ziplist.

As part of the PR there were few other changes in redis: 
1. new DEBUG sub-commands: 
   - QUICKLIST-PACKED-THRESHOLD - set the threshold of for the node type to
     be plan or ziplist. default (1GB)
   - QUICKLIST <key> - Shows low level info about the quicklist encoding of <key>
2. rdb format change:
   - A new type was added - RDB_TYPE_LIST_QUICKLIST_2 . 
   - container type (packed / plain) was added to the beginning of the rdb object
     (before the actual node list).
3. testing:
   - Tests that requires over 100MB will be by default skipped. a new flag was
     added to 'runtest' to run the large memory tests (not used by default)

Co-authored-by: sundb <sundbcn@gmail.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-11-03 20:47:18 +02:00
guybe7 f11a2d4dd7 Fix COMMAND GETKEYS on EVAL without keys (#9733) 
Add new no-mandatory-keys flag to support COMMAND GETKEYS of commands
which have no mandatory keys.

In the past we would have got this error:
```
127.0.0.1:6379> command getkeys eval "return 1" 0
(error) ERR Invalid arguments specified for command
```
 2021-11-03 14:38:26 +02:00
perryitay 77d3c6bff3 fix: lookupKey on SETNX and SETXX only once (#9640) 
When using SETNX and SETXX we could end up doing key lookup twice.
This presents a small inefficiency price.
Also once we have statistics of write hit and miss they'll be wrong (recording the same key hit twice)
 2021-11-03 14:12:33 +02:00
Oran Agra d25dc08932 Solve issues with tracking test in external mode (#9726) 
The issue was that setting maxmemory to used_memory and expecting
eviction is insufficient, since we need to take
mem_not_counted_for_evict into consideration.

This test got broken by #9166
 2021-11-02 16:07:51 -07:00
yiyuanerandguoyiyuan 78025c4a26 Add checks for overflow in redis-check-aof and loadAppendOnlyFile (#9669) 
Co-authored-by: guoyiyuan <guoyiyuan@sbrella.com>
 2021-11-02 17:03:07 +02:00
Oran Agra 87321deb3f attempt to fix tracking test issue with external tests due to lazy free (#9722) 
The External tests started failing recently for unclear reason:
```
*** [err]: Tracking invalidation message of eviction keys should be before response in tests/unit/tracking.tcl
Expected '0' to be equal to 'invalidate volatile-key' (context: type eval line 21 cmd {assert_equal $res {invalidate volatile-key}} proc ::test)
```

I suspect the issue is that the used_memory sample is taken while a lazy free is still being processed.
 2021-11-02 16:42:53 +02:00
menwen d5ca72e38b fix defrag test looking at the wrong latency metric (#9723) 
the latency event was renamed in #7726, and the outcome was that the test was
ineffective (unable to measure the max latency, always seeing 0)
 2021-11-02 15:52:56 +02:00
Wang Yuanandsundb 526cbb5cff Fix not updating backlog histlen when trimming repl backlog (#9713) 
Since the loop in incrementalTrimReplicationBacklog checks the size of histlen,
we cannot afford to update it only when the loop exits, this may cause deleting
much more replication blocks, and replication backlog may be less than setting size.

introduce in #9166 

Co-authored-by: sundb <sundbcn@gmail.com>
 2021-11-02 11:04:11 +02:00
zhaozhao.zz d08f0552ee rebuild replication backlog index when master restart (#9720) 
After PR #9166 , replication backlog is not a real block of memory, just contains a
reference points to replication buffer's block and the blocks index (to accelerate
search offset when partial sync), so we need update both replication buffer's block's
offset and replication backlog blocks index's offset when master restart from RDB,
since the `server.master_repl_offset` is changed.
The implications of this bug was just a slow search, but not a replication failure.
 2021-11-02 10:53:52 +02:00
BinbinandOran Agra 58a1d16ff6 Fix timing issue in replication test (#9719) 
So it looks like sampling set loglines [count_log_lines -2] was
executed too late, and the replication managed to complete before that.

```
*** [err]: diskless no replicas drop during rdb pipe in tests/integration/replication.tcl
log message of '"*Diskless rdb transfer, done reading from pipe, 2 replicas still up*"' not found in ./tests/tmp/server.6124.69/stdout after line: 52 till line: 52
```

Changes:
1. when we search the master log file, we start to search from before we sent the REPLICAOF
  command, to prevent a race in which the replication completed before we sampled the log line count.
2. we don't need to sample the replica loglines sine it's a fresh resplica that's just been started, so the message
  we're looking for is the first occurrence in the log, we can start search from 0.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-11-02 10:32:01 +02:00
BinbinandOran Agra cea7809cea Fix race condition in psync2-pingoff test (#9712) 
Test failed on freebsd:
```
*** [err]: Make the old master a replica of the new one and check conditions in tests/integration/psync2-pingoff.tcl
Expected '162' to be equal to '176' (context: type eval line 18 cmd {assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset]} proc ::test)
```

There are two possible race conditions in the test.

1. The code waits for sync_full to increment, and assumes that means the
master did the fork. But in fact there are cases the master will increment
that sync_full counter (after replica asks for sync), but will see that
there's already a fork running and will delay the fork creation.

In this case the INCR will be executed before the fork happens, so it'll
not be in the command stream. Solve that by waiting for `master_link_status: up`
on the replica before the INCR.

2. The repl-ping-replica-period is still high (1 second), so there's a chance the
master will send an additional PING between the two calls to INFO (the line that
fails is the one that samples INFO from both servers). So there's a chance one of
them will have an incremented offset due to PING and the other won't have it yet.

In theory we can wait for the repl_offset to match, but then we risk facing a
situation where that race will hide an offset mis-match. so instead, i think we
should just change repl-ping-replica-period to prevent further pings from being pushed.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-11-01 16:07:08 +02:00
Oran Agra f1f3cceb50 fix valgrind issues with long double module test (#9709) 
The module test in reply.tcl was introduced by #8521 but didn't run until recently (see #9639)
and then it started failing with valgrind.
This is because valgrind uses 64 bit long double (unlike most other platforms that have at least 80 bits)
But besides valgrind, the tests where also incompatible with ARM32, which also uses 64 bit long doubles.

We now use appropriate value to avoid issues with either valgrind or ARM32

In all the double tests, i use 3.141, which is safe since since addReplyDouble uses
`%.17Lg` which is able to represent this value without adding any digits due to precision loss. 

In the long double, since we use `%.17Lf` in ld2string, it preserves 17 significant
digits, rather than 17 digit after the decimal point (like in `%.17Lg`).
So to make these similar, i use value lower than 1 (no digits left of
the period)

Lastly, we have the same issue with TCL (no long doubles) so we read
raw protocol in that test.

Note that the only error before this fix (in both valgrind and ARM32 is this:
```
*** [err]: RM_ReplyWithLongDouble: a float reply in tests/unit/moduleapi/reply.tcl
Expected '3.141' to be equal to '3.14100000000000001' (context: type eval line 2 cmd {assert_equal 3.141 [r rw.longdouble 3.141]} proc ::test)
```
so the changes to debug.c and scripting.tcl aren't really needed, but i consider them a cleanup
(i.e. scripting.c validated a different constant than the one that's sent to it from debug.c).

Another unrelated change is to add the RESP version to the repeated tests in reply.tcl
 2021-11-01 13:41:35 +02:00
罗泽轩 155c291006 Remove duplicate SET_OP_XX definitions in t_set.c. (#4326) 
These definitions already exist in server.h.
 2021-11-01 11:09:29 +02:00
Oran Agra 48d54265ce Fix failing cluster tests (#9707) 
Fix failures introduced by #9695 which was an attempt to solve failures introduced by #9679.
And alternative to #9703 (i didn't like the extra argument to kill_instance).

Reverting #9695.
Instead of stopping AOF on all terminations, stop it only on the two which need it.
Do it as part of the test rather than the infra (it was add that kill_instance used `R`
to communicate to the instance)

Note that the original purpose of these tests was to trigger a crash, but that upsets
valgrind so in redis 6.2 i changed it to use SIGTERM, so i now rename the tests
(remove "kill" and "crash").

Also add some colors to failures, and the word "FAILED" so that it's searchable.

And solve a semi-related race condition in 14-consistency-check.tcl
 2021-10-31 19:22:21 +02:00
Yossi Gottlieb f26e90be0c Use 'gcc' instead of 'ld' to link test modules. (#9710) 
This solves several problems in a more elegant way:

* No need to explicitly use `-lc` on x86_64 when building with `-m32`.
* Avoids issues with undefined floating point emulation funcs on ARM.
 2021-10-31 16:25:57 +02:00
Binbin 033578839b Fix multiple COUNT in LMPOP/BLMPOP/ZMPOP/BZMPOP (#9701) 
The previous code did not check whether COUNT is set.
So we can use `lmpop 2 key1 key2 left count 1 count 2`.

This situation can occur in LMPOP/BLMPOP/ZMPOP/BZMPOP commands.
LMPOP/BLMPOP introduced in #9373, ZMPOP/BZMPOP introduced in #9484.
 2021-10-31 16:10:29 +02:00
lijinliangandlijinliang 215b909c1f fix typo in db.c: synchroneus -> synchronous(2 places) (#9702) 
Co-authored-by: lijinliang <lijl@newdt.cn>
 2021-10-31 16:01:54 +02:00
Rafi Einstein 734cde7e38 Fix memory leak when there's a read error of module aux data from rdb. (#9705) 2021-10-31 15:59:48 +02:00
Wang Yuan 68886de085 Fix timing issue in replication buffer test (#9697) 
Introduced in #9166
 2021-10-29 08:04:12 +03:00
Oran Agra 22a778c880 fix failing cluster tests (#9695) 
When stopping an instance in the cluster tests, disable appendonly first, so that SIGTERM won't be ignored.

Recently in #9679 i change the test infra to use SIGSEGV to kill servers that refuse
the SIGTERM rather than do SIGKILL directly.

This surfaced an issue that i've added in #7725 which changed SIGKILL to SIGTERM (to resolve valgrind issues).
So the current situation in the past months was that sometimes servers refused the
SIGTERM and waited 10 seconds for the SIGKILL, and this commit resolves that (faster termination).
 2021-10-28 12:16:27 +03:00
yoav-steinberg b7afac6bc2 Remove update-jemalloc.sh, it's not needed anymore (#9690) 
We now use git subtree for deps/jemalloc, updating
jemalloc is detailed in deps/README.md
 2021-10-27 14:48:08 +03:00
guybe7 975f51fe16 Add new SLOTSRANGE to subcommands table (#9689) 2021-10-27 10:44:14 +03:00
Wen Hui 5fb4adba65 New Cluster Command: CLUSTER DELSLOTSRANGE and CLUSTER ADDSLOTSRANGE (#9445) 2021-10-26 21:44:33 -07:00
Wang Yuan 37dc2f13b4 Fix not waiting for data loading to complete in AOF tests (#9683) 
Fix timing issue of a new test introduced in #9326
 2021-10-26 14:08:09 +03:00
Wen Hui 43b22f17dc Sentinel: don't log auth-pass value for better security (#9652) 2021-10-26 13:13:12 +03:00
Oran Agra 37559ca79f Fix race condition in lazy free test (#9682) 
The first test exited before all the memory was reclaimed, so when the second test
sampled used_memory, it was too early.
 2021-10-26 13:02:31 +03:00
Oran Agra 665e428435 Testsuite: attempt to find / avoid valgrind warnings of killed processes (#9679) 
I recently started seeing a lot of empty valgrind reports in the daily CI.
i.e. prints showing valgrind header but no leak report, which causes the tests to fail
https://github.com/redis/redis/runs/3991335416?check_suite_focus=true

This commit change 2 things:
* first, considering valgrind is just slow, we used to give processes 60 seconds timeout on shutdown
  instead of 10 seconds we give normally. this commit changes that to 120.
* secondly, when we reach the timeout, we first try to use SIGSEGV so that maybe we'll get a stack
  trace indicating where redis is hang, and we only resort to SIGKILL if double that time passed.

note that if there are indeed hang processes, we will normally not see that in the non-valgrind runs,
since the tests didn't use to detect any failure in that case, and now they will since `crashlog_from_file`
is run after `kill_server`.
 2021-10-26 08:34:30 +03:00
Wang Yuan 9ec3294b97 Add timestamp annotations in AOF (#9326) 
Add timestamp annotation in AOF, one part of #9325.

Enabled with the new `aof-timestamp-enabled` config option.

Timestamp annotation format is "#TS:${timestamp}\r\n"."
TS" is short of timestamp and this method could save extra bytes in AOF.

We can use timestamp annotation for some special functions. 
- know the executing time of commands
- restore data to a specific point-in-time (by using redis-check-rdb to truncate the file)
 2021-10-25 13:08:34 +03:00
Oran Agra 085615af97 Improve code doc of allowed_firstargs following #9504 (#9674) 
Improve code doc for allowed_firstargs (used to be allowed_commands before #9504.
I don't think the text in the code needs to refer to the history (it's not there just for backwards compatibility).
instead it should just describe what it does.
 2021-10-25 13:01:25 +03:00
Guy Korland 6cf6c36937 Replace deprecated REDISMODULE_POSTPONED_ARRAY_LEN in module tests and examples (#9677) 
REDISMODULE_POSTPONED_ARRAY_LEN is deprecated, use REDISMODULE_POSTPONED_LEN instead
 2021-10-25 12:00:43 +03:00
Itamar Haber 00362f2a94 Removes admin acl category from CLIENT TRACKINGINFO (#9662) 
overlooked in #9504
 2021-10-25 11:33:37 +03:00
Shaya PotterandOran Agra 12ce2c3925 Add RM_ReplyWithBigNumber module API (#9639) 
Let modules use additional type of RESP3 response (unused by redis so far)
Also fix tests that where introduced in #8521 but didn't actually run.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-10-25 11:31:20 +03:00
Wang Yuan c1718f9d86 Replication backlog and replicas use one global shared replication buffer (#9166) 
## Background
For redis master, one replica uses one copy of replication buffer, that is a big waste of memory,
more replicas more waste, and allocate/free memory for every reply list also cost much.
If we set client-output-buffer-limit small and write traffic is heavy, master may disconnect with
replicas and can't finish synchronization with replica. If we set  client-output-buffer-limit big,
master may be OOM when there are many replicas that separately keep much memory.
Because replication buffers of different replica client are the same, one simple idea is that
all replicas only use one replication buffer, that will effectively save memory.

Since replication backlog content is the same as replicas' output buffer, now we
can discard replication backlog memory and use global shared replication buffer
to implement replication backlog mechanism.

## Implementation
I create one global "replication buffer" which contains content of replication stream.
The structure of "replication buffer" is similar to the reply list that exists in every client.
But the node of list is `replBufBlock`, which has `id, repl_offset, refcount` fields.
```c
/* Replication buffer blocks is the list of replBufBlock.
 *
 * +--------------+       +--------------+       +--------------+
 * | refcount = 1 |  ...  | refcount = 0 |  ...  | refcount = 2 |
 * +--------------+       +--------------+       +--------------+
 *      |                                            /       \
 *      |                                           /         \
 *      |                                          /           \
 *  Repl Backlog                               Replia_A      Replia_B
 * 
 * Each replica or replication backlog increments only the refcount of the
 * 'ref_repl_buf_node' which it points to. So when replica walks to the next
 * node, it should first increase the next node's refcount, and when we trim
 * the replication buffer nodes, we remove node always from the head node which
 * refcount is 0. If the refcount of the head node is not 0, we must stop
 * trimming and never iterate the next node. */

/* Similar with 'clientReplyBlock', it is used for shared buffers between
 * all replica clients and replication backlog. */
typedef struct replBufBlock {
    int refcount;           /* Number of replicas or repl backlog using. */
    long long id;           /* The unique incremental number. */
    long long repl_offset;  /* Start replication offset of the block. */
    size_t size, used;
    char buf[];
} replBufBlock;
```
So now when we feed replication stream into replication backlog and all replicas, we only need
to feed stream into replication buffer `feedReplicationBuffer`. In this function, we set some fields of
replication backlog and replicas to references of the global replication buffer blocks. And we also
need to check replicas' output buffer limit to free if exceeding `client-output-buffer-limit`, and trim
replication backlog if exceeding `repl-backlog-size`.

When sending reply to replicas, we also need to iterate replication buffer blocks and send its
content, when totally sending one block for replica, we decrease current node count and
increase the next current node count, and then free the block which reference is 0 from the
head of replication buffer blocks.

Since now we use linked list to manage replication backlog, it may cost much time for iterating
all linked list nodes to find corresponding replication buffer node. So we create a rax tree to
store some nodes  for index, but to avoid rax tree occupying too much memory, i record
one per 64 nodes for index.

Currently, to make partial resynchronization as possible as much, we always let replication
backlog as the last reference of replication buffer blocks, backlog size may exceeds our setting
if slow replicas that reference vast replication buffer blocks, and this method doesn't increase
memory usage since they share replication buffer. To avoid freezing server for freeing unreferenced
replication buffer blocks when we need to trim backlog for exceeding backlog size setting,
we trim backlog incrementally (free 64 blocks per call now), and make it faster in
`beforeSleep` (free 640 blocks).

### Other changes
- `mem_total_replication_buffers`: we add this field in INFO command, it means the total
  memory of replication buffers used.
- `mem_clients_slaves`:  now even replica is slow to replicate, and its output buffer memory
  is not 0, but it still may be 0, since replication backlog and replicas share one global replication
  buffer, only if replication buffer memory is more than the repl backlog setting size, we consider
  the excess as replicas' memory. Otherwise, we think replication buffer memory is the consumption
  of repl backlog.
- Key eviction
  Since all replicas and replication backlog share global replication buffer, we think only the
  part of exceeding backlog size the extra separate consumption of replicas.
  Because we trim backlog incrementally in the background, backlog size may exceeds our
  setting if slow replicas that reference vast replication buffer blocks disconnect.
  To avoid massive eviction loop, we don't count the delayed freed replication backlog into
  used memory even if there are no replicas, i.e. we also regard this memory as replicas's memory.
- `client-output-buffer-limit` check for replica clients
  It doesn't make sense to set the replica clients output buffer limit lower than the repl-backlog-size
  config (partial sync will succeed and then replica will get disconnected). Such a configuration is
  ignored (the size of repl-backlog-size will be used). This doesn't have memory consumption
  implications since the replica client will share the backlog buffers memory.
- Drop replication backlog after loading data if needed
  We always create replication backlog if server is a master, we need it because we put DELs in
  it when loading expired keys in RDB, but if RDB doesn't have replication info or there is no rdb,
  it is not possible to support partial resynchronization, to avoid extra memory of replication backlog,
  we drop it.
- Multi IO threads
 Since all replicas and replication backlog use global replication buffer,  if I/O threads are enabled,
  to guarantee data accessing thread safe, we must let main thread handle sending the output buffer
  to all replicas. But before, other IO threads could handle sending output buffer of all replicas.

## Other optimizations
This solution resolve some other problem:
- When replicas disconnect with master since of out of output buffer limit, releasing the output
  buffer of replicas may freeze server if we set big `client-output-buffer-limit` for replicas, but now,
  it doesn't cause freezing.
- This implementation may mitigate reply list copy cost time(also freezes server) when one replication
  has huge reply buffer and another replica can copy buffer for full synchronization. now, we just copy
  reference info, it is very light.
- If we set replication backlog size big, it also may cost much time to copy replication backlog into
  replica's output buffer. But this commit eliminates this problem.
- Resizing replication backlog size doesn't empty current replication backlog content.
 2021-10-25 09:24:31 +03:00
Oran Agra 6b297cd646 Improve errno reporting on fork and fopen rdbLoad failures (#9649) 
I moved a bunch of stats in redisFork to be executed only on successful
fork, since they seem wrong to be done when it failed.
I guess when fork fails it does that immediately, no latency spike.
 2021-10-24 16:52:44 +03:00
Itamar Haber 48e4d77099 Fixes CLUSTER COUNTKEYSINSLOT (#9672) 
Introduced via typo in #9504. 
Also adds a sanity test for coverage.
 2021-10-24 12:32:53 +03:00
Yossi Gottlieb 5f89c1d4f7 Handle cross-compiling when configuring jemalloc. (#9659) 2021-10-24 10:15:32 +03:00
Shaya Potter cf860df599 Fix module blocked clients RESP version (#9634) 
Before this commit, module blocked clients did not carry through the original RESP version, resulting with RESP3 clients receiving unexpected RESP2 replies.
 2021-10-21 14:01:10 +03:00
guybe7 8f745da159 Fix sentinel commands, ACL dictIter leak (#9661) 2021-10-21 12:50:58 +03:00
Yossi Gottlieb 8bf4c2e38c Fix test modules build issue on OS X 11. (#9658) 2021-10-20 21:01:30 +03:00
Oran Agra 7d6744c739 fix new cluster tests issues (#9657) 
Following #9483 the daily CI exposed a few problems.

* The cluster creation code (uses redis-cli) is complicated to test with TLS enabled.
  for now i'm just skipping them since the tests we run there don't really need that kind of coverage
* cluster port binding failures
  note that `find_available_port` already looks for a free cluster port
  but the code in `wait_server_started` couldn't detect the failure of binding
  (the text it greps for wasn't found in the log)
 2021-10-20 15:40:28 +03:00
guybe7 43e736f79b Treat subcommands as commands (#9504) 
## Intro

The purpose is to allow having different flags/ACL categories for
subcommands (Example: CONFIG GET is ok-loading but CONFIG SET isn't)

We create a small command table for every command that has subcommands
and each subcommand has its own flags, etc. (same as a "regular" command)

This commit also unites the Redis and the Sentinel command tables

## Affected commands

CONFIG
Used to have "admin ok-loading ok-stale no-script"
Changes:
1. Dropped "ok-loading" in all except GET (this doesn't change behavior since
there were checks in the code doing that)

XINFO
Used to have "read-only random"
Changes:
1. Dropped "random" in all except CONSUMERS

XGROUP
Used to have "write use-memory"
Changes:
1. Dropped "use-memory" in all except CREATE and CREATECONSUMER

COMMAND
No changes.

MEMORY
Used to have "random read-only"
Changes:
1. Dropped "random" in PURGE and USAGE

ACL
Used to have "admin no-script ok-loading ok-stale"
Changes:
1. Dropped "admin" in WHOAMI, GENPASS, and CAT

LATENCY
No changes.

MODULE
No changes.

SLOWLOG
Used to have "admin random ok-loading ok-stale"
Changes:
1. Dropped "random" in RESET

OBJECT
Used to have "read-only random"
Changes:
1. Dropped "random" in ENCODING and REFCOUNT

SCRIPT
Used to have "may-replicate no-script"
Changes:
1. Dropped "may-replicate" in all except FLUSH and LOAD

CLIENT
Used to have "admin no-script random ok-loading ok-stale"
Changes:
1. Dropped "random" in all except INFO and LIST
2. Dropped "admin" in ID, TRACKING, CACHING, GETREDIR, INFO, SETNAME, GETNAME, and REPLY

STRALGO
No changes.

PUBSUB
No changes.

CLUSTER
Changes:
1. Dropped "admin in countkeysinslots, getkeysinslot, info, nodes, keyslot, myid, and slots

SENTINEL
No changes.

(note that DEBUG also fits, but we decided not to convert it since it's for
debugging and anyway undocumented)

## New sub-command
This commit adds another element to the per-command output of COMMAND,
describing the list of subcommands, if any (in the same structure as "regular" commands)
Also, it adds a new subcommand:
```
COMMAND LIST [FILTERBY (MODULE <module-name>|ACLCAT <cat>|PATTERN <pattern>)]
```
which returns a set of all commands (unless filters), but excluding subcommands.

## Module API
A new module API, RM_CreateSubcommand, was added, in order to allow
module writer to define subcommands

## ACL changes:
1. Now, that each subcommand is actually a command, each has its own ACL id.
2. The old mechanism of allowed_subcommands is redundant
(blocking/allowing a subcommand is the same as blocking/allowing a regular command),
but we had to keep it, to support the widespread usage of allowed_subcommands
to block commands with certain args, that aren't subcommands (e.g. "-select +select|0").
3. I have renamed allowed_subcommands to allowed_firstargs to emphasize the difference.
4. Because subcommands are commands in ACL too, you can now use "-" to block subcommands
(e.g. "+client -client|kill"), which wasn't possible in the past.
5. It is also possible to use the allowed_firstargs mechanism with subcommand.
For example: `+config -config|set +config|set|loglevel` will block all CONFIG SET except
for setting the log level.
6. All of the ACL changes above required some amount of refactoring.

## Misc
1. There are two approaches: Either each subcommand has its own function or all
   subcommands use the same function, determining what to do according to argv[0].
   For now, I took the former approaches only with CONFIG and COMMAND,
   while other commands use the latter approach (for smaller blamelog diff).
2. Deleted memoryGetKeys: It is no longer needed because MEMORY USAGE now uses the "range" key spec.
4. Bugfix: GETNAME was missing from CLIENT's help message.
5. Sentinel and Redis now use the same table, with the same function pointer.
   Some commands have a different implementation in Sentinel, so we redirect
   them (these are ROLE, PUBLISH, and INFO).
6. Command stats now show the stats per subcommand (e.g. instead of stats just
   for "config" you will have stats for "config|set", "config|get", etc.)
7. It is now possible to use COMMAND directly on subcommands:
   COMMAND INFO CONFIG|GET (The pipeline syntax was inspired from ACL, and
   can be used in functions lookupCommandBySds and lookupCommandByCString)
8. STRALGO is now a container command (has "help")

## Breaking changes:
1. Command stats now show the stats per subcommand (see (5) above)
 2021-10-20 11:52:57 +03:00
qetu3790andOran Agra 4962c5526d Release clients blocked on module commands in cluster resharding and down state (#9483) 
Prevent clients from being blocked forever in cluster when they block with their own module command
and the hash slot is migrated to another master at the same time.
These will get a redirection message when unblocked.
Also, release clients blocked on module commands when cluster is down (same as other blocked clients)

This commit adds basic tests for the main (non-cluster) redis test infra that test the cluster.
This was done because the cluster test infra can't handle some common test features,
but most importantly we only build the test modules with the non-cluster test suite.

note that rather than really supporting cluster operations by the test infra, it was added (as dup code)
in two files, one for module tests and one for non-modules tests, maybe in the future we'll refactor that.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-10-19 11:50:37 +03:00
Bjorn Svensson c9fabc2ef0 Move config unixsocketperm to generic configs (#9607) 
Since the size of mode_t is platform dependant we handle the
`unixsocketperm` configuration as a generic int type.
mode_t is either an unsigned int or unsigned short (macOS) and
the range-limits allows for a simple cast to a mode_t.
 2021-10-18 23:58:52 -07:00
Wen Hui 1c2b5f5318 Make Cluster-bus port configurable with new cluster-port config (#9389) 
Make Cluster-bus port configurable with new cluster-port config
 2021-10-18 22:28:27 -07:00
Viktor Söderqvist b7f2a1a217 Add RedisModule_KeyExists (#9600) 
The LRU of the key is not touched. Locically expired keys are
logically not existing, so they're treated as such.
 2021-10-18 22:21:19 +03:00
DarrenJiang13 aa6deff01e add missed error counting (#9646) 
* add: add missed error counting in sentinel.c and cluster.c
 2021-10-18 15:53:10 +03:00
yoav-steinberg 81095b1bd9 Skip Active-defrag edge case test until we fix it. (#9645) 
Test started failing consistently in 32bit builds after upgrading to jemalloc 5.2.1 (#9623).
 2021-10-18 13:28:52 +03:00
Oran Agra c4b4b6c06b Merge pull request #9623 from yoav-steinberg/upgrade_jemalloc_5.2.1 
Upgraded to jemalloc 5.2.1 from 5.1.0.
Cherry picked all relevant fixes (by diffing our 5.1.0 to upstream 5.10 and finding relevant commits).
Details of what was done:

[cherry-picked] fd7d51c 2021-05-03 Resolve nonsense static analysis warnings (Oran Agra)
[cherry-picked] 448c435 2020-09-29 Fix compilation warnings in Lua and jemalloc dependencies (#7785) (YoongHM)
[skipped - already in upstream] 9216b96 2020-09-21 Fix compilation warning in jemalloc's malloc_vsnprintf (#7789) (YoongHM)
[cherry-picked] 88d71f4 2020-05-20 fix a rare active defrag edge case bug leading to stagnation (Oran Agra)
[skipped - already in upstream] 2fec7d9 2019-05-30 Jemalloc: Avoid blocking on background thread lock for stats.
[cherry-picked] 920158e 2018-07-11 Active defrag fixes for 32bit builds (again) (Oran Agra)
[cherry-picked] e8099ca 2018-06-26 add defrag hint support into jemalloc 5 (Oran Agra)
[re-done] 4e729fc 2018-05-24 Generate configure for Jemalloc. (antirez)

Additionally had to do this:
7727cc2 2021-10-10 Fix defrag to support sharded bins in arena (added in v5.2.1) (Yoav Steinberg)

When reviewing please look at all except the first commit which is just replacing 5.1.0 with 5.2.1 sources.
Also I think we should merge this without squashing to preserve the changes we did to to jemalloc.
 2021-10-18 12:45:11 +03:00
Oran Agra 276b460ea9 Attempt to fix a valgrind test failure due to timing (#9643) 
in the past few days i've seen two failures in the valgrind daily test.

*** [err]: slave fails full sync and diskless load swapdb recovers it in tests/integration/replication.tcl
Replica didn't get into loading mode

can't reproduce it, but i'm hoping it's just too slow (to start loading within 5 seconds)
 2021-10-18 10:45:45 +03:00
七飒 afd8c4e007 sdstrim remove excessive check (#4045) 
there is no need to compare the value of ep and sp
```
    sp = start = s;
    // the only way that make ep > sp is sdslen(s) == 0
    // so when ep > sp,must exist ep-sp == -1
    ep = end = s+sdslen(s)-1;
    while(sp <= end && strchr(cset, *sp)) sp++;
    while(ep > sp && strchr(cset, *ep)) ep--;
    // -1 + 1 already equals 0
    len = (sp > ep) ? 0 : ((ep-sp)+1);
```

Signed-off-by: Bo Cai <charpty@gmail.com>
 2021-10-17 20:37:52 +03:00
Ilya Shipitsin 94fded4f4f Code cleanup, resolve an issue identified by cppcheck (#4373) 
[src/bitops.c:512] -> [src/bitops.c:507]: (warning) Either the condition 'if(o&&o->encoding==1)' is redundant or there is possible null pointer dereference: o.

This function has checks for `o` to be null or non-null, so it is odd that it accesses it first..
 2021-10-17 18:48:15 +03:00
Hanna Fadida 61bb044156 Modify mem_usage2 module callback to enable to take sample_size argument (#9612) 
This is useful for approximating size computation of complex module types.
Note that the mem_usage2 callback is new and has not been released yet, which is why we can modify it.
 2021-10-17 17:31:06 +03:00
Oran Agra e7864a2b70 fix typos in module doc / header (#9641) 
the RedisModule_ReplyWithPush prototype was merged by mistake (no such API yet)
 2021-10-17 17:15:27 +03:00
Yossi Gottlieb 6d5a911707 Fix daily failures due to macos-latest change. (#9637) 
* Fix test modules linking on macOS 11.x.
* Use macOS 10.x for FreeBSD VM as VirtualBox is not yet supported on
  11.
 2021-10-17 00:07:27 +03:00
guoxiang1996 3c9e5271c6 Use fcntl(fd,F_FULLFSYNC) instead of fsync on OSX, improve power failure safety (#9545) 
On MacOS calling fsync does not guarantee the cache on the disk itself is flushed.
 2021-10-15 08:44:25 +03:00
Shaya Potter 24b67d5520 Add RM_ReplyWithVerbatimStringType that takes an ext/type arg (#9632) 
Verbatim Stings in RESP3 have a type/extension.
The existing redismoule reply function, hard coded it to "txt".
 2021-10-14 09:53:46 +03:00
Ofir LuzonandOran Agra 49d26a9658 Add sleep interval to redis-cli --scan option (#3751) 
Adding -i option (sleep interval) of repeat and bigkeys to redis-cli --scan.
When the keyspace contains many already expired keys scanning the
dataset with redis-cli --scan can impact the performance

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-10-13 16:54:35 +03:00
Madelyn Olson a6b5d518a9 Improved the reliability of cluster replica sync tests (#9628) 
Improved the reliability of cluster replica sync tests
 2021-10-13 00:06:53 -07:00
Ning Xie 075ac34545 Fix redis-cli SCAN sleep interval for big/hot keys (could have been skipped) (#9624) 
bigkeys sleep is defined each 100 scanned keys, and it is checked it only between scan cycles.
In cases that scan does not return exactly 10 keys it will never sleep.
In addition the comment was sleep each 100 SCANs but it was 100 scanned keys.
 2021-10-12 23:00:49 +03:00
Yossi Gottlieb 922ef86a3b hiredis: improve calloc() overflow fix. (#9630) 
Cherry pick a more complete fix to 0215324a6 that also doesn't leak
memory from latest hiredis.
 2021-10-12 20:16:29 +03:00
Yoav Steinberg 85737e6745 Added jemalloc subtree upgrade instructions. 2021-10-12 12:55:35 +03:00
Oran Agra ed92a3e8ed Resolve nonsense static analysis warnings 2021-10-12 12:55:35 +03:00
YoongHMandOran Agra 5a82940452 Fix compilation warnings in Lua and jemalloc dependencies (#7785) 
- The argument `u` in for `ar` is ignored (and generates warnings since `D` became the default.
  All it does is avoid updating unchanged objects (shouldn't have any impact on our build)
- Enable `LUA_USE_MKSTEMP` to force the use of `mkstemp()` instead of `tmpname()` (which is dead
  code in redis anyway).
- Remove unused variable `c` in `f_parser()`
- Removed misleadingly indented space in `luaL_loadfile()` and ``addfield()`

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-10-12 12:55:35 +03:00
Oran Agra c6a26519a1 fix a rare active defrag edge case bug leading to stagnation 
There's a rare case which leads to stagnation in the defragger, causing
it to keep scanning the keyspace and do nothing (not moving any
allocation), this happens when all the allocator slabs of a certain bin
have the same % utilization, but the slab from which new allocations are
made have a lower utilization.

this commit fixes it by removing the current slab from the overall
average utilization of the bin, and also eliminate any precision loss in
the utilization calculation and move the decision about the defrag to
reside inside jemalloc.

and also add a test that consistently reproduce this issue.
 2021-10-12 12:55:35 +03:00
yoav-steinberg 252981914f XADD - skip rewrite the id arg if it was given and is valid. (#9599) 
When calling `XADD` with a predefined id (instead of `*`) there's no need to run
the code which replaces the supplied id with itself. Only when we pass a wildcard
id we need to do this.
For apps which always supply their own id this is a slight optimization.
 2021-10-11 13:09:18 +03:00
zhaozhao.zz 484a1ad67e master client should ignore proto_max_bulk_len in bitops (#9626) 2021-10-11 13:58:42 +08:00
Yoav Steinberg 908d3bdad9 Fix defrag to support sharded bins in arena (added in v5.2.1) 
See https://github.com/jemalloc/jemalloc/commit/37b89139252db18c95ebce3e0eac67817fa4a8ab
 2021-10-10 18:29:13 +03:00
Oran Agra 91bc78a8b8 Active defrag fixes for 32bit builds (again) 
* overflow in jemalloc fragmentation hint to the defragger
 2021-10-10 18:29:13 +03:00
Oran Agra 29d7f97c96 add defrag hint support into jemalloc 5 2021-10-10 18:29:13 +03:00
Yoav Steinberg 9e5cd2cb26 Generate configure for Jemalloc 5.2.1. 
./autogen.sh --with-version=5.2.1-0-g0
 2021-10-10 18:29:13 +03:00
Yoav Steinberg 4d5911b4e4 Merge commit '220a0f0880419450c9409202aac1fab4b8be0719' as 'deps/jemalloc' 2021-10-10 18:26:48 +03:00
Yoav Steinberg 220a0f0880 Squashed 'deps/jemalloc/' content from commit 886e40bb3 
git-subtree-dir: deps/jemalloc
git-subtree-split: 886e40bb339ec1358a5ff2a52fdb782ca66461cb
 2021-10-10 18:26:48 +03:00
Yoav Steinberg 4a884343f5 Delete old jemalloc before pulling in subtree. 2021-10-10 18:03:38 +03:00
menwen 7ff7536e2c Delete unused 'time' fields from struct bio_job (#9622) 
looks like this field was never actually used and the call to time() is excessive.
 2021-10-10 08:17:54 +03:00
Bjorn Svensson b874c6f1fc Move config logfile to generic config (#9592) 
Move config `logfile` to generic configs
 2021-10-07 22:33:08 -07:00
Bjorn Svensson 54d01e363a Move config cluster-config-file to generic configs (#9597) 2021-10-07 22:32:40 -07:00
yoav-steinberg 834e8843de obuf based eviction tests run until eviction occurs (#9611) 
obuf based eviction tests run until eviction occurs instead of assuming a certain
amount of writes will fill the obuf enough for eviction to occur.
This handles the kernel buffering written data and emptying the obuf even though
no one actualy reads from it.

The tests have a new timeout of 20sec: if the test doesn't pass after 20 sec it'll fail.
Hopefully this enough for our slow CI targets.

This also eliminates the need to skip some tests in TLS.
 2021-10-07 15:43:48 +03:00
Huang Zhw fd135f3e2d Make tracking invalidation messages always after command's reply (#9422) 
Tracking invalidation messages were sometimes sent in inconsistent order,
before the command's reply rather than after.
In addition to that, they were sometimes embedded inside other commands
responses, like MULTI-EXEC and MGET.
 2021-10-07 15:13:42 +03:00
GutovskyMaria d98d1ad574 Hide empty and loading replicas from CLUSTER SLOTS responses (#9287) 
Hide empty and loading replicas from CLUSTER SLOTS responses
 2021-10-06 22:22:27 -07:00
Andy PanViktor SöderqvistOran Agra
 		2391aefd82 Implement anetPipe() to combine creating pipe and setting flags (#9511) 
Implement createPipe() to combine creating pipe and setting flags, also reduce
system calls by prioritizing pipe2() over pipe().

Without createPipe(), we have to call pipe() to create a pipe and then call some
functions (like anetCloexec() and anetNonBlock()) of anet.c to set flags respectively,
which leads to some extra system calls, now we can leverage pipe2() to combine
them and make the process of creating pipe more convergent in createPipe().

Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-10-06 16:08:13 +03:00
yoav-steinberg 123cc1a1bc Test fails when flushdb triggers a bgsave (#9535) 
Flush db and *then* wait for the bgsave to complete.
 2021-10-06 11:50:47 +03:00
yoav-steinberg 5725088ff2 Avoid argv memcpy when queuing a multi command. (#9602) 
When queuing a multi command we duplicated the argv (meaning an alloc
and a memcpy). This isn't needed since we can use the previously allocated
argv and just reset the client objects argv to NULL. This should saves some
memory and is a minor optimization in heavy MULTI/EXEC traffic, especially
if there are lots of arguments.
 2021-10-06 11:39:09 +03:00
Meir Shpilraien (Spielrein) 4fb39b6700 Added module-acquire-GIL latency stats (#9608) 
The new value indicates how long Redis wait to
acquire the GIL after sleep. This can help identify
problems where a module perform some background
operation for a long time (with the GIL held) and
blocks the Redis main thread.
 2021-10-06 11:33:01 +03:00
tzongw f5160ed0aa improve latency when a client is unblocked by module timer (#9593) 
Scenario:
1. client block on command `XREAD BLOCK 0 STREAMS mystream  $`
2. in a module, calling `XADD mystream * field value` via lua from a timer callback
3. client will receive response after some latency up to 100ms

Reason:
When `XADD` signal the key `mystream` as ready, `beforeSleep` in next eventloop will call
`handleClientsBlockedOnKeys` to unblock the client and add pending data to write but not
actually install a write handler, so next redis will block in `aeApiPoll` up to 100ms given `hz`
config as default 10, pending data will be sent in another next eventloop by
`handleClientsWithPendingWritesUsingThreads`.

Calling `handleClientsBlockedOnKeys` before `handleClientsWithPendingWritesUsingThreads`
in `beforeSleep` solves the problem.
 2021-10-06 10:15:03 +03:00
yoav-steinberg 897c7bddf5 Attempt to fix rare pubsub oubuf maxmemory eviction test failure (#9603) 
* Reduce delay between publishes to allow less time to write the obufs.
* More subscribed clients to buffer more data per publish.
* Make sure main connection isn't evicted (it has a large qbuf).
 2021-10-05 18:00:19 +03:00
yoav-steinberg 83478e6102 argv mem leak during multi command execution. (#9598) 
Changes in #9528 lead to memory leak if the command implementation
used rewriteClientCommandArgument inside MULTI-EXEC.

Adding an explicit test for that case since the test that uncovered it
didn't specifically target this scenario
 2021-10-05 12:17:36 +03:00
Meir Shpilraien (Spielrein) 0f8b634cd5 Fix invalid memory write on lua stack overflow (CVE-2021-32626) (#9591) 
When LUA call our C code, by default, the LUA stack has room for 10
elements. In most cases, this is more than enough but sometimes it's not
and the caller must verify the LUA stack size before he pushes elements.

On 3 places in the code, there was no verification of the LUA stack size.
On specific inputs this missing verification could have lead to invalid
memory write:
1. On 'luaReplyToRedisReply', one might return a nested reply that will
   explode the LUA stack.
2. On 'redisProtocolToLuaType', the Redis reply might be deep enough
   to explode the LUA stack (notice that currently there is no such
   command in Redis that returns such a nested reply, but modules might
   do it)
3. On 'ldbRedis', one might give a command with enough arguments to
   explode the LUA stack (all the arguments will be pushed to the LUA
   stack)

This commit is solving all those 3 issues by calling 'lua_checkstack' and
verify that there is enough room in the LUA stack to push elements. In
case 'lua_checkstack' returns an error (there is not enough room in the
LUA stack and it's not possible to increase the stack), we will do the
following:
1. On 'luaReplyToRedisReply', we will return an error to the user.
2. On 'redisProtocolToLuaType' we will exit with panic (we assume this
   scenario is rare because it can only happen with a module).
3. On 'ldbRedis', we return an error.
 2021-10-04 15:17:50 +03:00
Oran Agra 9e3dca8bef Fix mem leak in loading AOF, introduced by #9528 (#9582) 
Recently merged PR introduced a leak when loading AOF files.
This was because argv_len wasn't set, so rewriteClientCommandArgument
would shrink the argv array and updating argc to a small value.
 2021-10-04 12:17:22 +03:00
Oran Agraandmeir@redislabs.com <meir@redislabs.com> b0ca3be2bb Fix protocol parsing on 'ldbReplParseCommand' (CVE-2021-32672) (#9590) 
The protocol parsing on 'ldbReplParseCommand' (LUA debugging)
Assumed protocol correctness. This means that if the following
is given:
*1
$100
test
The parser will try to read additional 94 unallocated bytes after
the client buffer.
This commit fixes this issue by validating that there are actually enough
bytes to read. It also limits the amount of data that can be sent by
the debugger client to 1M so the client will not be able to explode
the memory.

Co-authored-by: meir@redislabs.com <meir@redislabs.com>
 2021-10-04 12:14:12 +03:00
Oran Agraandsundb c5e6a6204c Fix ziplist and listpack overflows and truncations (CVE-2021-32627, CVE-2021-32628) (#9589) 
- fix possible heap corruption in ziplist and listpack resulting by trying to
  allocate more than the maximum size of 4GB.
- prevent ziplist (hash and zset) from reaching size of above 1GB, will be
  converted to HT encoding, that's not a useful size.
- prevent listpack (stream) from reaching size of above 1GB.
- XADD will start a new listpack if the new record may cause the previous
  listpack to grow over 1GB.
- XADD will respond with an error if a single stream record is over 1GB
- List type (ziplist in quicklist) was truncating strings that were over 4GB,
  now it'll respond with an error.

Co-authored-by: sundb <sundbcn@gmail.com>
 2021-10-04 12:11:02 +03:00
Oran Agra fba15850e5 Prevent unauthenticated client from easily consuming lots of memory (CVE-2021-32675) (#9588) 
This change sets a low limit for multibulk and bulk length in the
protocol for unauthenticated connections, so that they can't easily
cause redis to allocate massive amounts of memory by sending just a few
characters on the network.
The new limits are 10 arguments of 16kb each (instead of 1m of 512mb)
 2021-10-04 12:10:31 +03:00
Oran AgraandYossi Gottlieb 0215324a66 Fix redis-cli / redis-sential overflow on some platforms (CVE-2021-32762) (#9587) 
The redis-cli command line tool and redis-sentinel service may be vulnerable
to integer overflow when parsing specially crafted large multi-bulk network
replies. This is a result of a vulnerability in the underlying hiredis
library which does not perform an overflow check before calling the calloc()
heap allocation function.

This issue only impacts systems with heap allocators that do not perform their
own overflow checks. Most modern systems do and are therefore not likely to
be affected. Furthermore, by default redis-sentinel uses the jemalloc allocator
which is also not vulnerable.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
 2021-10-04 12:10:17 +03:00
Oran Agra 7cb89a5a1c Fix Integer overflow issue with intsets (CVE-2021-32687) (#9586) 
The vulnerability involves changing the default set-max-intset-entries
configuration parameter to a very large value and constructing specially
crafted commands to manipulate sets
 2021-10-04 12:09:25 +03:00
yiyuaner 24cc0b984d Fix integer overflow in _sdsMakeRoomFor (CVE-2021-41099) (#9558) 
The existing overflow checks handled the greedy growing, but didn't handle
a case where the addition of the header size is what causes the overflow.
 2021-10-04 11:11:09 +03:00
YaacovHazan 5becb7c9c6 improve the stability and correctness of "Test child sending info" (#9562) 
Since we measure the COW size in this test by changing some keys and reading
the reported COW size, we need to ensure that the "dismiss mechanism" (#8974)
will not free memory and reduce the COW size.

For that, this commit changes the size of the keys to 512B (less than a page).
and because some keys may fall into the same page, we are modifying ten keys
on each iteration and check for at least 50% change in the COW size.
 2021-10-04 10:32:26 +03:00
yoav-steinberg 6f4f31f167 decrby LLONG_MIN caused nagation overflow. (#9577) 
Note that this breaks compatibility because in the past doing:
DECRBY x -9223372036854775808
would succeed (and create an invalid result) and now this returns an error.
 2021-10-03 09:38:05 +03:00
yoav-steinberg 93e8534713 Remove argument count limit, dynamically grow argv. (#9528) 
Remove hard coded multi-bulk limit (was 1,048,576), new limit is INT_MAX.
When client sends an m-bulk that's higher than 1024, we initially only allocate
the argv array for 1024 arguments, and gradually grow that allocation as arguments
are received.
 2021-10-03 09:13:09 +03:00
Binbin dd3ac97ffe Cleanup typos, incorrect comments, and fixed small memory leak in redis-cli (#9153) 
1. Remove forward declarations from header files to functions that do not exist:
hmsetCommand and rdbSaveTime.
2. Minor phrasing fixes in #9519
3. Add missing sdsfree(title) and fix typo in redis-benchmark.
4. Modify some error comments in some zset commands.
5. Fix copy-paste bug comment in syncWithMaster about `ip-address`.
 2021-10-02 22:19:33 -07:00
Viktor Söderqvist 9a3bd07e9f Unify dbSyncDelete and dbAsyncDelete (#9573) 
Just a cleanup to make the code easier to maintain and reduce the risk of something being overlooked.
 2021-10-01 15:49:33 +03:00
Eduardo Semprebon d509675592 Fix error message for replica-serve-stale-data in redis.conf doc (#9519) 
seems that his piece of doc was always wrong (no such error in the code)
 2021-09-30 16:28:25 +03:00
Yunier Pérez 12e4f31d94 Allow to override OPENSSL_PREFIX (#9567) 
While the original issue was on Linux, this should work for other
platforms as well.
 2021-09-30 15:51:19 +03:00
Hanna Fadida ffafb434fb Modules: add RM_LoadDataTypeFromStringEncver (#9537) 
adding an advanced api to enable loading data that was sereialized with a specific encoding version
 2021-09-30 11:21:32 +03:00
yoav-steinberg d715655f16 verbose debug print in test to debug rare CI failure. (#9563) 2021-09-29 17:10:05 +03:00
Wen Hui 2c38caa176 adding missing error check for fstat (#9532) 2021-09-28 21:10:33 -07:00
Ozan Tezcan 4be2dd6ab9 Use __common__ attribute in redismodule.h for Clang C builds (#9541) 2021-09-27 09:51:33 +03:00
Oran Agra 5a4ab7c7d2 Fix stream sanitization for non-int first value (#9553) 
This was recently broken in #9321 when we validated stream IDs to be
integers but did that after to the stepping next record instead of before.
 2021-09-26 18:46:22 +03:00
yoav-steinberg 6600253046 Client eviction ci issues (#9549) 
Fixing CI test issues introduced in #8687
- valgrind warnings in readQueryFromClient when client was freed by processInputBuffer
- adding DEBUG pause-cron for tests not to be time dependent.
- skipping a test that depends on socket buffers / events not compatible with TLS
- making sure client got subscribed by not using deferring client
 2021-09-26 17:45:02 +03:00
Yossi Gottlieb 0af7fe2cab Add --skipfile and --skiptest regex support. (#9555) 
Empty patterns are not considered and skipped.
Also, improve help text.
 2021-09-26 15:12:37 +03:00
chenyang8094 7c1f9ef503 Fix obtain the AOF file length error when load AOF (#9510) 
this was a regression from #9012 (not released yet)
 2021-09-26 10:24:52 +03:00
Ozan Tezcan 3ff56a6dde Fix crash due to free() call for a string literal in redis-benchmark (#9546) 2021-09-24 22:03:19 +03:00
sundb 9967a53f4c Use dictGetFairRandomKey() for HRANDFIELD,SRANDMEMBER,ZRANDMEMBER (#9538) 
In the `HRANDFIELD`, `SRANDMEMBER` and `ZRANDMEMBER` commands,
There are some strategies that could in some rare cases return an unfair random.
these cases are where s small dict happens be be hashed unevenly.

Specifically when `count*ZRANDMEMBER_SUB_STRATEGY_MUL > size`,
using `dictGetRandomKey` to randomize from a dict will result in an unfair random result.
 2021-09-24 17:36:26 +03:00
Huang Zhw bdecbd30df Fix test randstring, compare string and int is wrong. (#9544) 
This will cause the generated string containing "\".
Fixes a broken change in #8687
 2021-09-24 16:58:38 +03:00
Huang ZhwandViktor Söderqvist f0fab99d6f Minor optimize getMaxmemoryState, when server.maxmemory is not set (#9533) 
Minor optimize getMaxmemoryState, when server.maxmemory is not set,
don't count AOF and replicas buffers.

Co-authored-by: Viktor Söderqvist <viktor@zuiderkwast.se>
 2021-09-23 17:12:11 +03:00
Yossi Gottlieb bebc7f8470 Add RM_TrimStringAllocation(). (#9540) 
This commit makes it possible to explicitly trim the allocation of a
RedisModuleString.

Currently, Redis automatically trims strings that have been retained by
a module command when it returns. However, this is not thread safe and
may result with corruption in threaded modules.

Supporting explicit trimming offers a backwards compatible workaround to
this problem.
 2021-09-23 15:00:37 +03:00
yoav-steinbergandOran Agra 2753429c99 Client eviction (#8687) 
### Description
A mechanism for disconnecting clients when the sum of all connected clients is above a
configured limit. This prevents eviction or OOM caused by accumulated used memory
between all clients. It's a complimentary mechanism to the `client-output-buffer-limit`
mechanism which takes into account not only a single client and not only output buffers
but rather all memory used by all clients.

#### Design
The general design is as following:
* We track memory usage of each client, taking into account all memory used by the
  client (query buffer, output buffer, parsed arguments, etc...). This is kept up to date
  after reading from the socket, after processing commands and after writing to the socket.
* Based on the used memory we sort all clients into buckets. Each bucket contains all
  clients using up up to x2 memory of the clients in the bucket below it. For example up
  to 1m clients, up to 2m clients, up to 4m clients, ...
* Before processing a command and before sleep we check if we're over the configured
  limit. If we are we start disconnecting clients from larger buckets downwards until we're
  under the limit.

#### Config
`maxmemory-clients` max memory all clients are allowed to consume, above this threshold
we disconnect clients.
This config can either be set to 0 (meaning no limit), a size in bytes (possibly with MB/GB
suffix), or as a percentage of `maxmemory` by using the `%` suffix (e.g. setting it to `10%`
would mean 10% of `maxmemory`).

#### Important code changes
* During the development I encountered yet more situations where our io-threads access
  global vars. And needed to fix them. I also had to handle keeps the clients sorted into the
  memory buckets (which are global) while their memory usage changes in the io-thread.
  To achieve this I decided to simplify how we check if we're in an io-thread and make it
  much more explicit. I removed the `CLIENT_PENDING_READ` flag used for checking
  if the client is in an io-thread (it wasn't used for anything else) and just used the global
  `io_threads_op` variable the same way to check during writes.
* I optimized the cleanup of the client from the `clients_pending_read` list on client freeing.
  We now store a pointer in the `client` struct to this list so we don't need to search in it
  (`pending_read_list_node`).
* Added `evicted_clients` stat to `INFO` command.
* Added `CLIENT NO-EVICT ON|OFF` sub command to exclude a specific client from the
  client eviction mechanism. Added corrosponding 'e' flag in the client info string.
* Added `multi-mem` field in the client info string to show how much memory is used up
  by buffered multi commands.
* Client `tot-mem` now accounts for buffered multi-commands, pubsub patterns and
  channels (partially), tracking prefixes (partially).
* CLIENT_CLOSE_ASAP flag is now handled in a new `beforeNextClient()` function so
  clients will be disconnected between processing different clients and not only before sleep.
  This new function can be used in the future for work we want to do outside the command
  processing loop but don't want to wait for all clients to be processed before we get to it.
  Specifically I wanted to handle output-buffer-limit related closing before we process client
  eviction in case the two race with each other.
* Added a `DEBUG CLIENT-EVICTION` command to print out info about the client eviction
  buckets.
* Each client now holds a pointer to the client eviction memory usage bucket it belongs to
  and listNode to itself in that bucket for quick removal.
* Global `io_threads_op` variable now can contain a `IO_THREADS_OP_IDLE` value
  indicating no io-threading is currently being executed.
* In order to track memory used by each clients in real-time we can't rely on updating
  these stats in `clientsCron()` alone anymore. So now I call `updateClientMemUsage()`
  (used to be `clientsCronTrackClientsMemUsage()`) after command processing, after
  writing data to pubsub clients, after writing the output buffer and after reading from the
  socket (and maybe other places too). The function is written to be fast.
* Clients are evicted if needed (with appropriate log line) in `beforeSleep()` and before
  processing a command (before performing oom-checks and key-eviction).
* All clients memory usage buckets are grouped as follows:
  * All clients using less than 64k.
  * 64K..128K
  * 128K..256K
  * ...
  * 2G..4G
  * All clients using 4g and up.
* Added client-eviction.tcl with a bunch of tests for the new mechanism.
* Extended maxmemory.tcl to test the interaction between maxmemory and
  maxmemory-clients settings.
* Added an option to flag a numeric configuration variable as a "percent", this means that
  if we encounter a '%' after the number in the config file (or config set command) we
  consider it as valid. Such a number is store internally as a negative value. This way an
  integer value can be interpreted as either a percent (negative) or absolute value (positive).
  This is useful for example if some numeric configuration can optionally be set to a percentage
  of something else.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-09-23 14:02:16 +03:00
YaacovHazan a56d4533b7 Adding ACL support for modules (#9309) 
This commit introduced a new flag to the RM_Call:
'C' - Check if the command can be executed according to the ACLs associated with it.

Also, three new API's added to check if a command, key, or channel can be executed or accessed
by a user, according to the ACLs associated with it.
- RM_ACLCheckCommandPerm
- RM_ACLCheckKeyPerm
- RM_ACLCheckChannelPerm

The user for these API's is a RedisModuleUser object, that for a Module user returned by the RM_CreateModuleUser API, or for a general ACL user can be retrieved by these two new API's:
- RM_GetCurrentUserName - Retrieve the user name of the client connection behind the current context.
- RM_GetModuleUserFromUserName - Get a RedisModuleUser from a user name

As a result of getting a RedisModuleUser from name, it can now also access the general ACL users (not just ones created by the module).
This mean the already existing API RM_SetModuleUserACL(), can be used to change the ACL rules for such users.
 2021-09-23 08:52:56 +03:00
Binbin 14d6abd8e9 Add ZMPOP/BZMPOP commands. (#9484) 
This is similar to the recent addition of LMPOP/BLMPOP (#9373), but zset.

Syntax for the new ZMPOP command:
`ZMPOP numkeys [<key> ...] MIN|MAX [COUNT count]`

Syntax for the new BZMPOP command:
`BZMPOP timeout numkeys [<key> ...] MIN|MAX [COUNT count]`

Some background:
- ZPOPMIN/ZPOPMAX take only one key, and can return multiple elements.
- BZPOPMIN/BZPOPMAX take multiple keys, but return only one element from just one key.
- ZMPOP/BZMPOP can take multiple keys, and can return multiple elements from just one key.

Note that ZMPOP/BZMPOP can take multiple keys, it eventually operates on just on key.
And it will propagate as ZPOPMIN or ZPOPMAX with the COUNT option.

As new commands, if we can not pop any elements, the response like:
- ZMPOP: Return a NIL in both RESP2 and RESP3, unlike ZPOPMIN/ZPOPMAX return emptyarray.
- BZMPOP: Return a NIL in both RESP2 and RESP3 when timeout is reached, like BZPOPMIN/BZPOPMAX.

For the normal response is nested arrays in RESP2 and RESP3:
```
ZMPOP/BZMPOP
1) keyname
2) 1) 1) member1
      2) score1
   2) 1) member2
      2) score2

In RESP2:
1) "myzset"
2) 1) 1) "three"
      2) "3"
   2) 1) "two"
      2) "2"

In RESP3:
1) "myzset"
2) 1) 1) "three"
      2) (double) 3
   2) 1) "two"
      2) (double) 2
```
 2021-09-23 08:34:40 +03:00
chenyang8094 59c9716f96 Fix redis-check-rdb rdb_type_string initialization error (#9534) 2021-09-22 10:40:31 +03:00
Oran Agra 5f7789d329 tune lazyfree test timeout (#9527) 
i've seen this CI failure a couple of times on MacOS:

*** [err]: lazy free a stream with all types of metadata in tests/unit/lazyfree.tcl
lazyfree isn't done

only reason i can think of is that 500ms is sometimes not enough on slow systems.
 2021-09-22 09:48:44 +03:00
Wen Hui 572abee6f1 Add const for relevant method parameters in redis-benchmark (#9516) 2021-09-20 09:32:33 -07:00
郭伟光 67fcbdf8a1 replace redis-trib to redis-cli since it is no longer available (#9525) 2021-09-20 09:30:22 -07:00
Wen Hui 7c376398b1 CLIENT LIST / INFO show resp version (#9508) 2021-09-19 12:13:46 +03:00
Oran Agra 16be742b08 fix replication test failure, probing the wrong log file (#9513) 2021-09-19 12:07:04 +03:00
Binbin f898a9e97d Adds limit to SINTERCARD/ZINTERCARD. (#9425) 
Implements the [LIMIT limit] variant of SINTERCARD/ZINTERCARD.
Now with the LIMIT, we can stop the searching when cardinality
reaching the limit, and return the cardinality ASAP.

Note that in SINTERCARD, the old synatx was: `SINTERCARD key [key ...]`
In order to add a optional parameter, we must break the old synatx.
So the new syntax of SINTERCARD will be consistent with ZINTERCARD.
New syntax: `SINTERCARD numkeys key [key ...] [LIMIT limit]`.

Note that this means that SINTERCARD has a different syntax than
SINTER and SINTERSTORE (taking numkeys argument)

As for ZINTERCARD, we can easily add a optional parameter to it.
New syntax: `ZINTERCARD numkeys key [key ...] [LIMIT limit]`
 2021-09-16 14:07:08 +03:00
guybe7 08f4e1335c createSharedObjects: zopomin and zpopmax apeear twice (#9505) 
Introduced by https://github.com/redis/redis/pull/9502
 2021-09-15 15:29:35 +03:00
Wen Hui 53ad5627b7 Sentinel: Fix failed daily tests, due to race condition (#9501) 2021-09-15 13:39:50 +03:00
yancz2000 157454ffa4 Support tclsh 8.7 (#9500) 2021-09-15 13:04:31 +03:00
guybe7 7759ec7c43 Cleanup: propagate and alsoPropagate do not need redisCommand (#9502) 
The `cmd` argument was completely unused, and all the code that bothered to pass it was unnecessary.
This is a prepartion for a future commit that treats subcommands as commands
 2021-09-15 12:53:42 +03:00
guybe7 03fcc211de A better approach for COMMAND INFO for movablekeys commands (#8324) 
Fix #7297

The problem:

Today, there is no way for a client library or app to know the key name indexes for commands such as
ZUNIONSTORE/EVAL and others with "numkeys", since COMMAND INFO returns no useful info for them.

For cluster-aware redis clients, this requires to 'patch' the client library code specifically for each of these commands or to
resolve each execution of these commands with COMMAND GETKEYS.

The solution:

Introducing key specs other than the legacy "range" (first,last,step)

The 8th element of the command info array, if exists, holds an array of key specs. The array may be empty, which indicates
the command doesn't take any key arguments or may contain one or more key-specs, each one may leads to the discovery
of 0 or more key arguments.

A client library that doesn't support this key-spec feature will keep using the first,last,step and movablekeys flag which will
obviously remain unchanged.

A client that supports this key-specs feature needs only to look at the key-specs array. If it finds an unrecognized spec, it
must resort to using COMMAND GETKEYS if it wishes to get all key name arguments, but if all it needs is one key in order
to know which cluster node to use, then maybe another spec (if the command has several) can supply that, and there's no
need to use GETKEYS.

Each spec is an array of arguments, first one is the spec name, the second is an array of flags, and the third is an array
containing details about the spec (specific meaning for each spec type)
The initial flags we support are "read" and "write" indicating if the keys that this key-spec finds are used for read or for write.
clients should ignore any unfamiliar flags.

In order to easily find the positions of keys in a given array of args we introduce keys specs. There are two logical steps of
key specs:
1. `start_search`: Given an array of args, indicate where we should start searching for keys
2. `find_keys`: Given the output of start_search and an array of args, indicate all possible indices of keys.

### start_search step specs
- `index`: specify an argument index explicitly
  - `index`: 0 based index (1 means the first command argument)
- `keyword`: specify a string to match in `argv`. We should start searching for keys just after the keyword appears.
  - `keyword`: the string to search for
  - `start_search`: an index from which to start the keyword search (can be negative, which means to search from the end)

Examples:
- `SET` has start_search of type `index` with value `1`
- `XREAD` has start_search of type `keyword` with value `[“STREAMS”,1]`
- `MIGRATE` has start_search of type `keyword` with value `[“KEYS”,-2]`

### find_keys step specs
- `range`: specify `[count, step, limit]`.
  - `lastkey`: index of the last key. relative to the index returned from begin_search. -1 indicating till the last argument, -2 one before the last
  - `step`: how many args should we skip after finding a key, in order to find the next one
  - `limit`: if count is -1, we use limit to stop the search by a factor. 0 and 1 mean no limit. 2 means ½ of the remaining args, 3 means ⅓, and so on.
- “keynum”: specify `[keynum_index, first_key_index, step]`.
  - `keynum_index`: is relative to the return of the `start_search` spec.
  - `first_key_index`: is relative to `keynum_index`.
  - `step`: how many args should we skip after finding a key, in order to find the next one

Examples:
- `SET` has `range` of `[0,1,0]`
- `MSET` has `range` of `[-1,2,0]`
- `XREAD` has `range` of `[-1,1,2]`
- `ZUNION` has `start_search` of type `index` with value `1` and `find_keys` of type `keynum` with value `[0,1,1]`
- `AI.DAGRUN` has `start_search` of type `keyword` with value `[“LOAD“,1]` and `find_keys` of type `keynum` with value
  `[0,1,1]` (see https://oss.redislabs.com/redisai/master/commands/#aidagrun)

Note: this solution is not perfect as the module writers can come up with anything, but at least we will be able to find the key
args of the vast majority of commands.
If one of the above specs can’t describe the key positions, the module writer can always fall back to the `getkeys-api` option.

Some keys cannot be found easily (`KEYS` in `MIGRATE`: Imagine the argument for `AUTH` is the string “KEYS” - we will
start searching in the wrong index). 
The guarantee is that the specs may be incomplete (`incomplete` will be specified in the spec to denote that) but we never
report false information (assuming the command syntax is correct).
For `MIGRATE` we start searching from the end - `startfrom=-1` - and if one of the keys is actually called "keys" we will
report only a subset of all keys - hence the `incomplete` flag.
Some `incomplete` specs can be completely empty (i.e. UNKNOWN begin_search) which should tell the client that
COMMAND GETKEYS (or any other way to get the keys) must be used (Example: For `SORT` there is no way to describe
the STORE keyword spec, as the word "store" can appear anywhere in the command).

We will expose these key specs in the `COMMAND` command so that clients can learn, on startup, where the keys are for
all commands instead of holding hardcoded tables or use `COMMAND GETKEYS` in runtime.

Comments:
1. Redis doesn't internally use the new specs, they are only used for COMMAND output.
2. In order to support the current COMMAND INFO format (reply array indices 4, 5, 6) we created a synthetic range, called
   legacy_range, that, if possible, is built according to the new specs.
3. Redis currently uses only getkeys_proc or the legacy_range to get the keys indices (in COMMAND GETKEYS for
   example).

"incomplete" specs:
the command we have issues with are MIGRATE, STRALGO, and SORT
for MIGRATE, because the token KEYS, if exists, must be the last token, we can search in reverse. it one of the keys is
actually the string "keys" will return just a subset of the keys (hence, it's "incomplete")
for SORT and STRALGO we can use this heuristic (the keys can be anywhere in the command) and therefore we added a
key spec that is both "incomplete" and of "unknown type"

if a client encounters an "incomplete" spec it means that it must find a different way (either COMMAND GETKEYS or have
its own parser) to retrieve the keys.
please note that all commands, apart from the three mentioned above, have "complete" key specs
 2021-09-15 11:10:29 +03:00
filipe oliveiraandyoav-steinberg b5a879e1c2 Added URI support to redis-benchmark (cli and benchmark share the same uri-parsing methods) (#9314) 
- Add `-u <uri>` command line option to support `redis://` URI scheme.
- included server connection information object (`struct cliConnInfo`),
  used to describe an ip:port pair, db num user input, and user:pass to
  avoid a large number of function arguments.
- Using sds on connection info strings for redis-benchmark/redis-cli

Co-authored-by: yoav-steinberg <yoav@monfort.co.il>
 2021-09-14 19:45:06 +03:00
Viktor Söderqvist ea36d4de17 Modules: Add remaining list API functions (#8439) 
List functions operating on elements by index:

* RM_ListGet
* RM_ListSet
* RM_ListInsert
* RM_ListDelete

Iteration is done using a simple for loop over indices.
The index based functions use an internal iterator as an optimization.
This is explained in the docs:

```
 * Many of the list functions access elements by index. Since a list is in
 * essence a doubly-linked list, accessing elements by index is generally an
 * O(N) operation. However, if elements are accessed sequentially or with
 * indices close together, the functions are optimized to seek the index from
 * the previous index, rather than seeking from the ends of the list.
 *
 * This enables iteration to be done efficiently using a simple for loop:
 *
 *     long n = RM_ValueLength(key);
 *     for (long i = 0; i < n; i++) {
 *         RedisModuleString *elem = RedisModule_ListGet(key, i);
 *         // Do stuff...
 *     }
```
 2021-09-14 17:48:06 +03:00
sundb 1376d83363 Fix memory leak due to missing freeCallback in blockonbackground moduleapi test (#9499) 
Before #9497, before redis-server was shut down, we did not manually shut down all the clients,
which would have prevented valgrind from detecting a memory leak in the client's argc.
 2021-09-14 15:14:09 +03:00
Altaf hussain f041990f2a fix typo in redis.conf. Interfece changed to interface (#9498) 2021-09-13 20:07:50 +03:00
yoav-steinberg 4c7827588d Fixed leaked client for "start_server" when running in --loop (#9497) 
* On `kill_server` make sure we close the default `"client"` connection.
* Don't reconnect when trying to execute the client's `close` command.
* On `restart_server` make sure to remove the (closed) default `"client"` after killing the old server.
 2021-09-13 18:16:47 +03:00
zhaozhao.zz 794442b130 PSYNC2: make partial sync possible after master reboot (#8015) 
The main idea is how to allow a master to load replication info from RDB file when rebooting, if master can load replication info it means that replicas may have the chance to psync with master, it can save much traffic.

The key point is we need guarantee safety and consistency, so there
are two differences between master and replica:

1. master would load the replication info as secondary ID and
   offset, in case other masters have the same replid.
2. when master loading RDB, it would propagate expired keys as DEL
   command to replication backlog, then replica can receive these
   commands to delete stale keys.
   p.s. the expired keys when RDB loading is useful for users, so
   we show it as `rdb_last_load_keys_expired` and `rdb_last_load_keys_loaded` in info persistence.

Moreover, after load replication info, master should update
`no_replica_time` in case loading RDB cost too long time.
 2021-09-13 15:39:11 +08:00
Huang Zhw 75dd230994 bitpos/bitcount add bit index (#9324) 
Make bitpos/bitcount support bit index:

```
BITPOS key bit [start [end [BIT|BYTE]]]
BITCOUNT key [start end [BIT|BYTE]]
```

The default behavior is `BYTE`, so these commands are still compatible with old.
 2021-09-12 11:31:22 +03:00
David CARLIER 418c2e7931 TLS build fix on OpenBSD when built with LibreSSL. (#9486) 2021-09-11 22:54:09 +03:00
zhaozhao.zz d7fa44f4da init client pause value in more appropriate place (#9479) 2021-09-10 14:02:45 +08:00
Meir Shpilraien (Spielrein) 05e6b97bed Fix RedisModule_Call tests on 32bit (#9481) 2021-09-09 23:03:02 +03:00
sundb 3ca6972ecd Replace all usage of ziplist with listpack for t_zset (#9366) 
Part two of implementing #8702 (zset), after #8887.

## Description of the feature
Replaced all uses of ziplist with listpack in t_zset, and optimized some of the code to optimize performance.

## Rdb format changes
New `RDB_TYPE_ZSET_LISTPACK` rdb type.

## Rdb loading improvements:
1) Pre-expansion of dict for validation of duplicate data for listpack and ziplist.
2) Simplifying the release of empty key objects when RDB loading.
3) Unify ziplist and listpack data verify methods for zset and hash, and move code to rdb.c.

## Interface changes
1) New `zset-max-listpack-entries` config is an alias for `zset-max-ziplist-entries` (same with `zset-max-listpack-value`).
2) OBJECT ENCODING will return listpack instead of ziplist.

## Listpack improvements:
1) Add `lpDeleteRange` and `lpDeleteRangeWithEntry` functions to delete a range of entries from listpack.
2) Improve the performance of `lpCompare`, converting from string to integer is faster than converting from integer to string.
3) Replace `snprintf` with `ll2string` to improve performance in converting numbers to strings in `lpGet()`.

## Zset improvements:
1) Improve the performance of `zzlFind` method, use `lpFind` instead of `lpCompare` in a loop.
2) Use `lpDeleteRangeWithEntry` instead of `lpDelete` twice to delete a element of zset.

## Tests
1) Add some unittests for `lpDeleteRange` and `lpDeleteRangeWithEntry` function.
2) Add zset RDB loading test.
3) Add benchmark test for `lpCompare` and `ziplsitCompare`.
4) Add empty listpack zset corrupt dump test.
 2021-09-09 18:18:53 +03:00
Madelyn Olson 86b0de5c41 Remove redundant validation and prevent duplicate users during ACL load (#9330) 
Throw an error when a user is provided multiple times on the command line instead of silently throwing one of them away.
Remove unneeded validation for validating users on ACL load.
 2021-09-09 07:40:33 -07:00
yancz2000 47c001dde6 Add make test-cluster option (#9478) 
Add make test-cluster option
 2021-09-09 06:52:21 -07:00
yvette903 f560531d5b Fix: client pause uses an old timeout (#9477) 
A write request may be paused unexpectedly because `server.client_pause_end_time` is old.

**Recreate this:**
redis-cli -p 6379
127.0.0.1:6379> client pause 500000000 write
OK
127.0.0.1:6379> client unpause
OK
127.0.0.1:6379> client pause 10000 write
OK
127.0.0.1:6379> set key value

The write request `set key value` is paused util  the timeout of 500000000 milliseconds was reached.

**Fix:**
reset `server.client_pause_end_time` = 0 in `unpauseClients`
 2021-09-09 13:44:48 +03:00
Kamil Cudnik 7f88923b40 Lua: Use all characters to calculate string hash (#9449) 
For a lot of long strings which have same prefix which extends beyond
hashing limit, there will be many hash collisions which result in
performance degradation using commands like KEYS
 2021-09-09 12:47:26 +03:00
Binbin c50af0aeba Add LMPOP/BLMPOP commands. (#9373) 
We want to add COUNT option for BLPOP.
But we can't do it without breaking compatibility due to the command arguments syntax.
So this commit introduce two new commands.

Syntax for the new LMPOP command:
`LMPOP numkeys [<key> ...] LEFT|RIGHT [COUNT count]`

Syntax for the new BLMPOP command:
`BLMPOP timeout numkeys [<key> ...] LEFT|RIGHT [COUNT count]`

Some background:
- LPOP takes one key, and can return multiple elements.
- BLPOP takes multiple keys, but returns one element from just one key.
- LMPOP can take multiple keys and return multiple elements from just one key.

Note that LMPOP/BLMPOP  can take multiple keys, it eventually operates on just one key.
And it will propagate as LPOP or RPOP with the COUNT option.

As a new command, it still return NIL if we can't pop any elements.
For the normal response is nested arrays in RESP2 and RESP3, like:
```
LMPOP/BLMPOP 
1) keyname
2) 1) element1
   2) element2
```
I.e. unlike BLPOP that returns a key name and one element so it uses a flat array,
and LPOP that returns multiple elements with no key name, and again uses a flat array,
this one has to return a nested array, and it does for for both RESP2 and RESP3 (like SCAN does)

Some discuss can see: #766 #8824
 2021-09-09 12:02:33 +03:00
Huang Zhw 216f168b2b Add INFO total_active_defrag_time and current_active_defrag_time (#9377) 
Add two INFO metrics:
```
total_active_defrag_time:12345
current_active_defrag_time:456
```
`current_active_defrag_time` if greater than 0, means how much time has
passed since active defrag started running. If active defrag stops, this metric is reset to 0.
`total_active_defrag_time` means total time the fragmentation
was over the defrag threshold since the server started.

This is a followup PR for #9031
 2021-09-09 11:38:10 +03:00
Wang Yuan cee3d67f50 Delay to discard cached master when full synchronization (#9398) 
* Delay to discard cache master when full synchronization
* Don't disconnect with replicas before loading transferred RDB when full sync

Previously, once replica need to start full synchronization with master,
it will discard cached master whatever full synchronization is failed or
not. 
Now we discard cached master only when transferring RDB is finished
and start to change data space, this make replica could start partial
resynchronization with another new master if new master is failed
during full synchronization.
 2021-09-09 11:32:29 +03:00
chenyang8094 bc0c22fabc Fix callReplyParseCollection memleak when use AutoMemory (#9446) 
When parsing an array type reply, ctx will be lost when recursively parsing its
elements, which will cause a memory leak in automemory mode.

This is a result of the changes in #9202

Add test for callReplyParseCollection fix
 2021-09-09 11:03:05 +03:00
Itamar Haber 7c80a65484 Adds the "bitmap" category to redis-cli help(#9474) 2021-09-09 10:16:30 +03:00
chenyang8094 7a0e668560 Add stdlib.h for RedisModule_Assert (#9470) 2021-09-08 23:22:01 +03:00
zhaozhao.zz 1b83353dc3 Fix wrong offset when replica pause (#9448) 
When a replica paused, it would not apply any commands event the command comes from master, if we feed the non-applied command to replication stream, the replication offset would be wrong, and data would be lost after failover(since replica's `master_repl_offset` grows but command is not applied).

To fix it, here are the changes:
* Don't update replica's replication offset or propagate commands to sub-replicas when it's paused in `commandProcessed`.
* Show `slave_read_repl_offset` in info reply.
* Add an assert to make sure master client should never be blocked unless pause or module (some modules may use block way to do background (parallel) processing and forward original block module command to the replica, it's not a good way but it can work, so the assert excludes module now, but someday in future all modules should rewrite block command to propagate like what `BLPOP` does).
 2021-09-08 16:07:25 +08:00
gavinshark 74d9a35621 fix Readme, fork in rdb will create a process, a not thread (#7436) 2021-09-07 14:33:48 +03:00
Viktor Söderqvist 547c3405d4 Optimize quicklistIndex to seek from the nearest end (#9454) 
Until now, giving a negative index seeks from the end of a list and a
positive seeks from the beginning. This change makes it seek from
the nearest end, regardless of the sign of the given index.

quicklistIndex is used by all list commands which operate by index.

LINDEX key 999999 in a list if 1M elements is greately optimized by
this change. Latency is cut by 75%.

LINDEX key -1000000 in a list of 1M elements, likewise.

LRANGE key -1 -1 is affected by this, since LRANGE converts the
indices to positive numbers before seeking.

The tests for corrupt dumps are updated to make sure the corrup
data is seeked in the same direction as before.
 2021-09-06 09:12:38 +03:00
Wen Hui 763fd09416 Speed up sentinel tests (#9408) 
Use sentinel debug to reduce default timeouts and allow tests to execute faster.
 2021-09-05 13:26:29 +03:00
Madelyn Olson 8b8f05c86c Add test verifying PUBSUB NUMPAT behavior (#9209) 2021-09-03 15:52:39 -07:00
guybe7 6aa2285e32 Fix two minor bugs (MIGRATE key args and getKeysUsingCommandTable) (#9455) 
1. MIGRATE has a potnetial key arg in argv[3]. It should be reflected in the command table.
2. getKeysUsingCommandTable should never free getKeysResult, it is always freed by the caller)
   The reason we never encountered this double-free bug is that almost always getKeysResult
   uses the statis buffer and doesn't allocate a new one.
 2021-09-02 17:19:27 +03:00
sundb 306a5ccd2d Fix the timing of read and write events under kqueue (#9416) 
Normally we execute the read event first and then the write event.
When the barrier is set, we will do it reverse.
However, under `kqueue`, if an `fd` has both read and write events,
reading the event using `kevent` will generate two events, which will
result in uncontrolled read and write timing.

This also means that the guarantees of AOF `appendfsync` = `always` are
not met on MacOS without this fix.

The main change to this pr is to cache the events already obtained when reading
them, so that if the same `fd` occurs again, only the mask in the cache is updated,
rather than a new event is generated.

This was exposed by the following test failure on MacOS:
```
*** [err]: AOF fsync always barrier issue in tests/integration/aof.tcl
Expected 544 != 544 (context: type eval line 26 cmd {assert {$size1 != $size2}} proc ::test)
```
 2021-09-02 11:07:51 +03:00
Yossi Gottlieb c9931ddba5 Use fchmod to update command history file. (#9447) 
This is considered a safer approach as it prevents a race condition that
could lead to chmod executed on a different file.

Not a major risk, but CodeQL alerted this so it makes sense to fix.
 2021-09-02 10:00:00 +03:00
Viktor SöderqvistandJim Brunner f24c63a292 Slot-to-keys using dict entry metadata (#9356) 
* Enhance dict to support arbitrary metadata carried in dictEntry

Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>

* Rewrite slot-to-keys mapping to linked lists using dict entry metadata

This is a memory enhancement for Redis Cluster.

The radix tree slots_to_keys (which duplicates all key names prefixed with their
slot number) is replaced with a linked list for each slot. The dict entries of
the same cluster slot form a linked list and the pointers are stored as metadata
in each dict entry of the main DB dict.

This commit also moves the slot-to-key API from db.c to cluster.c.

Co-authored-by: Jim Brunner <brunnerj@amazon.com>
 2021-08-30 23:25:36 -07:00
Oran Agra 1e7ad894d2 Tune timeout of active defrag test (#9426) 
Failed on Raspberry Pi 3b where that single test took about 170 seconds
 2021-08-30 12:39:09 +03:00
Wang Yuan 9a0c0617f1 Use sync_file_range to optimize fsync if possible (#9409) 
We implement incremental data sync in rio.c by call fsync, on slow disk, that may cost a lot of time,
sync_file_range could provide async fsync, so we could serialize key/value and sync file data at the same time.

> one tip for sync_file_range usage: http://lkml.iu.edu/hypermail/linux/kernel/1005.2/01845.html

Additionally, this change avoids a single large write to be used, which can result in a mass of dirty
pages in the kernel (increasing the risk of someone else's write to block).

On HDD, current solution could reduce approximate half of dumping RDB time,
this PR costs 50s for dump 7.7G rdb but unstable branch costs 93s.
On NVME SSD, this PR can't reduce much time,  this PR costs 40s, unstable branch costs 48s.

Moreover, I find calling data sync every 4MB is better than 32MB.
 2021-08-30 10:24:53 +03:00
Binbin aefbc23451 Better error handling for updateClientOutputBufferLimit. (#9308) 
This one follow #9313 and goes deeper (validation of config file parsing)

Move the check/update logic to a new updateClientOutputBufferLimit
function. So that it can be used in CONFIG SET and config file parsing.
 2021-08-29 15:03:05 +03:00
Viktor Söderqvist 97dcf95cc8 redis-benchmark: improved help and warnings (#9419) 
1. The output of --help:

  * On the Usage line, just write [OPTIONS] [COMMAND ARGS...] instead listing
    only a few arbitrary options and no command.
  * For --cluster, describe that if the command is supplied on the command line,
    the key must contain "{tag}". Otherwise, the command will not be sent to the
    right cluster node.
  * For -r, add a note that if -r is omitted, all commands in a benchmark will
    use the same key. Also align the description.
  * For -t, describe that -t is ignored if a command is supplied on the command
    line.

2. Print a warning if -t is present when a specific command is supplied.

3. Print all warnings and errors to stderr.

4. Remove -e from calls in redis-benchmark test suite.
 2021-08-29 14:31:08 +03:00
Huang Zhw b375f5919e redis-benchmark: make show throughput in only one thread. (#9146) 
In multipe threads mode, every thread output throughput info. This
may cause some problems:
- Bug in https://github.com/redis/redis/pull/8615;
- The show throughput is called too frequently;
- showThroughput which updates shared variable lacks synchronization
mechanism.

This commit also reverts changes in #8615 and changes time event
interval to macro.
 2021-08-25 14:58:35 +03:00
Garen Chan 945a83d406 Fix boundary problem of adjusting open files limit. (#5722) 
When `decr_step` is greater than `oldlimit`, the final `bestlimit` may be invalid.

    For example, oldlimit = 10, decr_step = 16.
    Current bestlimit = 15 and setrlimit() failed. Since bestlimit  is less than decr_step , then exit the loop.
    The final bestlimit is larger than oldlimit but is invalid.

Note that this only matters if the system fd limit is below 16, so unlikely to have any actual effect.
 2021-08-24 22:54:21 +03:00
Wen Hui 641780a9c6 config memory limits: handle values larger than (signed) LLONG_MAX (#9313) 
This aims to solve the issue in CONFIG SET maxmemory can only set maxmemory to up
to 9223372036854775807 (2^63) while the maxmemory should be ULLONG.
Added a memtoull function to convert a string representing an amount of memory
into the number of bytes (similar to memtoll but for ull). Also added ull2string to
convert a ULLong to string (Similar to ll2string).
 2021-08-23 21:00:40 +03:00
Viktor Söderqvist 74590f8345 redis-cli: Assert > 0 before dividing, to silence warning by tool (#9396) 
Also make sure function can't return NULL by another assert.
 2021-08-22 13:57:18 +03:00
Binbin 0835f596b8 BITSET and BITFIELD SET only propagate command when the value changed. (#9403) 
In old way, we always increase server.dirty in BITSET and BITFIELD SET.
Even the command doesn't really change anything. This commit make 
sure BITSET and BITFIELD SET only increase dirty when the value changed.

Because of that, if the value not changed, some others implications:
- Avoid adding useless AOF
- Reduce replication traffic
- Will not trigger keyspace notifications (setbit)
- Will not invalidate WATCH
- Will not sent the invalidation message to the tracking client
 2021-08-22 10:20:53 +03:00
Viktor Söderqvist 8f59c1ecae Let CONFIG GET * show both replicaof and its alias (#9395) 2021-08-21 19:43:18 -07:00
sundb 492d8d0961 Sanitize dump payload: fix double free after insert dup nodekey to stream rax and returns 0 (#9399) 2021-08-20 10:37:45 +03:00
Yossi Gottlieb 1d9c8d61d8 Skip OOM-related tests on incompatible platforms. (#9386) 
We only run OOM related tests on x86_64 and aarch64, as jemalloc on other
platforms (notably s390x) may actually succeed very large allocations. As
a result the test may hang for a very long time at the cleanup phase,
iterating as many as 2^61 hash table slots.
 2021-08-18 16:00:22 +03:00
yoav-steinberg 0e8d469f82 More generic crash report for unsupported archs (#9385) 
Following compilation warnings on s390x.
 2021-08-18 15:46:11 +03:00
Yossi Gottlieb fe359cbfc2 Fix: don't assume char is unsigned. (#9375) 
On systems that have unsigned char by default (s390x, arm), redis-server
could crash as soon as it populates the command table.
 2021-08-15 21:37:44 +03:00
Wang Yuan 8edc3cd62c Fix the wrong detection of sync_file_range system call (#9371) 
If we want to check `defined(SYNC_FILE_RANGE_WAIT_BEFORE)`, we should include fcntl.h.
otherwise, SYNC_FILE_RANGE_WAIT_BEFORE is not defined, and there is alway not `sync_file_range` system call.
Introduced by #8532
 2021-08-14 23:52:44 +03:00
Madelyn Olson 0cf2df84d4 Added additional validation for cluster SETSLOT (#9360) 2021-08-12 14:59:17 -07:00
Madelyn Olson 2402f5a7a1 Update cluster debug log to include human readable packet type (#9361) 2021-08-12 14:50:09 -07:00
Yossi Gottlieb 1221f7cd5e Improve setup operations order after fork. (#9365) 
The order of setting things up follows some reasoning: Setup signal
handlers first because a signal could fire at any time. Adjust OOM score
before everything else to assist the OOM killer if memory resources are
low.

The trigger for this is a valgrind test failure which resulted with the
child catching a SIGUSR1 before initializing the handler.
 2021-08-12 14:31:12 +03:00
Yossi Gottlieb 08c46f2b86 Add debian:oldoldstable build target for CI. (#9358) 
Making sure Redis builds properly on older compiler is important given the wide range of systems it is built for. So far Ubuntu 16.04 has been used for this purpose, but as it's getting phased out we'll move to `oldoldstable` Debian as an "old system" precursor.
 2021-08-11 16:19:54 +03:00
sundb 5705cec68e Fix missing dismiss hash listpack memory due to ziplist->listpack migration (#9353) 2021-08-10 16:54:19 +03:00
Huang Zhwandcharsyam 82528d2678 Redis-cli monitor and pubsub can be aborted with Ctrl+C, keeping the cli alive (#9347) 
Abort cli blocking modes with SIGINT without exiting the cli.

Co-authored-by: charsyam <charsyam@gmail.com>
 2021-08-10 15:03:49 +03:00
yoav-steinberg 19bc83716c support regex in "--only" in runtest (#9352) 2021-08-10 14:28:24 +03:00
DarrenJiang13 8ab33c18e4 fix a compilation error around madvise when make with jemalloc on MacOS (#9350) 
We only use MADV_DONTNEED on Linux, that's were it was tested.
 2021-08-10 11:32:27 +03:00
Meir Shpilraien (Spielrein) 8f8117f78e Format fixes and naming. SentReplyOnKeyMiss -> addReplyOrErrorObject (#9346) 
Following the comments on #8659, this PR fix some formatting
and naming issues.
 2021-08-10 10:19:21 +03:00
sundbandOran Agra 02fd76b97c Replace all usage of ziplist with listpack for t_hash (#8887) 
Part one of implementing #8702 (taking hashes first before other types)

## Description of the feature
1. Change ziplist encoded hash objects to listpack encoding.
2. Convert existing ziplists on RDB loading time. an O(n) operation.

## Rdb format changes
1. Add RDB_TYPE_HASH_LISTPACK rdb type.
2. Bump RDB_VERSION to 10

## Interface changes
1. New `hash-max-listpack-entries` config is an alias for `hash-max-ziplist-entries` (same with `hash-max-listpack-value`)
2. OBJECT ENCODING will return `listpack` instead of `ziplist`

## Listpack improvements:
1. Support direct insert, replace integer element (rather than convert back and forth from string)
3. Add more listpack capabilities to match the ziplist ones (like `lpFind`, `lpRandomPairs` and such)
4. Optimize element length fetching, avoid multiple calculations
5. Use inline to avoid function call overhead.

## Tests
1. Add a new test to the RDB load time conversion
2. Adding the listpack unit tests. (based on the one in ziplist.c)
3. Add a few "corrupt payload: fuzzer findings" tests, and slightly modify existing ones.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-08-10 09:18:49 +03:00
sundb cbda492909 Sanitize dump payload: handle remaining empty key when RDB loading and restore command (#9349) 
This commit mainly fixes empty keys due to RDB loading and restore command,
which was omitted in #9297.

1) When loading quicklsit, if all the ziplists in the quicklist are empty, NULL will be returned.
    If only some of the ziplists are empty, then we will skip the empty ziplists silently.
2) When loading hash zipmap, if zipmap is empty, sanitization check will fail.
3) When loading hash ziplist, if ziplist is empty, NULL will be returned.
4) Add RDB loading test with sanitize.
 2021-08-09 17:13:46 +03:00
Qu Chen 0b643e930d Cleanup: createAOFClient uses createClient to avoid overlooked mismatches (#9338) 
AOF fake client creation (createAOFClient) was doing similar work as createClient,
with some minor differences, most of which unintended, this was dangerous and
meant that many changes to createClient should have always been reflected to aof.c

This cleanup changes createAOFClient to call createClient with NULL, like we
do in module.c and elsewhere.
 2021-08-09 11:03:59 +03:00
Eduardo Semprebon d3356bf614 Add SORT_RO command (#9299) 
Add a readonly variant of the STORE command, so it can be used on
read-only workloads (replica, ACL, etc)
 2021-08-09 09:40:29 +03:00
Qu Chen e8eeba7bee Allow master to replicate command longer than replica's query buffer limit (#9340) 
Replication client no longer checks incoming command length against the client-query-buffer-limit. This makes the master able to replicate commands longer than replica's configured client-query-buffer-limit
 2021-08-08 17:34:11 -07:00
Yossi Gottlieb 3307958bd0 Propagate OPENSSL_PREFIX to hiredis. (#9345) 2021-08-08 18:30:17 +03:00
Binbin e1dc979054 sds.c: Fix potential overflow in sdsll2str. (#8910) 
Fixes an undefined behavior, same way as our `ll2string` does.
 2021-08-08 14:30:47 +03:00
Binbin 563ba7a3f0 Fix the wrong method used in quicklistTest. (#8951) 
The test try to test `insert before 1 element`, but it use quicklist
InsertAfter, a copy-paste typo.

The commit also add an assert to verify results in some tests
to make sure it is as expected.
 2021-08-08 09:03:52 +03:00
DarrenJiang13 43eb0ce3bf [BUGFIX] Add some missed error statistics (#9328) 
add error counting for some missed behaviors.
 2021-08-06 19:27:24 -07:00
yoav-steinberg 0a9377535b Ignore resize threshold on idle qbuf resizing (#9322) 
Also update qbuf tests to verify both idle and peak based resizing logic.
And delete unused function: getClientsMaxBuffers
 2021-08-06 20:50:34 +03:00
Oran Agra 3f3f678a47 corrupt-dump-fuzzer test, avoid creating junk keys (#9302) 
The execution of the RPOPLPUSH command by the fuzzer created junk keys,
that were later being selected by RANDOMKEY and modified.
This also meant that lists were statistically tested more than other
files.

Fix the fuzzer not to pass junk key names to RPOPLPUSH, and add a check
that detects that new keys are not added by the fuzzer to detect future
similar issues.
 2021-08-05 22:57:05 +03:00
Oran Agraandsundb 0c90370e6d Improvements to corrupt payload sanitization (#9321) 
Recently we found two issues in the fuzzer tester: #9302 #9285
After fixing them, more problems surfaced and this PR (as well as #9297) aims to fix them.

Here's a list of the fixes
- Prevent an overflow when allocating a dict hashtable
- Prevent OOM when attempting to allocate a huge string
- Prevent a few invalid accesses in listpack
- Improve sanitization of listpack first entry
- Validate integrity of stream consumer groups PEL
- Validate integrity of stream listpack entry IDs
- Validate ziplist tail followed by extra data which start with 0xff

Co-authored-by: sundb <sundbcn@gmail.com>
 2021-08-05 22:56:14 +03:00
sundbandOran Agra 8ea777a6a0 Sanitize dump payload: fix empty keys when RDB loading and restore command (#9297) 
When we load rdb or restore command, if we encounter a length of 0, it will result in the creation of an empty key.
This could either be a corrupt payload, or a result of a bug (see #8453 )

This PR mainly fixes the following:
1) When restore command will return `Bad data format` error.
2) When loading RDB, we will silently discard the key.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-08-05 22:42:20 +03:00
Madelyn Olson 39a4a44d7d Add debug config flag to print certain config values on engine crash (#9304) 
Add debug config flag to print certain config values on engine crash
 2021-08-05 11:59:12 -07:00
BinbinandOran Agra d0244bfc3d Make sure execute SLAVEOF command in the right order in psync2 test. (#9316) 
The psync2 test has failed several times recently.
In #9159 we only solved half of the problem.
i.e. reordering of the replica that's already connected to
the newly promoted master.

Consider this scenario:
0 slaveof 2
1 slaveof 2
3 slaveof 2
4 slaveof 1
0 slaveof no one, became a new master got a new replid
2 slaveof 0, partial resync and got the new replid
3 reconnect 2, inherit the new replid
3 slaveof 4, use the new replid and got a full resync

And another scenario:
1 slaveof 3
2 slaveof 4
3 slaveof 0
4 slaveof 0
4 slaveof no one, became a new master got a new replid
2 reconnect 4, inherit the new replid
2 slaveof 1, use the new replid and got a full resync

So maybe we should reattach replicas in the right order.
i.e. In the above example, if it would have reattached 1, 3 and 0 to
the new chain formed by 4 before trying to attach 2 to 1, it would succeed.

This commit break the SLAVEOF loop into two loops. (ideas from oran)

First loop that uses random to decide who replicates from who.
Second loop that does the actual SLAVEOF command.
In the second loop, we make sure to execute it in the right order,
and after each SLAVEOF, wait for it to be connected before we proceed.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-08-05 11:26:09 +03:00
Wen HuiandLucas Guang Yang 63e2a6d212 Add sentinel debug option command (#9291) 
This makes it possible to tune many parameters that were previously hard coded.
We don't intend these to be user configurable, but only used by tests to accelerate certain conditions which would otherwise take a long time and slow down the test suite.

Co-authored-by: Lucas Guang Yang <l84193800@china.huawei.com>
 2021-08-05 11:12:55 +03:00
menwen ca559819f7 Add latency monitor sample when key is deleted via lazy expire (#9317) 
Fix that there is no sample latency after the key expires via expireIfNeeded().
Some refactoring for shared code.
 2021-08-05 11:09:24 +03:00
yoav-steinberg d32f8641ed fix dict access broken by #9228 (#9319) 2021-08-05 09:02:30 +03:00
yoav-steinberg 5e908a290c dict struct memory optimizations (#9228) 
Reduce dict struct memory overhead
on 64bit dict size goes down from jemalloc's 96 byte bin to its 56 byte bin.

summary of changes:
- Remove `privdata` from callbacks and dict creation. (this affects many files, see "Interface change" below).
- Meld `dictht` struct into the `dict` struct to eliminate struct padding. (this affects just dict.c and defrag.c)
- Eliminate the `sizemask` field, can be calculated from size when needed.
- Convert the `size` field into `size_exp` (exponent), utilizes one byte instead of 8.

Interface change: pass dict pointer to dict type call back functions.
This is instead of passing the removed privdata field. In the future if
we'd like to have private data in the callbacks we can extract it from
the dict type. We can extend dictType to include a custom dict struct
allocator and use it to allocate more data at the end of the dict
struct. This data can then be used to store private data later acccessed
by the callbacks.
 2021-08-05 08:25:58 +03:00
Viktor Söderqvist 1c59567a7f redis-cli ASK redirect test: Add retry loop to fix timing issue (#9315) 2021-08-05 08:20:30 +03:00
Wang YuanandOran Agra d4bca53cd9 Use madvise(MADV_DONTNEED) to release memory to reduce COW (#8974) 
## Backgroud
As we know, after `fork`, one process will copy pages when writing data to these
pages(CoW), and another process still keep old pages, they totally cost more memory.
For redis, we suffered that redis consumed much memory when the fork child is serializing
key/values, even that maybe cause OOM.

But actually we find, in redis fork child process, the child process don't need to keep some
memory and parent process may write or update that, for example, child process will never
access the key-value that is serialized but users may update it in parent process.
So we think it may reduce COW if the child process release memory that it is not needed.

## Implementation
For releasing key value in child process, we may think we call `decrRefCount` to free memory,
but i find the fork child process still use much memory when we don't write any data to redis,
and it costs much more time that slows down bgsave. Maybe because memory allocator doesn't
really release memory to OS, and it may modify some inner data for this free operation, especially
when we free small objects.

Moreover, CoW is based on  pages, so it is a easy way that we only free the memory bulk that is
not less than kernel page size. madvise(MADV_DONTNEED) can quickly release specified region
pages to OS bypassing memory allocator, and allocator still consider that this memory still is used
and don't change its inner data.

There are some buffers we can release in the fork child process:
- **Serialized key-values**
  the fork child process never access serialized key-values, so we try to free them.
  Because we only can release big bulk memory, and it is time consumed to iterate all
  items/members/fields/entries of complex data type. So we decide to iterate them and
  try to release them only when their average size of item/member/field/entry is more
  than page size of OS.
- **Replication backlog**
  Because replication backlog is a cycle buffer, it will be changed quickly if redis has heavy
  write traffic, but in fork child process, we don't need to access that.
- **Client buffers**
  If clients have requests during having the fork child process, clients' buffer also be changed
  frequently. The memory includes client query buffer, output buffer, and client struct used memory.

To get child process peak private dirty memory, we need to count peak memory instead
of last used memory, because the child process may continue to release memory (since
COW used to only grow till now, the last was equivalent to the peak).
Also we're adding a new `current_cow_peak` info variable (to complement the existing
`current_cow_size`)

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-08-04 23:01:46 +03:00
Meir Shpilraien (Spielrein) 56eb7f7de4 Fix tests failure on 32bit build (#9318) 
Fix test introduced in #9202 that failed on 32bit CI.
The failure was due to a wrong double comparison.
Change code to stringify the double first and then compare.
 2021-08-04 21:33:38 +03:00
Meir Shpilraien (Spielrein)Oran AgraYossi Gottlieb
 		2237131e15 Unified Lua and modules reply parsing and added RESP3 support to RM_Call (#9202) 
## Current state
1. Lua has its own parser that handles parsing `reds.call` replies and translates them
  to Lua objects that can be used by the user Lua code. The parser partially handles
  resp3 (missing big number, verbatim, attribute, ...)
2. Modules have their own parser that handles parsing `RM_Call` replies and translates
  them to RedisModuleCallReply objects. The parser does not support resp3.

In addition, in the future, we want to add Redis Function (#8693) that will probably
support more languages. At some point maintaining so many parsers will stop
scaling (bug fixes and protocol changes will need to be applied on all of them).
We will probably end up with different parsers that support different parts of the
resp protocol (like we already have today with Lua and modules)

## PR Changes
This PR attempt to unified the reply parsing of Lua and modules (and in the future
Redis Function) by introducing a new parser unit (`resp_parser.c`). The new parser
handles parsing the reply and calls different callbacks to allow the users (another
unit that uses the parser, i.e, Lua, modules, or Redis Function) to analyze the reply.

### Lua API Additions
The code that handles reply parsing on `scripting.c` was removed. Instead, it uses
the resp_parser to parse and create a Lua object out of the reply. As mentioned
above the Lua parser did not handle parsing big numbers, verbatim, and attribute.
The new parser can handle those and so Lua also gets it for free.
Those are translated to Lua objects in the following way:
1. Big Number - Lua table `{'big_number':'<str representation for big number>'}`
2. Verbatim - Lua table `{'verbatim_string':{'format':'<verbatim format>', 'string':'<verbatim string value>'}}`
3. Attribute - currently ignored and not expose to the Lua parser, another issue will be open to decide how to expose it.

Tests were added to check resp3 reply parsing on Lua

### Modules API Additions
The reply parsing code on `module.c` was also removed and the new resp_parser is used instead.
In addition, the RedisModuleCallReply was also extracted to a separate unit located on `call_reply.c`
(in the future, this unit will also be used by Redis Function). A nice side effect of unified parsing is
that modules now also support resp3. Resp3 can be enabled by giving `3` as a parameter to the
fmt argument of `RM_Call`. It is also possible to give `0`, which will indicate an auto mode. i.e, Redis
will automatically chose the reply protocol base on the current client set on the RedisModuleCtx
(this mode will mostly be used when the module want to pass the reply to the client as is).
In addition, the following RedisModuleAPI were added to allow analyzing resp3 replies:

* New RedisModuleCallReply types:
   * `REDISMODULE_REPLY_MAP`
   * `REDISMODULE_REPLY_SET`
   * `REDISMODULE_REPLY_BOOL`
   * `REDISMODULE_REPLY_DOUBLE`
   * `REDISMODULE_REPLY_BIG_NUMBER`
   * `REDISMODULE_REPLY_VERBATIM_STRING`
   * `REDISMODULE_REPLY_ATTRIBUTE`

* New RedisModuleAPI:
   * `RedisModule_CallReplyDouble` - getting double value from resp3 double reply
   * `RedisModule_CallReplyBool` - getting boolean value from resp3 boolean reply
   * `RedisModule_CallReplyBigNumber` - getting big number value from resp3 big number reply
   * `RedisModule_CallReplyVerbatim` - getting format and value from resp3 verbatim reply
   * `RedisModule_CallReplySetElement` - getting element from resp3 set reply
   * `RedisModule_CallReplyMapElement` - getting key and value from resp3 map reply
   * `RedisModule_CallReplyAttribute` - getting a reply attribute
   * `RedisModule_CallReplyAttributeElement` - getting key and value from resp3 attribute reply
   
* New context flags:
   * `REDISMODULE_CTX_FLAGS_RESP3` - indicate that the client is using resp3

Tests were added to check the new RedisModuleAPI

### Modules API Changes
* RM_ReplyWithCallReply might return REDISMODULE_ERR if the given CallReply is in resp3
  but the client expects resp2. This is not a breaking change because in order to get a resp3
  CallReply one needs to specifically specify `3` as a parameter to the fmt argument of
  `RM_Call` (as mentioned above).

Tests were added to check this change

### More small Additions
* Added `debug set-disable-deny-scripts` that allows to turn on and off the commands no-script
flag protection. This is used by the Lua resp3 tests so it will be possible to run `debug protocol`
and check the resp3 parsing code.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
 2021-08-04 16:28:07 +03:00
sundb b4eda14257 Fix head and tail check with negative offset in _quicklistInsert (#9311) 
Some background:
This fixes a problem that used to be dead code till now,
but became alive (only in the unit tests, not in redis) when #9113 got merged.
The problem it fixes doesn't actually cause any significant harm,
but that PR also added a test that fails verification because of that.
This test was merged with that problem due to human error, we didn't run it
on the last modified version before merging.
The fix in this PR existed in #8641 (closed because it's just dead code)
and #4674 (still pending but has other changes in it).

Now to the actual fix:
On quicklist insertion, if the insertion offset is -1 or `-(quicklist->count)`,
we can insert into the head of the next node rather than the tail of the
current node. this is especially important when the current node is full,
and adding anything to it will cause it to be split (or be over it's fill limit setting).

The bug was that the code attempted to determine that we're adding to
the tail of the current node by matching `offset == node->count` when in
fact it should have been `offset == node->count-1` (so it never entered that `if`).
and also that since we take negative offsets too, we can also match `-1`.
same applies for the head, i.e. `0` and `-count`.

The bug will cause the code to attempt inserting into the current node (thinking
we have to insert into the middle of the node rather than head or tail), and
in case the current node is full it'll have to be split (something that also
happens in valid cases).
On top of that, since it calls _quicklistSplitNode with an edge case, it'll actually
split the node in a way that all the entries fall into one split, and 0 into the other,
and then still insert the new entry into the first one, causing it to be populated
beyond it's intended fill limit.

This problem does not create any bug in redis, because the existing code does
not iterate from tail to head, and the offset never has a negative value when insert.

The other change this PR makes in the test code is just for some coverage,
insertion at index 0 is tested a lot, so it's nice to test some negative offsets too.
 2021-08-04 11:19:48 +03:00
Oran Agra 52df350fe5 Skip new redis-cli ASK test in TLS mode (#9312) 2021-08-03 13:19:03 -07:00
filipe oliveiraandOran Agra 4aa927d16d Enabled -x option (Read last argument from STDIN) on redis-benchmark (#9130) 
Add the -x option (Read last argument from STDIN) on redis-benchmark.

Other changes:
To be able to use the code from redis-cli some helper methods were moved to cli_common.(h|c)

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-08-03 12:21:29 +03:00
Jonah H. HarrisandOran Agra 432c92d8df Add SINTERCARD/ZINTERCARD Commands (#8946) 
Add SINTERCARD and ZINTERCARD commands that are similar to
ZINTER and SINTER but only return the cardinality with minimum
processing and memory overheads.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-08-03 11:45:27 +03:00
Ariel ShtulGuy BenoishOran Agra
 		bdbf5eedae Module api support for RESP3 (#8521) 
Add new Module APS for RESP3 responses:
- RM_ReplyWithMap
- RM_ReplyWithSet
- RM_ReplyWithAttribute
- RM_ReplySetMapLength
- RM_ReplySetSetLength
- RM_ReplySetAttributeLength
- RM_ReplyWithBool

Deprecate REDISMODULE_POSTPONED_ARRAY_LEN in favor of a generic REDISMODULE_POSTPONED_LEN

Improve documentation
Add tests

Co-authored-by: Guy Benoish <guy.benoish@redislabs.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-08-03 11:37:19 +03:00
Yossi Gottlieb 4bd7748362 Tests: fix commandfilter crash on alpine. (#9307) 
Loading and unloading the shared object does not initialize global vars
on alpine.
 2021-08-02 15:50:45 +03:00
Huang ZhwViktor SöderqvistOran Agra
 		cf61ad14cc When redis-cli received ASK, it didn't handle it (#8930) 
When redis-cli received ASK, it used string matching wrong and didn't
handle it. 

When we access a slot which is in migrating state, it maybe
return ASK. After redirect to the new node, we need send ASKING
command before retry the command.  In this PR after redis-cli receives 
ASK, we send a ASKING command before send the origin command 
after reconnecting.

Other changes:
* Make redis-cli -u and -c (unix socket and cluster mode) incompatible 
  with one another.
* When send command fails, we avoid the 2nd reconnect retry and just
  print the error info. Users will decide how to do next. 
  See #9277.
* Add a test faking two redis nodes in TCL to just send ASK and OK in 
  redis protocol to test ASK behavior. 

Co-authored-by: Viktor Söderqvist <viktor.soderqvist@est.tech>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-08-02 14:59:08 +03:00
Binbin 4000cb7d34 Modify some error logs printing level. (#9306) 
1. In sendBulkToSlave, we used LL_VERBOSE in the past, changed to
LL_WARNING. (all the other places that do freeClient(slave) use LL_WARNING)
2. The old style LOG_WARNING, chang it to LL_WARNING. Introduced in an
old pr (#1690).
 2021-08-02 11:18:35 +03:00
Ning Sun f74af0e61d Add NX/XX/GT/LT options to EXPIRE command group (#2795) 
Add NX, XX, GT, and LT flags to EXPIRE, PEXPIRE, EXPIREAT, PEXAPIREAT.
- NX - only modify the TTL if no TTL is currently set 
- XX - only modify the TTL if there is a TTL currently set 
- GT - only increase the TTL (considering non-volatile keys as infinite expire time)
- LT - only decrease the TTL (considering non-volatile keys as infinite expire time)
return value of the command is 0 when the operation was skipped due to one of these flags.

Signed-off-by: Ning Sun <sunng@protonmail.com>
 2021-08-02 08:57:49 +03:00
menwen 82c3158ad5 Fix if consumer is created as a side effect without notify and dirty++ (#9263) 
Fixes:
- When a consumer is created as a side effect, redis didn't issue a keyspace notification,
  nor incremented the server.dirty (affects periodic snapshots).
  this was a bug in XREADGROUP, XCLAIM, and XAUTOCLAIM.
- When attempting to delete a non-existent consumer, don't issue a keyspace notification
  and don't increment server.dirty
  this was a bug in XGROUP DELCONSUMER

Other changes:
- Changed streamLookupConsumer() to always only do lookup consumer (never do implicit creation),
  Its last seen time is updated unless the SLC_NO_REFRESH flag is specified.
- Added streamCreateConsumer() to create a new consumer. When the creation is successful,
  it will notify and dirty++ unless the SCC_NO_NOTIFY or SCC_NO_DIRTIFY flags is specified.
- Changed streamDelConsumer() to always only do delete consumer.
- Added keyspace notifications tests about stream events.
 2021-08-02 08:31:33 +03:00
cmemory 27a68a4d1b improve quicklist insert head/tail while head/tail node is full. (#9113) 
In _quicklistInsert when `at_head` / `at_tail` is true, but `prev` / `next` is NULL,
the code was reaching the last if-else block at the bottom of the function,
and would have unnecessarily executed _quicklistSplitNode, instead of just creating a new node.
This was because the penultimate if-else was checking `node->next && full_next`.
but in fact it was unnecessary to check if `node->next` exists, if we're gonna create one anyway,
we only care that it's not full, or doesn't exist, so the condition could have been changed to `!node->next || full_next`.

Instead, this PR makes a small refactory to negate `full_next` to a more meaningful variable
`avail_next` that indicates that the next node is available for pushing additional elements or
not (this would be true only if it exists and it is non-full)
 2021-08-02 08:30:25 +03:00
SmartKeyerror 8ea18fa8d2 Fix copy-paste typo in t_list.c comment (#9305) 2021-08-02 08:06:36 +03:00
BinbinandOran Agra 86555ae0f7 GEO* STORE with empty src key delete the dest key and return 0, not empty array (#9271) 
With an empty src key, we need to deal with two situations:
1. non-STORE: We should return emptyarray.
2. STORE: Try to delete the store key and return 0.

This applies to both GEOSEARCHSTORE (new to v6.2), and
also GEORADIUS STORE (which was broken since forever)

This pr try to fix #9261. i.e. both STORE variants would have behaved
like the non-STORE variants when the source key was missing,
returning an empty array and not deleting the destination key,
instead of returning 0, and deleting the destination key.

Also add more tests for some commands.
- GEORADIUS: wrong type src key, non existing src key, empty search,
  store with non existing src key, store with empty search
- GEORADIUSBYMEMBER: wrong type src key, non existing src key,
  non existing member, store with non existing src key
- GEOSEARCH: wrong type src key, non existing src key, empty search,
  frommember with non existing member
- GEOSEARCHSTORE: wrong type key, non existing src key,
  fromlonlat with empty search, frommember with non existing member

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-08-01 19:32:24 +03:00
Yossi Gottlieb 68b8b45cd5 Tests: avoid short reads on redis-cli output. (#9301) 
In some cases large replies on slow systems may only be partially read
by the test suite, resulting with parsing errors.

This fix is still timing sensitive but should greatly reduce the chances
of this happening.
 2021-08-01 15:07:27 +03:00
Guy KorlandandYossi Gottlieb 1483f5aa9b Remove const from CommandFilterArgGet result (#9247) 
Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
 2021-08-01 11:29:32 +03:00
Long Dai 89fdcbec8c tests: fix a typo (#9294) 
Signed-off-by: Long Dai <long0dai@foxmail.com>
 2021-07-30 10:00:07 +03:00
Quinn Klassen 8647addf93 Free unused capacity in the cluster send buffer. (#9255) 
* Free unused capacity in the cluster send buffer.
* Refactor cluster cron to include a dedicated loop for node based cron jobs
 2021-07-29 17:31:51 -07:00
Ewg-c a403816405 Minor refactoring for rioConnRead and adding errno (#9280) 
minor refactoring for rioConnRead and adding errno
 2021-07-29 17:29:23 -07:00
Yossi Gottlieb 8bf433dc86 Clean unused var compiler warning in module test. (#9289) 2021-07-29 19:45:29 +03:00
Wen Hui db41536454 Remove duplicate zero-port sentinels (#9240) 
The issue is that when a sentinel with the same address and IP is turned on with a different runid, its port is set to 0 but it is still present in the dictionary master->sentinels which contain all the sentinels for a master.

This causes a problem when we do INFO SENTINEL because it takes the size of the dictionary of sentinels. This might also cause a problem for failover if enough sentinels have their port set to 0 since the number of voters in failover is also determined by the size of the dictionary of sentinels.

This commits removes the sentinels with the port set to zero from the dictionary of sentinels.
Fixes #8786
 2021-07-29 12:32:28 +03:00
Oran Agra b458b2999b Fix LRU blue moon bug in RESTORE, RDB loading, module API (#9279) 
The `lru_clock` and `lru` bits in `robj` save the least significant 24 bits of the unixtime (seconds since 1/1/1970),
and wrap around every 194 days.
The `objectSetLRUOrLFU` function, which is used in RESTORE with IDLETIME argument, and also in replica
or master loading an RDB that contains LRU, and by a module API had a bug that's triggered when that happens.

The scenario was that the idle time that came from the user, let's say RESTORE command is about 1000 seconds
(e.g. in the `RESTORE can set LRU` test we have), and the current `lru_clock` just wrapped around and is less than
1000 (i.e. a period of 1000 seconds once in some 6 months), the expression in that function would produce a negative
value and the code (and comment) specified that the best way to solve that is push the idle time backwards into the
past by 3 months. i.e. an idle time of 3 months instead of 1000 seconds.

instead, the right thing to do is to unwrap it, and put it near LRU_CLOCK_MAX. since now `lru_clock` is smaller than
`obj->lru` it will be unwrapped again by `estimateObjectIdleTime`.

bug was introduced by 052e03495f, but the code before it also seemed wrong.
 2021-07-29 12:11:29 +03:00
sundb 3db0f1a284 Fix missing check for sanitize_dump in corrupt-dump-fuzzer test (#9285) 
this means the assertion that checks that when deep sanitization is enabled,
there are no crashes, was missing.
 2021-07-29 11:53:21 +03:00
Huang ZhwandOran Agra 17511df59b Add INFO stat total_eviction_exceeded_time and current_eviction_exceeded_time (#9031) 
Add two INFO metrics:
```
total_eviction_exceeded_time:69734
current_eviction_exceeded_time:10230
```
`current_eviction_exceeded_time` if greater than 0, means how much time current used memory is greater than `maxmemory`. And we are still over the maxmemory. If used memory is below `maxmemory`, this metric is reset to 0.
`total_eviction_exceeded_time` means total time used memory is greater than `maxmemory` since server startup. 
The units of these two metrics are ms.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-07-26 10:07:20 +03:00
ZhaolongLiandlizhaolong.lzl 8d00493485 tests: fix exec fails when grep exists with status other than 0 (#9066) 
Co-authored-by: lizhaolong.lzl <lizhaolong.lzl@B-54MPMD6R-0221.local>
 2021-07-25 09:58:21 +03:00
Oran Agraandminjian.cai 9ca5e8c547 fix zslGetRank bug in dead-code (#9246) 
This fixes an issue with zslGetRank which will happen only if the
skiplist data stracture is added two entries with the same element name,
this can't happen in redis zsets (we use dict), but in theory this is a
bug in the underlaying skiplist code.

Fixes #3081 and #4032

Co-authored-by: minjian.cai <cmjgithub@163.com>
 2021-07-22 13:40:00 +03:00
Huang Zhw 71d452876e On 32 bit platform, the bit position of GETBIT/SETBIT/BITFIELD/BITCOUNT,BITPOS may overflow (see CVE-2021-32761) (#9191) 
GETBIT, SETBIT may access wrong address because of wrap.
BITCOUNT and BITPOS may return wrapped results.
BITFIELD may access the wrong address but also allocate insufficient memory and segfault (see CVE-2021-32761).

This commit uses `uint64_t` or `long long` instead of `size_t`.
related https://github.com/redis/redis/pull/8096

At 32bit platform:
> setbit bit 4294967295 1
(integer) 0
> config set proto-max-bulk-len 536870913
OK
> append bit "\xFF"
(integer) 536870913
> getbit bit 4294967296
(integer) 0

When the bit index is larger than 4294967295, size_t can't hold bit index. In the past,  `proto-max-bulk-len` is limit to 536870912, so there is no problem.

After this commit, bit position is stored in `uint64_t` or `long long`. So when `proto-max-bulk-len > 536870912`, 32bit platforms can still be correct.

For 64bit platform, this problem still exists. The major reason is bit pos 8 times of byte pos. When proto-max-bulk-len is very larger, bit pos may overflow.
But at 64bit platform, we don't have so long string. So this bug may never happen.

Additionally this commit add a test cost `512MB` memory which is tag as `large-memory`. Make freebsd ci and valgrind ci ignore this test.
 2021-07-21 16:25:19 +03:00
Oran Agra 32e61ee295 Fix ACL category for SELECT, WAIT, ROLE, LASTSAVE, READONLY, READWRITE, ASKING (#9208) 
- SELECT and WAIT don't read or write from the keyspace (unlike DEL, EXISTS, EXPIRE, DBSIZE, KEYS, etc).
they're more similar to AUTH and HELLO (and maybe PING and COMMAND).
they only affect the current connection, not the server state, so they should be `@connection`, not `@keyspace`

- ROLE, like LASTSAVE is `@admin` (and `@dangerous` like INFO) 

- ASKING, READONLY, READWRITE are `@connection` too (not `@keyspace`)

- Additionally, i'm now documenting the exact meaning of each ACL category so it's clearer which commands belong where.
 2021-07-20 21:48:43 +03:00
Huang ZhwandOran Agra 1895e134a7 Fix missing separator in module info line (usedby and using lists) (#9241) 
Fix module info genModulesInfoStringRenderModulesList lack separator when there's more than one module in the list.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-07-19 11:10:25 +03:00
Huang Zhw d54c9086c2 Remove testmodule in src/modules/Makefile. (#9250) 
src/modules make failed. As in #3718 testmodule.c was removed. But the makefile was not updated
 2021-07-19 08:23:25 +03:00
Binbin e7ccdeebc9 Fix rank overflow in zslInsert with more than 2B entries. (#9249) 
If there are more than 2B entries in a zset.
The calculated span will overflow.
 2021-07-18 16:47:33 +03:00
Erik Dubbelboer ee4bdf10ee Fix format strings serverLogObjectDebugInfo to use unsigned (#2927) 
This doesn't have any real impact, just a cleanup.
 2021-07-18 15:27:42 +03:00
Binbin c9ca3b7065 Add 'ist' to wordlist. (#9248) 
fix spell checker CI after recently merged old PR
 2021-07-18 14:59:00 +03:00
Paul Kulchenko 8a73cd1f9a Update Lua debugging help message commands for consistency. (#2946) 2021-07-18 14:11:09 +03:00
Lawrence Chou 71c0466c04 Fix typo in redis.conf comment (#9245) 2021-07-17 21:03:28 +03:00
Binbin 11dc4e59b3 SMOVE only notify dstset when the addition is successful. (#9244) 
in case dest key already contains the member, the dest key isn't modified, so the command shouldn't invalidate watch.
 2021-07-17 09:54:06 +03:00
qetu3790 f03af47a34 Set TCP keepalive on inbound clusterbus connections (#9230) 
Set TCP keepalive on inbound clusterbus connections to prevent memory leak
 2021-07-15 20:40:25 -07:00
ZEEKLING 334a9789f1 fix timeout spell error (#9150) 2021-07-15 16:01:11 -07:00
Oran Agra 6a5bac309e Test infra, handle RESP3 attributes and big-numbers and bools (#9235) 
- promote the code in DEBUG PROTOCOL to addReplyBigNum
- DEBUG PROTOCOL ATTRIB skips the attribute when client is RESP2
- networking.c addReply for push and attributes generate assertion when
  called on a RESP2 client, anything else would produce a broken
  protocol that clients can't handle.
 2021-07-14 19:14:31 +03:00
Yossi Gottlieb 277e4dc203 Fix compatibility with OpenSSL 1.1.0. (#9233) 2021-07-14 12:32:13 +03:00
gourav 7ac42ab7b3 Return -1 when connTLSWrite fails to write due to socket being closed (#9222) 2021-07-14 10:46:33 +03:00
Binbin d0f36eeda0 Add missing comma in memory/xgroup command help message. (#9210) 
This would have resulted in missing newline in the help message
 2021-07-13 18:16:05 +03:00
perryitayandOran Agra ac8b1df885 Fail EXEC command in case a watched key is expired (#9194) 
There are two issues fixed in this commit: 
1. we want to fail the EXEC command in case there is a watched key that's logically
   expired but not yet deleted by active expire or lazy expire.
2. we saw that currently cache time is update in every `call()` (including nested calls),
   this time is being also being use for the isKeyExpired comparison, we want to update
   the cache time only in the first call (execCommand)

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-07-11 13:17:23 +03:00
Huang Zhw cb961d8c8e Do not install a file event to send data to rewrite child when parent stop sending diff to child in aof rewrite. (#8767) 
In aof rewrite, when parent stop sending data to child, if there is
new rewrite data, aofChildWriteDiffData write event will be installed.
Then this event is issued and deletes the file event without do anyting.
This will happen over and over again until aof rewrite finish.

This bug used to waste a few system calls per excessive wake-up
(epoll_ctl and epoll_wait) per cycle, each cycle triggered by receiving
a write command from a client.
 2021-07-11 13:00:17 +03:00
Binbin fe5d325107 Add test for ziplist (nextdiff == -4 && reqlen < 4) (#9218) 
The if judgement `nextdiff == -4 && reqlen < 4` in __ziplistInsert.
It's strange, but it's useful. Without it there will be problems during
chain update.
Till now these lines didn't have coverage in the tests, and there was
a question if they are at all needed (#7170)
 2021-07-11 11:26:31 +03:00
Yossi Gottlieb 92e8004705 Pre-test bind-source-addr before running test. (#9214) 
This attempts to catch any non-standard configuration where the test may
fail and produce a false positive.
 2021-07-11 09:54:07 +03:00
Shogo Hayashi 350c6306b4 fix explanation of sha256 (#9220) 2021-07-10 10:04:54 -05:00
Huang Zhw 9a22457e8f Start redis-check-aof/redis-check-rdb only if executable name contains (#9215) 
redis-check-aof/redis-check-rdb.

Related to #9176. Before this commit, redis-server starts as
redis-check-aof/redis-check-rdb if the directory it is started from
contains the string redis-check-aof/redis-check-rdb. We check the
executable name instead of directory.
 2021-07-09 21:45:58 +03:00
Mikhail FesenkoOran AgraViktor Söderqvist
 		1eb4baa5b8 Direct redis-cli repl prints to stderr, because --rdb can print to stdout. fflush stdout after responses (#9136) 
1. redis-cli can output --rdb data to stdout
   but redis-cli also write some messages to stdout which will mess up the rdb.

2. Make redis-cli flush stdout when printing a reply
  This was needed in order to fix a hung in redis-cli test that uses
  --replica.
   Note that printf does flush when there's a newline, but fwrite does not.

3. fix the redis-cli --replica test which used to pass previously
   because it didn't really care what it read, and because redis-cli
   used printf to print these other things to stdout.

4. improve redis-cli --replica test to run with both diskless and disk-based.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Viktor Söderqvist <viktor@zuiderkwast.se>
 2021-07-07 08:26:26 +03:00
Binbin 3a26f61fb3 Add range check for master port in replicaof. (#9201) 
So that we can avoid commands that are obviously wrong.

Also unified with loadServerConfigFromString
because we also checked the range.
 2021-07-06 10:46:10 +03:00
zhaozhao.zz 5ea3c3db15 a little optimization in touchAllWatchedKeysInDb (#9200) 
touch keys only if the key is in one of the dbs,
in case rewind the list when the key doesn't exist.
 2021-07-06 15:01:37 +08:00
Omer Shadmi f06d782f5a Avoid exiting to allow diskless loading to recover from RDB short read on module AUX data (#9199) 
Currently a replica is able to recover from a short read (when diskless loading
is enabled) and avoid crashing/exiting, replying to the master and then the rdb
could be sent again by the master for another load attempt by the replica.
There were a few scenarios that were not behaving similarly, such as when
there is no end-of-file marker, or when module aux data failed to load, which
should be allowed to occur due to a short read.
 2021-07-06 08:21:17 +03:00
Oran Agra f8da33ae59 update issue template, replace IRC with discussions (#9198) 2021-07-05 18:09:48 +03:00
uriyage 67732bacb1 redis-cli: reset dbnum and tx prompt state after RESET (#9096) 2021-07-05 10:55:55 +03:00
BinbinandOran Agra a418a2d3fc hrandfield and zrandmember with count should return emptyarray when key does not exist. (#9178) 
due to a copy-paste bug, it used to reply with null response rather than empty array.
this commit includes new tests that are looking at the RESP response directly in
order to be able to tell the difference between them.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-07-05 10:41:57 +03:00
Oran AgraRajiv KurianYoav Steinberg
 		5a3de81925 Use accept4 on linux instead of fcntl to make a client socket non-blocking (#9177) 
This reduces system calls on linux when a new connection is made / accepted.

Changes:
* Add the SOCK_CLOEXEC option to the accept4() call
  This  ensure that a fork/exec call does not leak a file descriptor.
* Move anetCloexec and connNonBlock info anetGenericAccept
* Moving connNonBlock from accept handlers to anetGenericAccept

Moving connNonBlock from createClient, is safe because createClient is
used in the following ways:
1. without a connection (fake client)
2. on an accepted connection (see above)
3. creating the master client by using connConnect (see below)

The third case, can either use anetTcpNonBlockConnect, or connTLSConnect
which is by default non-blocking.

Co-authored-by: Rajiv Kurian <geetasen@gmail.com>
Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Yoav Steinberg <yoav@redislabs.com>
 2021-07-05 10:34:20 +03:00
Oran Agrasundbyoav-steinberg
 		ec582cc7ad Query buffer shrinking improvements (#5013) 
when tracking the peak, don't reset the peak to 0, reset it to the
maximum of the current used, and the planned to be used by the current
arg.

when shrining, split the two separate conditions.
the idle time shrinking will remove all free space.
but the peak based shrinking will keep room for the current arg.

when we resize due to a peak (rahter than idle time), don't trim all
unused space, let the qbuf keep a size that's sufficient for the
currently process bulklen, and the current peak.

Co-authored-by: sundb <sundbcn@gmail.com>
Co-authored-by: yoav-steinberg <yoav@monfort.co.il>
 2021-07-05 09:30:16 +03:00
zhaozhao.zz 2248eaacac resize query buffer more accurately 
1. querybuf_peak has not been updated correctly in readQueryFromClient.
2. qbuf shrinking uses sdsalloc instead of sdsAllocSize

see more details in issue #4983
 2021-07-05 09:30:16 +03:00
Oran Agra 9894ce5fc4 add missing skips to daily cycle (#9196) 
minor cleanups.
 2021-07-05 08:46:19 +03:00
Oran Agra 7103367ad4 Tests: add a way to read raw RESP protocol reponses (#9193) 
This makes it possible to distinguish between null response and an empty
array (currently the tests infra translates both to an empty string/list)
 2021-07-04 19:43:58 +03:00
Oran Agra a8518cce95 fix valgrind issues with recently added test in modules/blockonbackground (#9192) 
fixes test issue introduced in #9167

1. invalid reads due to accessing non-retained string (passed as unblock context).
2. leaking module blocked client context, see #6922 for info.
 2021-07-04 14:21:53 +03:00
Madelyn Olson 8f59f131e5 Update incrDecrCommand to use addReplyLongLong (#9188) 
Update incrDecrCommand to use addReplyLongLong
 2021-07-03 10:51:53 -05:00
Yossi Gottlieb aa139e2f02 Fix CLIENT UNBLOCK crashing modules. (#9167) 
Modules that use background threads with thread safe contexts are likely
to use RM_BlockClient() without a timeout function, because they do not
set up a timeout.

Before this commit, `CLIENT UNBLOCK` would result with a crash as the
`NULL` timeout callback is called. Beyond just crashing, this is also
logically wrong as it may throw the module into an unexpected client
state.

This commits makes `CLIENT UNBLOCK` on such clients behave the same as
any other client that is not in a blocked state and therefore cannot be
unblocked.
 2021-07-01 17:11:27 +03:00
Oran Agraandstafuc de9bae21ef Fix bug in sdscatfmt when % is the last format char (#9173) 
For the sdscatfmt function in sds.c, when the parameter fmt ended up with '%',
the behavior is undefined. This commit fix this bug.

Co-authored-by: stafuc <stafuc@gmail.com>
 2021-07-01 08:19:04 +03:00
Wang Yuan 16e04ed944 Don't start in sentinel mode if only the folder name contains redis-sentinel (#9176) 
Before this commit, redis-server starts in sentinel mode if the first startup
argument has the string redis-sentinel, so redis also starts in sentinel mode
if the directory it was started from contains the string redis-sentinel.
Now we check the executable name instead of directory.

Some examples:
1. Execute ./redis-sentinel/redis/src/redis-sentinel, starts in sentinel mode.
2. Execute ./redis-sentinel/redis/src/redis-server, starts in server mode,
   but before, redis will start in sentinel mode.
3. Execute ./redis-sentinel/redis/src/redis-server --sentinel, of course, like
   before, starts in sentinel mode.
 2021-07-01 08:05:18 +03:00
ZhaolongLilizhaolong.lzlOran Agraguybe7
 		6476c8e856 Fix range issues in default value of LIMIT argument to XADD and XTRIM (#9147) 
This seems to be an unimportant bug that was accidentally generated. If the user does not specify limit in streamParseAddOrTrimArgsOrReply, the initial value of args->limit is 100 * server.stream_node_max_entries, which may lead to out of bounds, and then the default function of limit in xadd becomes invalid (this failure occurs in streamTrim).

Additionally, provide sane default for args->limit in case stream_node_max_entries is set to 0.

Co-authored-by: lizhaolong.lzl <lizhaolong.lzl@B-54MPMD6R-0221.local>
Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: guybe7 <guy.benoish@redislabs.com>
 2021-06-30 16:55:09 +03:00
Mikhail FesenkoOran AgraWang Yuan
 		74fe15b360 redis-cli --rdb: fix broken fsync/ftruncate for stdout (#9135) 
A change in redis 6.2 caused redis-cli --rdb that's directed to stdout to fail because fsync fails.
This commit avoids doing ftruncate (fails with a warning) and fsync (fails with an error) when the
output file is `-`, and adds the missing documentation that `-` means stdout.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: Wang Yuan <wangyuancode@163.com>
 2021-06-30 16:49:54 +03:00
Rob Snyder eaa52719a3 Fix ziplist length updates on bigendian platforms (#2080) 
Adds call to intrev16ifbe to ensure ZIPLIST_LENGTH is compared correctly
 2021-06-30 16:46:06 +03:00
Huang Zhw 70dac4b436 Make redis-cli --help/-h output to stdout and exit with 0. (#9124) 
If user executes redis-cli --help, the content should be output to stdout and exits with 0.
 2021-06-30 14:56:44 +03:00
Binbin 5dddf496ce Add missing pause tcl test to test_helper.tcl (#9158) 
* Add keyname tags to avoid CROSSSLOT errors in external server CI
* Use new wait_for_blocked_clients_count in pause.tcl
 2021-06-30 13:32:51 +03:00
luvine 4278c45c91 HSETNX can lead coding type change even when skipped due to NX (#4615) 
1. Add one key-value pair to myhash, which the length of key and value both less than hash-max-ziplist-value, for example:
>hset myhash key value
2. Then execute the following command
>hsetnx myhash key value1 (the length greater than hash-max-ziplist-value)
3. This will add nothing, but the code type of "myhash" changed from ziplist to dict even there are only one key-value pair in "myhash", and both of them less than hash-max-ziplist-value.
 2021-06-30 12:14:11 +03:00
BinbinandOran Agra 1d5aa37d68 Fix timing issue in psync2 test. (#9159) 
*** [err]: PSYNC2: total sum of full synchronizations is exactly 4 intests/integration/psync2.tcl
Expected 5 == 4 (context: type eval line 8 cmd {assert {$sum == 4}} proc::test)

Sometime the test got an unexpected full sync since a replica switch to master,
before the new master change propagated the new replid to all replicas,
a replica attempted to sync with it using a wrong replid and triggered a full resync.

Consider this scenario:
    1 slaveof 4 full resync
    0 slaveof 4 full resync
    2 slaveof 0 full resync
    3 slaveof 1 full resync

    1 slaveof no one, replid changed
    3 reconnect 1, did a partial resyn and got the new replid

    Before 2 inherits the new replid.
    3 slaveof 2
    3 try to do a partial resyn with 2.
    But their replication ids are inconsistent, so a full resync happens.

:) A special thank you for oran and helping me in this test case.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-06-30 09:18:10 +03:00
ZhaolongLiandlizhaolong.lzl cf1d49adbf Simplify logic in raxSeek, eliminate it->key reassembly on gt and lt (#9115) 
In the original version, the operation of traversing the stack only seems to
reconstruct the key that does not contain the current node.

But in fact We have got the matched length and splitpos in the key in the
raxlowwalk, so I think we can simplify the logic of this part.

Co-authored-by: lizhaolong.lzl <lizhaolong.lzl@B-54MPMD6R-0221.local>
 2021-06-30 08:21:19 +03:00
Yossi Gottlieb 5d8ea4b326 Add missing needs:repl tag. (#9169) 2021-06-29 16:48:52 +03:00
Leibale EidelmanandOran Agra 95274f1f8a fix ZRANGESTORE - should return 0 when src points to an empty key (#9089) 
mistakenly it used to return an empty array rather than 0.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-06-29 16:38:10 +03:00
Tsonglewandyoav-steinberg 7913d34d7c Improve doc comment about AE_DONT_WAIT (#9165) 
Co-authored-by: yoav-steinberg <yoav@monfort.co.il>
 2021-06-29 14:37:02 +03:00
guybe7 4434cebbb3 Include sizeof(struct stream) in objectComputeSize (#9164) 
Affects MEMORY USAGE
 2021-06-29 14:34:18 +03:00
Binbin 4bc5a8324d ZRANDMEMBER WITHSCORES with negative COUNT may return bad score (#9162) 
Return a bad score when used with negative count (or count of 1), and non-ziplist encoded zset.
Also add test to validate the return value and cover the issue.
 2021-06-29 10:14:28 +03:00
Wang Yuan 4fa3e23092 Remove unnecessary replication backlog memory copy (#9157) 
in the past, the reply list was a list of sds objects, so this didn't have any overhead,
but now addReplySds just copies the data from the sds and frees it, so there's no
need to make a copy of the buffer before copying again.
this reduces an excessive allocation and free and a memcpy.
 2021-06-28 09:43:40 +03:00
Yossi Gottlieb 1071430875 Corrections about the new protected-mode usage. (#9143) 2021-06-27 11:34:48 +03:00
Jerry Chan 29 b71c5849e3 Fix link in README (#9145) 2021-06-25 19:04:04 -07:00
Yossi Gottlieb f233c4c59d Add bind-source-addr configuration argument. (#9142) 
In the past, the first bind address that was explicitly specified was
also used to bind outgoing connections. This could result with some
problems. For example: on some systems using `bind 127.0.0.1` would
result with outgoing connections also binding to `127.0.0.1` and failing
to connect to remote addresses.

With the recent change to the way `bind` is handled, this presented
other issues:

* The default first bind address is '*' which is not a valid address.
* We make no distinction between user-supplied config that is identical
to the default, and the default config.

This commit addresses both these issues by introducing an explicit
configuration parameter to control the bind address on outgoing
connections.
 2021-06-24 19:48:18 +03:00
Huang Zhw d1a21e0292 Clean redis-benchmark Throughput output. (#9139) 
some leftovers from print are visible when the new line is printed.
 2021-06-24 18:04:19 +03:00
ZhaolongLi 89ae353748 change streamAppendItem to use raxEOF instead of raxNext (#9138) 
The call to raxNext didn't really progress in the rax, since we were already on the last item.
instead, all it does is check that it is indeed a valid item, so the new code clearer.
 2021-06-24 15:08:26 +03:00
Oran Agra a5bc54f01e Re-fix daily CI (#9141) 
The freebsd and macos jobs were still broken.
also add a few more skip jobs options.
 2021-06-24 12:50:10 +03:00
Oran Agra 5ffdbae1f6 Fix failing basics moduleapi test on 32bit CI (#9140) 2021-06-24 12:44:13 +03:00
Oran Agra eb09fbce1e daily CI, fix odd yaml parsing error (#9137) 2021-06-23 13:56:40 -07:00
Oran Agra 0676a9fc6b Fix broken daily due to manual action triggers. and attempt to improve them (#9134) 
The daily CI was broken by #9119 seems that for cron scheduled tasks, these ifs aren't evaluated to false.
But also it turns out that workflow_dispatch is only able to run CI on branches in the main repo (not on PRs).
this is an attempt to overcome that by being able to checkout from any repo we want.
 2021-06-23 22:13:24 +03:00
Oran Agra b466471a52 Add manual triggers for Daily CI (#9119) 
Hopefully now we'll be able to manually trigger extensive tests to PR branches without modifying them.
 2021-06-22 17:23:59 +03:00
Oran Agra ae418eca24 Adjustments to recent RM_StringTruncate fix (#3718) (#9125) 
- Introduce a new sdssubstr api as a building block for sdsrange.
  The API of sdsrange is many times hard to work with and also has
  corner case that cause bugs. sdsrange is easy to work with and also
  simplifies the implementation of sdsrange.
- Revert the fix to RM_StringTruncate and just use sdssubstr instead of
  sdsrange.
- Solve valgrind warnings from the new tests introduced by the previous
  PR.
 2021-06-22 17:22:40 +03:00
Yossi Gottlieb a49b766860 Remove leftover after CONFIG SET bind change. (#9129) 2021-06-22 14:03:00 +03:00
Yossi Gottlieb 8284544adb Fix typo in test. (#9128) 2021-06-22 13:30:20 +03:00
Yossi Gottlieb 07b0d144ce Improve bind and protected-mode config handling. (#9034) 
* Specifying an empty `bind ""` configuration prevents Redis from listening on any TCP port. Before this commit, such configuration was not accepted.
* Using `CONFIG GET bind` will always return an explicit configuration value. Before this commit, if a bind address was not specified the returned value was empty (which was an anomaly).

Another behavior change is that modifying the `bind` configuration to a non-default value will NO LONGER DISABLE protected-mode implicitly.
 2021-06-22 12:50:17 +03:00
Evan 1ccf2ca2f4 modules: Add newlen == 0 handling to RM_StringTruncate (#3717) (#3718) 
Previously, passing 0 for newlen would not truncate the string at all.
This adds handling of this case, freeing the old string and creating a new empty string.

Other changes:
- Move `src/modules/testmodule.c` to `tests/modules/basics.c`
- Introduce that basic test into the test suite
- Add tests to cover StringTruncate
- Add `test-modules` build target for the main makefile
- Extend `distclean` build target to clean modules too
 2021-06-22 12:26:48 +03:00
Oran Agra d0819d618e solve test timing issues in replication tests (#9121) 
# replication-3.tcl
had a test timeout failure with valgrind on daily CI:
```
*** [err]: SLAVE can reload "lua" AUX RDB fields of duplicated scripts in tests/integration/replication-3.tcl
Replication not started.
```
replication took more than 70 seconds.
https://github.com/redis/redis/runs/2854037905?check_suite_focus=true

on my machine it takes only about 30, but i can see how 50 seconds isn't enough.

# replication.tcl
loading was over too quickly in freebsd daily CI:
```
*** [err]: slave fails full sync and diskless load swapdb recovers it in tests/integration/replication.tcl
Expected '0' to be equal to '1' (context: type eval line 44 cmd {assert_equal [s -1 loading] 1} proc ::start_server)
```

# rdb.tcl
loading was over too quickly.
increase the time loading takes, and decrease the amount of work we try to achieve in that time.
 2021-06-22 11:10:11 +03:00
Binbin bf92000e2d Remove extra semicolon (#9117) 
Remove extra semicolon.
 2021-06-21 22:53:28 -07:00
Wen Hui 81d5f05b6e add tilt mode since in sentinel info (#9000) 
Sentinel shows tilt mode boolean in info, now it'll show an indication of how long ago it started.
 2021-06-22 07:37:46 +03:00
Oran Agra 9b564b525d Fix race in client side tracking (#9116) 
The `Tracking gets notification of expired keys` test in tracking.tcl
used to hung in valgrind CI quite a lot.

It turns out the reason is that with valgrind and a busy machine, the
server cron active expire cycle could easily run in the same event loop
as the command that created `mykey`, so that when they key got expired,
there were two change events to broadcast, one that set the key and one
that expired it, but since we used raxTryInsert, the client that was
associated with the "last" change was the one that created the key, so
the NOLOOP filtered that event.

This commit adds a test that reproduces the problem by using lazy expire
in a multi-exec which makes sure the key expires in the same event loop
as the one that added it.
 2021-06-22 07:35:59 +03:00
Binbin f004073b54 Check cluster_enabled in readwriteCommand just like readonlyCommand. (#9015) 2021-06-21 14:02:44 -07:00
Maxim Galushka 96bb078577 redis-cli: support for REDIS_REPLY_SET in CSV and RAW output. (#7338) 
Fixes #6792. Added support of REDIS_REPLY_SET in raw and csv output of `./redis-cli`

Test:

run commands to test:
  ./redis-cli -3 --csv COMMAND
  ./redis-cli -3 --raw COMMAND

Now they are returning resuts, were failing with: "Unknown reply type: 10" before the change.
 2021-06-21 11:01:31 +03:00
Binbin 1fabb89a41 Make CONFIG GET OOM-score-adj-values case insensitive. (#9114) 2021-06-20 12:32:02 +03:00
SUN 4c52aa9faa chdir to dir before fopen logfile in loadServerConfigFromString() (#6741) 
Open the log file only after parsing the entire config file, so that it's
location isn't dependent on the order of configs (`dir` and `logfile`).
Also solves the problem of creating multiple log files if the `logfile`
directive appears many times in the config file.
 2021-06-20 11:34:04 +03:00
Binbin 561c69c285 Improve the debug help command message (#9098) 
cleanups:
1: Re-introduce debug leak subcommand in help text.
Mistankenly deleted in https://github.com/redis/redis/pull/5531

2: Formatted the text.
Some text lacks commas resulting in no line breaks.

3: Supplementary debug restart command descriptions of delay arg.
 2021-06-20 09:46:27 +03:00
Binbin 89152f8e41 Use loaderr label to handle error. (#9111) 
So that we can easily see the lines of the config.
Also unified with other error handling.
 2021-06-20 08:04:12 +03:00
gourav 6248127c43 Use exitFromChild to exit from child process in linuxMadvFreeForkBugCheck (#9101) 2021-06-17 13:37:38 -07:00
sundb 1a22445d30 Make readQueryFromClient more aggressive when reading big arg again (Followup for #9003) (#9100) 
Due to the change in #9003, a long-standing bug was raised under `valgrind`.
This bug can cause the master-slave sync to take a very long time, causing the `pendingquerybuf.tcl` test to fail.
This problem does not only occur in master-slave sync, it is triggered when the big arg is greater than 32k.
step:
```sh
dd if=/dev/zero of=bigfile bs=1M count=32
./src/redis-cli -x hset a a < bigfile
```

1) Make room for querybuf in processMultibulkBuffer, now the alloc of querybuf will be more than 32k.
2) If this happens to trigger the `clientsCronResizeQueryBuffer`, querybuf will be resized to 0.
3) Finally, in readQueryFromClient, we expand the querybuf non-greedily, from 0 to 32k.
    Old code, make room for querybuf is greedy, so it only needs 11 times to expand to 32M(16k*(2^11)),
    but now we need 2048(32*1024/16) times to reach it, due to the slow allocation under valgrind that exposed the problem.

The fix for the excessive shrinking of the query buf to 0, will be handled in #5013 (that other change on it's own can fix failing test too), but the fix in this PR will also fix the failing test.

The fix in this PR will makes the reading in `readQueryFromClient` more aggressive when working on a big arg (so that it is in par with the same code in `processMultibulkBuffer` (i.e. the two calls to `sdsMakeRoomForNonGreedy` should both use the bulk size).
In the code before this fix the one in readQueryFromClient always has `readlen = PROTO_IOBUF_LEN`
 2021-06-17 21:49:15 +03:00
sundb eae0983d2d Fix leak and double free issues in datatype2 module test (#9102) 
* Add missing call for RedisModule_DictDel in datatype2 test
* Fix memory leak in datatype2 test
 2021-06-17 21:45:21 +03:00
DarrenJiang13 ada60d8003 Improve objectComputeSize() with allocator fragmentaiton. (#9095) 
This commit improve MEMORY USAGE command to include internal fragmentation overheads of:
1. EMBSTR encoded strings
2. ziplist encoded zsets and hashes
3. List type nodes
 2021-06-17 13:30:37 +03:00
sundb b586d5b567 Fix querybuf test failure (#9091) 
Fix test failure which introduced by #9003.
The following case will occur when querybuf expansion will allocate memory equal to (16*1024)k.
1) make use ```CFLAGS=-DNO_MALLOC_USABLE_SIZE```.
2) ```malloc``` will not allocate more under ```alpine```.
 2021-06-16 22:01:37 +03:00
Sam Bortman c2b93ff83f Support glob pattern matching for config include files (#8980) 
This will allow distros to use an "include conf.d/*.conf" statement in the default configuration file
which will facilitate customization across upgrades/downgrades.

The change itself is trivial: instead of opening an individual file, the glob call creates a vector of files to open, and each file is opened in turn, and its content is added to the configuration.
 2021-06-16 21:59:38 +03:00
yoav-steinberg 362786c58a Remove gopher protocol support. (#9057) 
Gopher support was added mainly because it was simple (trivial to add).
But apparently even something that was trivial at the time, does cause complications
down the line when adding more features.
We recently ran into a few issues with io-threads conflicting with the gopher support.
We had to either complicate the code further in order to solve them, or drop gopher.
AFAIK it's completely unused, so we wanna chuck it, rather than keep supporting it.
 2021-06-16 09:47:25 +03:00
chenyang8094 e0cd3ad0de Enhance mem_usage/free_effort/unlink/copy callbacks and add GetDbFromIO api. (#8999) 
Create new module type enhanced callbacks: mem_usage2, free_effort2, unlink2, copy2.
These will be given a context point from which the module can obtain the key name and database id.
In addition the digest and defrag context can now be used to obtain the key name and database id.
 2021-06-16 09:45:49 +03:00
Jason Elbaum 7f342020dc Change return value type for ZPOPMAX/MIN in RESP3 (#8981) 
When using RESP3, ZPOPMAX/ZPOPMIN should return nested arrays for consistency
with other commands (e.g. ZRANGE).

We do that only when COUNT argument is present (similarly to how LPOP behaves).
for reasoning see https://github.com/redis/redis/issues/8824#issuecomment-855427955

This is a breaking change only when RESP3 is used, and COUNT argument is present!
 2021-06-16 09:29:57 +03:00
Uri Shachar c7e502a07b Cleaning up the cluster interface by moving almost all related declar… (#9080) 
* Cleaning up the cluster interface by moving almost all related declarations into cluster.h
(no logic change -- just moving declarations/definitions around)

This initial effort leaves two items out of scope - the configuration parsing into the server
struct and the internals exposed by the clusterNode struct.

* Remove unneeded declarations of dictSds*
Ideally all the dictSds functionality would move from server.c into a dedicated module
so we can avoid the duplication in redis-benchmark/cli

* Move crc16 back into server.h, will be moved out once we create a seperate header file for
hashing functions
 2021-06-15 20:35:13 -07:00
sundb e5d8a5eb85 Fix the wrong reisze of querybuf (#9003) 
The initialize memory of `querybuf` is `PROTO_IOBUF_LEN(1024*16) * 2` (due to sdsMakeRoomFor being greedy), under `jemalloc`, the allocated memory will be 40k.
This will most likely result in the `querybuf` being resized when call `clientsCronResizeQueryBuffer` unless the client requests it fast enough.

Note that this bug existed even before #7875, since the condition for resizing includes the sds headers (32k+6).

## Changes
1. Use non-greedy sdsMakeRoomFor when allocating the initial query buffer (of 16k).
1. Also use non-greedy allocation when working with BIG_ARG (we won't use that extra space anyway)
2. in case we did use a greedy allocation, read as much as we can into the buffer we got (including internal frag), to reduce system calls.
3. introduce a dedicated constant for the shrinking (same value as before)
3. Add test for querybuf.
4. improve a maxmemory test by ignoring the effect of replica query buffers (can accumulate many ACKs on slow env)
5. improve a maxmemory by disabling slowlog (it will cause slight memory growth on slow env).
 2021-06-15 14:46:19 +03:00
Binbin eb15c456c9 Check CLIENT_DIRTY_CAS flag before process transaction. (#9086) 
Do not queue command in an already aborted MULTI state.
We can detect an error (watched key).
So in queueMultiCommand, we also can return early.
Like we deal with `CLIENT_DIRTY_EXEC`.
 2021-06-15 14:36:04 +03:00
DarrenJiang13 678e67b58e presize hashtable to avoid rehashing when hashTypeConvertZiplist() (#8943) 2021-06-15 12:52:36 +03:00
Binbin b109977301 Fix XINFO help for unexpected options. (#9075) 
Small cleanup and consistency.
 2021-06-15 10:01:11 +03:00
yvette903 096c5fd5d2 Fix coredump after Client Unpause command when threaded I/O is enabled (#9041) 
Fix crash when using io-threads-do-reads and issuing CLIENT PAUSE and
CLIENT UNPAUSE.
This issue was introduced in redis 6.2 together with the FAILOVER command.
 2021-06-15 09:21:52 +03:00
Binbin 7900b48bc7 slowlog get command supports passing in -1 to get all logs. (#9018) 
This was already the case before this commit, but it wasn't clear / intended in the code, now it does.
 2021-06-14 16:46:45 +03:00
ZhaolongLiandlizhaolong.lzl dee378d76d Simplify a redundant condition in computeDefragCycles (#9076) 
Co-authored-by: lizhaolong.lzl <lizhaolong.lzl@B-54MPMD6R-0221.local>
 2021-06-14 11:48:15 +03:00
ZhaolongLiandlizhaolong.lzl 688cdb05b4 Remove dead code in defrag.c (ziplist encoded list) (#9074) 
Co-authored-by: lizhaolong.lzl <lizhaolong.lzl@B-54MPMD6R-0221.local>
 2021-06-14 10:55:17 +03:00
YaacovHazan 1677efb9da cleanup around loadAppendOnlyFile (#9012) 
Today when we load the AOF on startup, the loadAppendOnlyFile checks if
the file is openning for reading.
This check is redundent (dead code) as we open the AOF file for writing at initServer,
and the file will always be existing for the loadAppendOnlyFile.

In this commit:
- remove all the exit(1) from loadAppendOnlyFile, as it is the caller
  responsibility to decide what to do in case of failure.
- move the opening of the AOF file for writing, to be after we loading it.
- avoid return -ERR in DEBUG LOADAOF, when the AOF is existing but empty
 2021-06-14 10:38:08 +03:00
fishsalter 2727a2cec0 msetnx reply error (#9061) 2021-06-13 20:35:23 -07:00
Binbin b8a5da80c4 Fix accidental deletion of sinterstore command when we meet wrong type error. (#9032) 
SINTERSTORE would have deleted the dest key right away,
even when later on it is bound to fail on an (WRONGTYPE) error.

With this change it first picks up all the input keys, and only later
delete the dest key if one is empty.

Also add more tests for some commands.
Mainly focus on 
- `wrong type error`: 
	expand test case (base on sinter bug) in non-store variant
	add tests for store variant (although it exists in non-store variant, i think it would be better to have same tests)
- the dstkey result when we meet `non-exist key (empty set)` in *store

sdiff:
- improve test case about wrong type error (the one we found in sinter, although it is safe in sdiff)
- add test about using non-exist key (treat it like an empty set)
sdiffstore:
- according to sdiff test case, also add some tests about `wrong type error` and `non-exist key`
- the different is that in sdiffstore, we will consider the `dstkey` result

sunion/sunionstore add more tests (same as above)

sinter/sinterstore also same as above ...
 2021-06-13 10:53:46 +03:00
Binbin 5517f3624d Remove duplicate dbid lookup in performEvictions. (#9063) 
Minor code cleanup.
 2021-06-13 09:31:19 +03:00
ny0312 fb140a1bff Fix flaky test case for absolute TTL replication (#9069) 
The root cause is that one test (`5 keys in, 5 keys out`) is leaking a volatile key
that can expire while another later test(`All TTL in commands are propagated
as absolute timestamp in replication stream`) is running.
Such leaked expiration injects an unexpected `DEL` command into the
replication command during the later test, causing it to fail.

The fixes are two fold:
1. Plug the leak in the first test.
2. Add FLUSHALL to the later test, to avoid future interference from other tests.
 2021-06-13 08:42:20 +03:00
ZhaolongLi 63da66bb63 Delete an unnecessary function declaration (#9065) 2021-06-10 16:33:16 -07:00
Binbin 0bfccc55e2 Fixed some typos, add a spell check ci and others minor fix (#8890) 
This PR adds a spell checker CI action that will fail future PRs if they introduce typos and spelling mistakes.
This spell checker is based on blacklist of common spelling mistakes, so it will not catch everything,
but at least it is also unlikely to cause false positives.

Besides that, the PR also fixes many spelling mistakes and types, not all are a result of the spell checker we use.

Here's a summary of other changes:
1. Scanned the entire source code and fixes all sorts of typos and spelling mistakes (including missing or extra spaces).
2. Outdated function / variable / argument names in comments
3. Fix outdated keyspace masks error log when we check `config.notify-keyspace-events` in loadServerConfigFromString.
4. Trim the white space at the end of line in `module.c`. Check: https://github.com/redis/redis/pull/7751
5. Some outdated https link URLs.
6. Fix some outdated comment. Such as:
    - In README: about the rdb, we used to said create a `thread`, change to `process`
    - dbRandomKey function coment (about the dictGetRandomKey, change to dictGetFairRandomKey)
    - notifyKeyspaceEvent fucntion comment (add type arg)
    - Some others minor fix in comment (Most of them are incorrectly quoted by variable names)
7. Modified the error log so that users can easily distinguish between TCP and TLS in `changeBindAddr`
 2021-06-10 15:39:33 +03:00
Yossi Gottlieb 8a86bca5ed Improve test suite to handle external servers better. (#9033) 
This commit revives the improves the ability to run the test suite against
external servers, instead of launching and managing `redis-server` processes as
part of the test fixture.

This capability existed in the past, using the `--host` and `--port` options.
However, it was quite limited and mostly useful when running a specific tests.
Attempting to run larger chunks of the test suite experienced many issues:

* Many tests depend on being able to start and control `redis-server` themselves,
and there's no clear distinction between external server compatible and other
tests.
* Cluster mode is not supported (resulting with `CROSSSLOT` errors).

This PR cleans up many things and makes it possible to run the entire test suite
against an external server. It also provides more fine grained controls to
handle cases where the external server supports a subset of the Redis commands,
limited number of databases, cluster mode, etc.

The tests directory now contains a `README.md` file that describes how this
works.

This commit also includes additional cleanups and fixes:

* Tests can now be tagged.
* Tag-based selection is now unified across `start_server`, `tags` and `test`.
* More information is provided about skipped or ignored tests.
* Repeated patterns in tests have been extracted to common procedures, both at a
  global level and on a per-test file basis.
* Cleaned up some cases where test setup was based on a previous test executing
  (a major anti-pattern that repeats itself in many places).
* Cleaned up some cases where test teardown was not part of a test (in the
  future we should have dedicated teardown code that executes even when tests
  fail).
* Fixed some tests that were flaky running on external servers.
 2021-06-09 15:13:24 +03:00
Wang Yuan c396fd91a0 Mem efficiency, make full use of client struct memory for reply buffers (#8968) 
When we allocate a client struct with 16k reply buffer, the allocator we may give us 20K,
This commit makes use of that extra space.
Additionally, it tries to store whatever it can from the reply into the static 'buf' before
allocating a new node for the reply list.
 2021-06-08 13:40:12 +03:00
Yang Bodong 119121c739 Verbose log enhancement: print client info when client exit (#9053) 
It could be useful for debugging to know which client got disconnected.
 2021-06-08 12:39:27 +03:00
guybe7 e16d3eb998 Module API for current command name (#8792) 
sometimes you can be very deep in the call stack, without access to argv.
once you're there you may want your reply/log to contain the command name.
 2021-06-08 12:36:05 +03:00
ikeberlein 77d44bb7b4 Make redis-cli grep friendly in pubsub mode (#9013) 
redis-cli is grep friendly for all commands but SUBSCRIBE/PSUBSCRIBE.
it is unable to process output from these commands line by line piped
to another program because of output buffering. to overcome this
situation I propose to flush stdout each time when it is written with
reply from these commands the same way it is already done for all other
commands.
 2021-06-08 11:34:38 +03:00
Huang Zhw 1df8c129bc Fix some minor bugs in redis-cli (#8982) 
* clusterManagerAddSlots check argv_idx error.

* clusterManagerLoadInfoFromNode remove unused param opts.

* redis-cli node->ip may be an sds or a c string. Using %s to format
is always right, %S may be wrong.

* In clusterManagerFixOpenSlot clusterManagerBumpEpoch call is redundant,
because it is already called in clusterManagerSetSlotOwner.

* redis-cli cluster help add more commands in help messages.
 2021-06-08 11:25:52 +03:00
Fabian Eichinger 39b0f0dd73 Add support for combining NX and GET flags on SET command (#8906) 
Till now GET and NX were mutually exclusive.
This change make their combination mean a "Get or Set" command.

If the key exists it returns the old value and avoids setting,
and if it does't exist it returns nil and sets it to the new value (possibly with expiry time)
 2021-06-07 16:47:58 +03:00
Huang Zhw eaa7a7bb93 Fix XTRIM or XADD with LIMIT may delete more entries than Count. (#9048) 
The decision to stop trimming due to LIMIT in XADD and XTRIM was after the limit was reached.
i.e. the code was deleting **at least** that count of records (from the LIMIT argument's perspective, not the MAXLEN),
instead of **up to** that count of records.
see #9046
 2021-06-07 14:43:36 +03:00
Jeremy Kun b438bc5a0b comment typo: form -> from (#8921) 2021-06-07 14:31:56 +03:00
ZhaolongLiandlizhaolong.lzl a972503f57 Fix typo in comment of flushAppendOnlyFile (#9051) 
Co-authored-by: lizhaolong.lzl <lizhaolong.lzl@B-54MPMD6R-0221.local>
 2021-06-07 10:12:25 +03:00
Oran Agra 3e39ea0b83 tests: prevent name clash in variables leading to wrong test name (#8995) 
running the "geo" unit would have shown that it completed a unit named
"north". this was because the variable `$name` was overwritten.
This commit isn't perfect, but it slightly reduces the chance for
variable name clash.

```
$ ./runtest --single unit/geo
.......
Testing unit/geo
.......
[1/1 done]: north (15 seconds)
```
 2021-06-06 17:35:30 +03:00
Huang Zhw 91f3689bf5 XTRIM call streamParseAddOrTrimArgsOrReply use wrong arg xadd. (#9047) 
xtrimCommand call streamParseAddOrTrimArgsOrReply should use xadd==0.

When the syntax is valid, it does not cause any bugs because the params of XADD is superset of XTRIM.
Just XTRIM will not respond with error on invalid syntax. The syntax of XADD will also be accpeted by XTRIM.
 2021-06-06 15:09:06 +03:00
Binbin bb671d9016 Add missing zuiClearIterator in zrandmemberWithCountCommand. (#8979) 
Also the bug that currently does not cause memory leaks.
Because op->type = OBJ_ZSET, in zuiClearIterator, we will do nothing.
Just a cleanup that zuiInitIterator and zuiClearIterator should appear in pairs.
 2021-06-06 12:13:00 +03:00
Wen Hui 156599ba2d Fix SENTINEL SET to require arguments (#9037) 2021-06-04 18:33:06 +03:00
Oran Agra b512dfe794 tests: add details when test fails on malformed info (#9042) 2021-06-03 20:34:54 +03:00
Wen Hui 3fbdbf5084 document module unload error codes in comments (#9038) 2021-06-03 09:18:28 -07:00
Huang Zhw a049f6295a redis-cli cluster import command may issue wrong MIGRATE command. (#8945) 
In clusterManagerCommandImport strcat was used to concat COPY and
REPLACE, the space maybe not enough.
If we use --cluster-replace but not --cluster-copy, the MIGRATE
command contained COPY instead of REPLACE.
 2021-06-02 17:30:08 +03:00
pgxiaolianzi f63bb9583d Fix typo on buckup to backup (#8919) 2021-06-01 22:54:30 -07:00
Oran Agra 7cb42c9c36 add test for modules load/unload and config rewrite 2021-06-01 13:43:48 +03:00
zhaozhao.zz 2ff53060d2 Modules: show path and args in LIST reply 2021-06-01 13:43:48 +03:00
zhaozhao.zz babe3c7b29 Modules: rewrite config loadmodule option 2021-06-01 13:43:48 +03:00
zhaozhao.zz 09e435c8d4 Modules: free node after module loaded from server.loadmodule_queue 2021-06-01 13:43:48 +03:00
Oran Agra 1ddecf1958 Fix integer overflow in STRALGO LCS (CVE-2021-32625) (#9011) 
An integer overflow bug in Redis version 6.0 or newer can be exploited using the
STRALGO LCS command to corrupt the heap and potentially result with remote code
execution. This is a result of an incomplete fix by CVE-2021-29477.
 2021-06-01 09:12:45 +03:00
Oran Agra ae67539c8b Improve new time sensitive pexpireat propagation test (#9010) 
The test that was merged yesterday fails with valgrind and freebsd CI
that are too slow, and 10 seconds apparently passed between the time the
command was sent to redis and the time it was actually executed.

```
*** [err]: All TTL in commands are propagated as absolute timestamp in replication stream in tests/unit/expire.tcl
Expected 'del a' to match 'set foo1 bar PXAT *' (context: type source line 778 file /home/runner/work/redis/redis/tests/test_helper.tcl cmd {assert_match [lindex $patterns $j] [read_from_replication_stream $s]} proc ::assert_replication_stream level 1)
```
 2021-06-01 08:01:10 +03:00
Binbin 01495c674a Extend freeSlotsToKeysMapAsync and freeTrackingRadixTreeAsync to check LAZYFREE_THRESHOLD. (#8969) 
Without this fix, FLUSHALL ASYNC would have freed these in a background thread,
even if they didn't contain many elements (unlike how it works with other structures), which could be inefficient.
 2021-05-30 17:56:04 +03:00
Wang Yuan 58a03eca67 Make full use of aofrwblock's buf (#8975) 
Make aof rewrite buffer memory size more accurate, before, there may be 20%
deviation with its real memory usage.

The implication are both lower memory usage, and also a more accurate INFO.
 2021-05-30 11:57:36 +03:00
ny0312 53d1acd598 Always replicate time-to-live(TTL) as absolute timestamps in milliseconds (#8474) 
Till now, on replica full-sync we used to transfer absolute time for TTL,
however when a command arrived (EXPIRE or EXPIREAT),
we used to propagate it as is to replicas (possibly with relative time),
but always translate it to EXPIREAT (absolute time) to AOF.

This commit changes that and will always use absolute time for propagation.
see discussion in #8433

Furthermore, we Introduce new commands: `EXPIRETIME/PEXPIRETIME`
that allow extracting the absolute TTL time from a key.
 2021-05-30 09:20:32 +03:00
YaacovHazan 501d775583 unregister AE_READABLE from the read pipe in backgroundSaveDoneHandlerSocket (#8991) 
In diskless replication, we create a read pipe for the RDB, between the child and the parent.
When we close this pipe (fd), the read handler also needs to be removed from the event loop (if it still registered).
Otherwise, next time we will use the same fd, the registration will be fail (panic), because
we will use EPOLL_CTL_MOD (the fd still register in the event loop), on fd that already removed from epoll_ctl
 2021-05-26 14:51:53 +03:00
Wen Hui be6ce8a92a fix sentinel test failure (#8983) 
fix for recent breakage from #8958, did a mistake when updating the pr.
 2021-05-24 08:36:51 +03:00
Wen Hui ae6f58690b [SENTINEL] reset sentinel-user/pass to NULL when user config with empty string (#8958) 2021-05-23 14:31:01 +03:00
wuYin a319327ffb remove unnecessary determine statement in sdsrange (#8978) 
Already verified that start < len, so if end >= len, then start cannot be > len.
 2021-05-22 14:49:35 +03:00
YaacovHazan 32a2584e07 stabilize tests that involved with load handlers (#8967) 
When test stop 'load handler' by killing the process that generating the load,
some commands that already in the input buffer, still might be processed by the server.
This may cause some instability in tests, that count on that no more commands
processed after we stop the `load handler'

In this commit, new proc 'wait_load_handlers_disconnected' added, to verify that no more
cammands from any 'load handler' prossesed, by checking that the clients who
genreate the load is disconnceted.

Also, replacing check of dbsize with wait_for_ofs_sync before comparing debug digest, as
it would fail in case the last key the workload wrote was an overridden key (not a new one).

Affected tests
Race fix:
- failover command to specific replica works
- Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl
- AOF rewrite during write load: RDB preamble=$rdbpre

Cleanup and speedup:
- Test replication with blocking lists and sorted sets operations
- Test replication with parallel clients writing in different DBs
- Test replication partial resync: $descr (diskless: $mdl, $sdl, reconnect: $reconnect
 2021-05-20 15:29:43 +03:00
Wang YuanandOran Agra 8627751ec6 Make aof buf alloc size more accurate in overhead (#8970) 
In theory we should have used zmalloc_usable_size, but it may be slower,
and now after #7875, there's no actual difference.
So this change isn't making a dramatic change.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-05-20 15:04:17 +03:00
Wang Yuan 0b2d0be35a Make full use of replication backlog memory (#8966) 
According jemalloc size classes, we may allocate much more memory
than our setting of repl_backlog_size, but we don't make full use of it.
 2021-05-20 08:23:53 +03:00
Madelyn Olson a59e75a475 Hide migrate command from slowlog if they include auth (#8859) 
Redact commands that include sensitive data from slowlog and monitor
 2021-05-19 08:23:54 -07:00
Oran Agra d67e66de72 Fix race in new lazyfree test (#8965) 
I recently saw this failure:
[err]: lazy free a stream with all types of metadata in tests/unit/lazyfree.tcl
Expected '2' to be equal to '1' (context: type eval line 23 cmd {assert_equal [s lazyfreed_objects] 1} proc ::test)

The only explanation for such a thing is that the async flushdb wasn't
done before we did the resetstat
 2021-05-19 16:06:43 +03:00
Huang Zhw ed2602ab99 redis-cli clusterManagerCommandCreate calculate interleaved_len wrong (#8964) 
which make interleaved_len bigger and may access array out of range.
 2021-05-19 14:49:38 +03:00
Bonsai 20268d04c0 Fix different reply on SENTINEL SIMULATE-FAILURE help command (#8962) 2021-05-19 14:42:33 +03:00
DarrenJiang13 1bab4cd198 try a more concise rdbTryIntegerEncoding using string2ll rather than strtoll. (#8926) 
when string2ll was made to replace isStringRepresentableAsLongLong
(which was similar to what rdbTryIntegerEncoding does),
rdbTryIntegerEncoding was probably forgotten.
 2021-05-19 09:24:26 +03:00
Binbin 09a99fcd5f In activeDefragStringOb, we have took care defragged so not need to self-incr. (#8952) 
incrementing the defragged hits counter twice in activeDefragSdsListAndDict
 2021-05-18 18:33:01 +03:00
Eduardo FelipeEduardo Felipe CastegnaroOran Agra
 		25d827d949 Add documentation for firstkey, lastkey and keystep parameters of RedisModule_CreateCommand (#8883) 
These parameters of RedisModule_CreateCommand were previously
undocumented but they are needed for ACL to check permission on keys and
by Redis Cluster to figure our how to route the command.

Co-authored-by: Eduardo Felipe Castegnaro <edufelipe@onsign.tv>
Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-05-18 17:19:30 +03:00
Guy Korland e5a8928290 Avoid passing GeoHashRadius by value (#8941) 
This parameter of type GeoHashRadius is 192 bytes - passing a `const` pointer instead.
 2021-05-18 17:14:34 +03:00
Oran Agra 8458baf6a9 longer timeout in replication test (#8963) 
the test normally passes. but we saw one failure in a valgrind run in github actions
 2021-05-18 17:13:59 +03:00
Oran Agra 7b48de68ce Remove useless - from help.h on commands with no arguments (#8939) 
As far as i can tell it shows up in redis-cli in both HELP, e.g.
`help client list`, and also in the command completion tips, but it is
unclear what it was needed for.
It exists since the very first commit that added this mechanism.
 2021-05-18 17:13:10 +03:00
Oran Agra cf41c0b5ff fix race in config rewrite test (#8960) 2021-05-18 17:10:06 +03:00
Huang Zhw 55fa91ae53 redis-cli: some commands should bypass history file. (#8895) 
Currently in redis-cli only AUTH and ACL SETUSER bypass history
file. We add CONFIG SET masterauth/masteruser/requirepass,
HELLO with AUTH, MIGRATE with AUTH or AUTH2 to bypass history
file too.

The drawback is HELLO and MIGRATE's code is a mess. Someday if
we change these commands, we have to change here too.
 2021-05-18 10:30:24 +03:00
BinbinandOran Agra 8351a10b95 redis-cli: Sleep for a while in each cliConnect when we got connect error in cluster mode. (#8884) 
There's an infinite loop when redis-cli fails to connect in cluster mode.
This commit adds a 1 second sleep to prevent flooding the console with errors.
It also adds a specific error print in a few places that could have error without printing anything.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-05-18 10:09:45 +03:00
zhaozhao.zz 0ecc814c81 AOF: remove memmove in aofChildWriteDiffData and record the latency (#5362) 
Improve performance by avoiding inefficiencies in the parent process during AOFRW.
* AOF: record the latency of aofChildWriteDiffData
* AOF: avoid memmove in aofChildWriteDiffData
 2021-05-17 16:57:17 +03:00
Oran Agra fbc0e2b834 Reset lazyfreed_objects info field with RESETSTAT, test for stream lazyfree (#8934) 
And also add tests to cover lazy free of streams with various types of
metadata (see #8932)
 2021-05-17 16:54:37 +03:00
Huang Zhw 8827aae83b redis-cli: fix bugs in hints of commands with subcommands. (#8914) 
There are two bugs in redis-cli hints:
* The hints of commands with subcommands lack first params.
* When search matching command of currently input, we should find the
command with longest matching prefix. If not COMMAND INFO will always
match COMMAND and display no hints.
 2021-05-17 16:52:40 +03:00
patpatbearandsrzhao 46d9f31e94 sinterstore: add missing keyspace del event when any source set not exists. (#8949) 
this patch fixes sinterstore by add missing keyspace del event when any source set not exists.

Co-authored-by: srzhao <srzhao@sysnew.com>
 2021-05-15 15:43:09 +03:00
Madelyn OlsonandYossi Gottlieb df4d916007 Moved security bugs and vulnerability policy to SECURITY.md (#8938) 
Moved security bugs and vulnerability policy to SECURITY.MD and extended security policy.

Co-authored-by: Yossi Gottlieb <yossigo@gmail.com>
 2021-05-13 21:16:27 -07:00
Raghav Muddur 31edc22ecc EVALSHA_RO and EVAL_RO Commands (#8820) 
* EVALSHA_RO and EVAL_RO Commands

Added new readonly versions of EVAL
and EVALSHA.
 2021-05-12 21:07:34 -07:00
George Anderson Guimarães e01c92a5ef improve error handing in anetFDToString to avoid returning successfully with garbage. (#8678) 
Check for errors in inet_ntop and snprintf rather than ignore them
and return success (with garbage output).
The check for ip_len == 0 seems like dead code, removed.
 2021-05-11 14:27:08 +03:00
Huang Zhw 8526e04e7b cluster process publish message only for known nodes. (#8856) 2021-05-10 08:29:11 -07:00
Oran Agra 97108845e2 Fix crash unlinking a stream with groups rax and no groups (#8932) 
When estimating the effort for unlink, we try to compute the effort of
the first group and extrapolate.
If there's a groups rax that's empty, there'a an assertion.

reproduce:
xadd s * a b
xgroup create s bla $
xgroup destroy s bla
unlink s
 2021-05-10 17:08:43 +03:00
Binbin fb66c1b58c Fix typos, and consistent function argument names in quicklist (#8915) 
Also fixes a chance for compilation error if REDIS_TEST_VERBOSE would be used.
 2021-05-10 14:02:25 +03:00
Huang Zhw 4d4db9797f Cleanup: sentinel config, latency extra time() (#8810) 2021-05-09 18:45:45 +03:00
Oran Agra d32fd31588 redis-benchmark bad check for NOAUTH and NOPERM (#8931) 
also, print errors to stderr rather than stdout
 2021-05-09 18:19:25 +03:00
Binbin 56976ffb49 Use function instead of code in dict.c and delete dead code in dict.h (#8878) 
Use function instead of code in dict.c and delete dead code in dict.h
 2021-05-09 15:21:18 +03:00
Binbin 57b94eacaa Typo: Change port_get to port_getn in aeApiPoll panic message. (#8920) 2021-05-07 15:34:35 +03:00
Huang ZhwandOran Agra 6b47598998 redis-cli when SELECT fails, we should reset dbnum to 0 (#8898) 
when SELECT fails, we should reset dbnum to 0, so the prompt will not
display incorrectly.

Additionally when SELECT and HELLO fail, we output message to inform
it.

Add config.input_dbnum which means the dbnum about to select.
And config.dbnum means currently selected dbnum. When users succeed to
select db, config.dbnum and config.input_dbnum will be the same. When
users select db failed, config.input_dbnum will be kept. Next time if users
auth success, config.input_dbnum will be automatically selected.
When reconnect, we should select the origin dbnum.

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-05-06 18:34:45 +03:00
Oran Agra 4d1094e8be fix redis-benchmark to ignore unsupported configs (#8916) 
Redis Enterprise supports the CONFIG GET command, but it replies with am
empty array since the save and appendonly configs are not supported.
before this fix redis-benchmark would segfault for trying to access the
error string on an array type reply.
see #8869
 2021-05-06 17:45:49 +03:00
Wang Yuan 81e2d7272b Fix wrong COW memory in log (#8917) 
Always 0 MB of memory used by copy-on-write, introduced in #8645.
 2021-05-06 10:52:11 +03:00
Zerong Yao bcc519fa7c remove duplicate fd lookup in aeProcessEvents (#3886) 
Minor code cleanup.
 2021-05-05 15:12:53 +03:00
Binbin a62801eb53 redis-cli: Fix integer overflow in intrinsic-latency (#8907) 2021-05-04 20:21:14 +03:00
yoav-steinbergOran Agrasundb
 		152fce5e2c Enforce client output buffer soft limit when no traffic. (#8833) 
When client breached the output buffer soft limit but then went idle,
we didn't disconnect on soft limit timeout, now we do.
Note this also resolves some sporadic test failures in due to Linux
buffering data which caused tests to fail if during the test we went
back under the soft COB limit.

Co-authored-by: Oran Agra <oran@redislabs.com>
Co-authored-by: sundb <sundbcn@gmail.com>
 2021-05-04 13:45:08 +03:00
Oran Agra fd7d51c353 Resolve nonsense static analysis warnings 2021-05-03 18:59:47 +03:00
Oran Agra f0c5f920d0 Fix integer overflow in STRALGO LCS (CVE-2021-29477) 
An integer overflow bug in Redis version 6.0 or newer could be exploited using
the STRALGO LCS command to corrupt the heap and potentially result with remote
code execution.
 2021-05-03 18:59:47 +03:00
Oran Agra 29900d4e6b Fix integer overflow in intset (CVE-2021-29478) 
An integer overflow bug in Redis 6.2 could be exploited to corrupt the heap and
potentially result with remote code execution.

The vulnerability involves changing the default set-max-intset-entries
configuration value, creating a large set key that consists of integer values
and using the COPY command to duplicate it.

The integer overflow bug exists in all versions of Redis starting with 2.6,
where it could result with a corrupted RDB or DUMP payload, but not exploited
through COPY (which did not exist before 6.2).
 2021-05-03 18:59:47 +03:00
Wen Hui 611e11734c Add missing NOLOOP tracking flag in help (#8892) 2021-05-03 17:13:47 +03:00
Uri Shachar f3ee2d98af Fix typos in comments and improve readability (#8899) 2021-05-03 15:18:52 +03:00
BinbinandOran Agra ccb76e5688 Free value if dup succeed but listAddNodeTail failed. (#8901) 
This fix is in dead code.
see redisOutOfMemoryHandler an allocation can't fail.
but maybe someone will copy this code to a different project
some day, better have this fixed

Co-authored-by: Oran Agra <oran@redislabs.com>
 2021-05-03 13:39:07 +03:00
sundb 5100ef9f82 Fix memory leak in moduleDefragGlobals (#8853) 2021-05-02 10:32:57 +03:00
Huang Zhw 0b1b9edb28 Improve redis-cli help. When help command, we only match command (#8879) 
prefix args not all args. So when we help commands with subcommands,
all subcommands will be output.
 2021-04-29 12:08:52 +03:00
Binbin 9c927e9de9 Delete some unimplemented prototype. (#8882) 
Remove forward declarations from header files to functions that do not exist.
 2021-04-29 08:25:10 +03:00
Binbin 1eff8564c7 redis-benchmark: Add zfree(data) and fix lrange size / text mismatch (#8872) 
missing zfree(data) in redis-benchmark.

And also correct the wrong size in lrange.
the text mentioned 500, but size was 450, changed to 500
 2021-04-28 16:03:24 +03:00
Binbin 416f277339 redis-cli: Do not use hostsocket when we got redirected in cluster mode (#8870) 
When redis-cli was used with both -c (cluster) and -s (unix socket),
it would have kept trying to use that unix socket, even if it got
redirected by the cluster (resulting in an infinite loop).
 2021-04-28 13:19:55 +03:00
filipe oliveira ef6f902372 redis-benchmark: Error/Warning handling updates. (#8869) 
- Immediately exit on errors that are not related to topology updates.
- Deprecates the `-e` option ( retro compatible ) and warns that we now
  exit immediately on errors that are not related to topology updates.
- Fixed wrongfully failing on config fetch error (warning only). This only affects RE.

Bottom line:
- MOVED and ASK errors will not show any warning (unlike the throttled error with `-e` before).
- CLUSTERDOWN still prints an error unconditionally and sleeps for 1 second.
- other errors are fatal.
 2021-04-28 09:51:07 +03:00
Huang Zhw 2b22fffc78 Fix potential CONFIG SET bind test failure. (#8875) 
Use an invalid IP address to trigger CONFIG SET bind failure, instead of DNS which is not guaranteed to always fail.
 2021-04-27 18:02:23 +03:00
yoav-steinberg 2e88b06396 Bump freebsd-vm version to fix CI failures (#8876) 
Specifically we had issues with NTP sync failure which was resolved here: https://github.com/vmactions/freebsd-vm/commit/457af7345642e154a79d219971a2d4a7c7fe2118
 2021-04-27 16:22:22 +03:00
Andy Pan 3d56be3788 Fixes some typos (#8874) 2021-04-27 14:25:02 +03:00
avimredis 93e7750699 GEOSEARCHSTORE: removed unsupported args from comment (#8873) 2021-04-27 10:37:18 +03:00
Oran Agra 46f4ebbe84 Prevent replicas from sending commands that interact with keyspace (#8868) 
This solves an issue reported in #8712 in which a replica would bypass
the client write pause check and cause an assertion due to executing a
write command during failover.

The fact is that we don't expect replicas to execute any command other
than maybe REPLCONF and PING, etc. but matching against the ADMIN
command flag is insufficient, so instead i just block keyspace access
for now.
 2021-04-27 08:15:10 +03:00
sundb bcf1b384e3 Remove dead code from lpStringToInt64() (#8863) 2021-04-26 16:40:58 -07:00
avimredis 4bfcf4654a Fix error reply of GEOSEARCHSTORE unsupported args (#8865) 
When [WITHCOORD] [WITHDIST] [WITHHASH] are used, the error mentioned GEORADIUS command
 2021-04-26 21:11:38 +03:00
Yossi Gottlieb ebfbb09109 Remove redundant -latomic on arm64. (#8867) 2021-04-26 18:43:57 +03:00
sundb 074e28a46e Simplify rax free for RM_FreeServerInfo (#8866) 2021-04-26 16:30:34 +03:00
Andy Pan a8b6596d23 Fail fast when systemic error occurs in poll (#8749) 
Most of the ae.c backends didn't explicitly handle errors, and instead
ignored all errors and did an implicit retry.
This is desired for EAGAIN and EINTER, but in case of other systematic
errors, we prefer to fail and log the error we got rather than get into a busy loop.
 2021-04-26 15:52:06 +03:00
Guy Korland d1ca1e665f Add codeql static analysis in PR CI (#8854) 2021-04-26 12:55:54 +03:00
Huang Zhw 20735a5354 Distinguish between pong and meet messages in clusterProcessPacket log (#8848) 
Fix clusterProcessPacket log don't distinguish between pong and meet.
Additionally log node name instead of pointer and delete a redundant
log.
 2021-04-25 19:58:54 -07:00
Huang Zhw 72aa376237 Fix comments and typos in sentinel.c. (#8801) 2021-04-25 15:50:15 +03:00
Yang Bodong 8423b77f14 When the password is wrong, redis-benchmark should exit (#8855) 2021-04-25 14:00:35 +03:00
Oran Agra 611959eee5 fuzz tester, try to print hung command (#8837) 2021-04-25 13:08:46 +03:00
ZheNing Hu 828ae842b9 Improve ziplistRandomPairs code logic (#8851) 
After sorting, each item in picks is sorted according
to its index.

In the original code logic, we traverse from the first
element of ziplist until `zipindex == picks[pickindex].index`.

We may be able to start directly in `picks[0].index`,
this will bring small performance improvement.

Signed-off-by: ZheNing Hu <adlternative@gmail.com>
 2021-04-25 10:53:53 +03:00
sundb f29a0b9351 Fix some wrongly used constants in module.c (#8844) 
This change does not fix any bugs.
1. ```moduleUnload``` should return ```C_OK``` or ```C_ERR```, not ```REDISMODULE_ERR``` or ```REDISMODULE_OK```.
2. The ```where``` parameter of ```listTypePush``` and ```listTypePop``` should be ```LIST_HEAD``` or ```LIST_TAIL```
 2021-04-25 10:36:02 +03:00
pfee cc0091f0f9 Typo fix in gen-test-certs.sh (#8841) 2021-04-25 10:11:16 +03:00
Istemi Ekin Akkus af035c1e1d Modules: Fix RM_GetClusterNodeInfo() to correctly populate the master_id (#8846) 2021-04-25 10:05:12 +03:00
bugwz e43dbd03f3 Fixes a comment typo in sentinel.c (#8845) 2021-04-24 14:19:25 +03:00
彬彬同学丶 c4a5816ccd Changes http to https in README.md (#8843) 2021-04-24 14:16:39 +03:00
zyxwvu Shi f61c37cec9 Use monotonic clock to check for Lua script timeout. (#8812) 
This prevents a case where NTP moves the system clock
forward resulting in a false detection of a busy script.

Signed-off-by: zyxwvu Shi <i@shiyc.cn>
 2021-04-22 08:59:10 +03:00
Wang Yuan 63acfe4b00 Expire key firstly and then notify keyspace event (#8830) 
Modules event subscribers may get wrong things in notifyKeyspaceEvent callback,
such as wrong number of keys, or be able to lookup this key.
This commit changes the order to be like the one in evict.c.

Cleanup:
Since we know the key exists (it expires now), db*Delete is sure to return 1,
so there's no need to check it's output (misleading).
 2021-04-22 08:32:43 +03:00
Javier Li Sam b9d7d55b2c Update redis conf (#8707) 
* Fix typo for compatible
* Trim trailing whitespace
 2021-04-21 13:51:52 +03:00
Ikko Ashimine ac36161694 Fix typo in server.c (#8809) 2021-04-21 13:43:38 +03:00
xsddz 473588c399 Fix typo in rdb.c (#8827) 2021-04-21 13:43:06 +03:00
Uri Shachar 9fab26904f Fix typo and (hopefully) slightly improve comment readability (#8834) 2021-04-21 13:42:46 +03:00
Seunghyun Lee 8636c3e003 Fix typo in redis-cli.c (#8772) 2021-04-21 13:41:25 +03:00
Oran Agraandlibo bb16cf0f4f fix anetGenericAccept error return check (#8835) 
the anetGenericAccept return ANET_ERR but outside use constant -1

Co-authored-by: libo <buaa_libo@126.com>
 2021-04-21 13:33:18 +03:00
Andy Pan 5f6e166bce Shrink code lines and reduce code cyclomatic complexity (#8748) 
code cleanup in anetGenericAccept
 2021-04-21 09:04:36 +03:00
bugwz 761d7d2771 Print the number of abnormal line in AOF (#8823) 
When redis-check-aof finds an error, it prints the line number for faster troubleshooting.
 2021-04-20 21:51:24 +03:00
Huang Zhw 080d4579db Fix migrateCommand may migrate wrong value. (#8815) 
This scene is hard to happen. When first attempt some keys expired,
only kv position is updated not ov. Then socket err happens, second
attempt is taken. This time kv items may be mismatching with ov items.
 2021-04-20 10:59:44 +03:00
Madelyn Olson c73b4ddfd9 Fix memory leak when doing lazyfreeing client tracking table (#8822) 
Interior rax pointers were not being freed
 2021-04-19 22:16:27 -07:00
Bonsai c2aff4c744 clean dead assignment, start is never used in sdstrim function (#8796) 2021-04-19 21:37:37 -07:00
996 changed files with 106479 additions and 22636 deletions
Expand all files Collapse all files
.codespell/.codespellrc
+5
View File
@@ -0,0 +1,5 @@
[codespell]
quiet-level = 2
count =
skip = ./deps,./src/crc16_slottable.h,tmp*,./.git,./lcov-html
ignore-words = ./.codespell/wordlist.txt
.codespell/requirements.txt
+1
View File
@@ -0,0 +1 @@
codespell==2.1.0
.codespell/wordlist.txt
+18
View File
@@ -0,0 +1,18 @@
ake
bale
fle
fo
gameboy
mutli
nd
nees
oll
optin
ot
smove
te
tre
cancelability
ist
statics
filetest
.gitattributes
+6
View File
@@ -0,0 +1,6 @@
# We set commands.c's merge driver to `binary` so when it conflicts during a
# merge git will leave the local version unmodified. This way our Makefile
# will rebuild it based on src/commands/*.json before trying to compile it.
# Otherwise the file gets modified and gets the same timestamp as the .json
# files. So the Makefile doesn't attempt to rebuild it before compiling.
src/commands.c merge=binary
.github/ISSUE_TEMPLATE/crash_report.md
+7 -2
View File
@@ -1,12 +1,17 @@
---
name: Crash report
about: Submit a crash report
title: '[CRASH]'
title: '[CRASH] <short description>'
labels: ''
assignees: ''
---
Notice!
- If a Redis module was involved, please open an issue in the module's repo instead!
- If you're using docker on Apple M1, please make sure the image you're using was compiled for ARM!
**Crash report**
Paste the complete crash log between the quotes below. Please include a few lines from the log preceding the crash report to provide some context.
@@ -14,7 +19,7 @@ Paste the complete crash log between the quotes below. Please include a few line
```
```
**Aditional information**
**Additional information**
1. OS distribution and version
2. Steps to reproduce (if any)
.github/ISSUE_TEMPLATE/question.md
+2 -2
View File
@@ -14,8 +14,8 @@ Generally, questions about using Redis should be directed to the [community](htt
* [the mailing list](https://groups.google.com/forum/#!forum/redis-db)
* [the `redis` tag at StackOverflow](http://stackoverflow.com/questions/tagged/redis)
* [/r/redis subreddit](http://www.reddit.com/r/redis)
* [the irc channel #redis](http://webchat.freenode.net/?channels=redis) on freenode
* [github discussions](https://github.com/redis/redis/discussions)
It is also possible that your question was already asked here, so please do a quick issues search before submitting. Lastly, if your question is about one of Redis' [clients](https://redis.io/clients), you may to contact your client's developers for help.
That said, please feel free to replace all this with your question :)
That said, please feel free to replace all this with your question :)
.github/dependabot.yml
+15
View File
@@ -0,0 +1,15 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: github-actions
directory: /
schedule:
interval: weekly
- package-ecosystem: pip
directory: /.codespell
schedule:
interval: weekly
.github/workflows/ci.yml
+28 -12
View File
@@ -7,36 +7,52 @@ jobs:
test-ubuntu-latest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: make
# Fail build if there are warnings
# build with TLS just for compilation coverage
run: make REDIS_CFLAGS='-Werror' BUILD_TLS=yes
- name: test
run: |
sudo apt-get install tcl8.6
./runtest --verbose
sudo apt-get install tcl8.6 tclx
./runtest --verbose --tags -slow --dump-logs
- name: module api test
run: ./runtest-moduleapi --verbose
run: ./runtest-moduleapi --verbose --dump-logs
build-ubuntu-old:
runs-on: ubuntu-16.04
test-sanitizer-address:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: make
run: make SANITIZER=address REDIS_CFLAGS='-Werror'
- name: testprep
run: sudo apt-get install tcl8.6 tclx -y
- name: test
run: ./runtest --verbose --tags -slow --dump-logs
- name: module api test
run: ./runtest-moduleapi --verbose --dump-logs
build-debian-old:
runs-on: ubuntu-latest
container: debian:oldoldstable
steps:
- uses: actions/checkout@v3
- name: make
run: make REDIS_CFLAGS='-Werror'
run: |
apt-get update && apt-get install -y build-essential
make REDIS_CFLAGS='-Werror'
build-macos-latest:
runs-on: macos-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: make
run: make REDIS_CFLAGS='-Werror'
build-32bit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: make
run: |
sudo apt-get update && sudo apt-get install libc6-dev-i386
@@ -45,7 +61,7 @@ jobs:
build-libc-malloc:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: make
run: make REDIS_CFLAGS='-Werror' MALLOC=libc
@@ -53,7 +69,7 @@ jobs:
runs-on: ubuntu-latest
container: centos:7
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: make
run: |
yum -y install gcc make
.github/workflows/codeql-analysis.yml
+33
View File
@@ -0,0 +1,33 @@
name: "CodeQL"
on:
pull_request:
schedule:
# run weekly new vulnerability was added to the database
- cron: '0 0 * * 0'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
if: github.event_name != 'schedule' || github.repository == 'redis/redis'
strategy:
fail-fast: false
matrix:
language: [ 'cpp' ]
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
- name: Autobuild
uses: github/codeql-action/autobuild@v2
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
.github/workflows/daily.yml
+649 -131
View File
@@ -7,302 +7,820 @@ on:
- '[0-9].[0-9]'
schedule:
- cron: '0 0 * * *'
workflow_dispatch:
inputs:
skipjobs:
description: 'jobs to skip (delete the ones you wanna keep, do not leave empty)'
default: 'valgrind,sanitizer,tls,freebsd,macos,alpine,32bit,iothreads,ubuntu,centos,malloc'
skiptests:
description: 'tests to skip (delete the ones you wanna keep, do not leave empty)'
default: 'redis,modules,sentinel,cluster,unittest'
test_args:
description: 'extra test arguments'
default: ''
cluster_test_args:
description: 'extra cluster / sentinel test arguments'
default: ''
use_repo:
description: 'repo owner and name'
default: 'redis/redis'
use_git_ref:
description: 'git branch or sha to use'
default: 'unstable'
jobs:
test-ubuntu-jemalloc:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'ubuntu')
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
echo "skipping: ${{github.event.inputs.skipjobs}} and ${{github.event.inputs.skiptests}}"
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make REDIS_CFLAGS='-Werror -DREDIS_TEST'
- name: testprep
run: sudo apt-get install tcl8.6 tclx
- name: test
run: |
sudo apt-get install tcl8.6
./runtest --accurate --verbose --dump-logs
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
run: ./runtest-moduleapi --verbose
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
run: ./runtest-sentinel
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
run: ./runtest-cluster
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
- name: unittest
run: ./src/redis-server test all
if: true && !contains(github.event.inputs.skiptests, 'unittest')
run: ./src/redis-server test all --accurate
test-ubuntu-libc-malloc:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'malloc')
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: make
run: make MALLOC=libc
- name: test
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
sudo apt-get install tcl8.6
./runtest --accurate --verbose --dump-logs
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make MALLOC=libc REDIS_CFLAGS='-Werror'
- name: testprep
run: sudo apt-get install tcl8.6 tclx
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
run: ./runtest-moduleapi --verbose
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
run: ./runtest-sentinel
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
run: ./runtest-cluster
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
test-ubuntu-no-malloc-usable-size:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'malloc')
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: make
run: make MALLOC=libc CFLAGS=-DNO_MALLOC_USABLE_SIZE
- name: test
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
sudo apt-get install tcl8.6
./runtest --accurate --verbose --dump-logs
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make MALLOC=libc CFLAGS=-DNO_MALLOC_USABLE_SIZE REDIS_CFLAGS='-Werror'
- name: testprep
run: sudo apt-get install tcl8.6 tclx
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
run: ./runtest-moduleapi --verbose
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
run: ./runtest-sentinel
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
run: ./runtest-cluster
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
test-ubuntu-32bit:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, '32bit')
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
sudo apt-get update && sudo apt-get install libc6-dev-i386
make 32bit REDIS_CFLAGS='-Werror -DREDIS_TEST'
- name: testprep
run: sudo apt-get install tcl8.6 tclx
- name: test
run: |
sudo apt-get install tcl8.6
./runtest --accurate --verbose --dump-logs
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: |
make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time
./runtest-moduleapi --verbose
./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
run: ./runtest-sentinel
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
run: ./runtest-cluster
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
- name: unittest
run: ./src/redis-server test all
if: true && !contains(github.event.inputs.skiptests, 'unittest')
run: ./src/redis-server test all --accurate
test-ubuntu-tls:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'tls')
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
make BUILD_TLS=yes
- name: test
make BUILD_TLS=yes REDIS_CFLAGS='-Werror'
- name: testprep
run: |
sudo apt-get install tcl8.6 tcl-tls
sudo apt-get install tcl8.6 tclx tcl-tls
./utils/gen-test-certs.sh
./runtest --accurate --verbose --tls --dump-logs
./runtest --accurate --verbose --dump-logs
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: |
./runtest --accurate --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: |
./runtest-moduleapi --verbose --tls
./runtest-moduleapi --verbose
./runtest-moduleapi --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: |
./runtest-sentinel --tls
./runtest-sentinel
./runtest-sentinel --tls ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: |
./runtest-cluster --tls
./runtest-cluster
./runtest-cluster --tls ${{github.event.inputs.cluster_test_args}}
test-ubuntu-tls-no-tls:
runs-on: ubuntu-latest
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'tls')
timeout-minutes: 14400
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
make BUILD_TLS=yes REDIS_CFLAGS='-Werror'
- name: testprep
run: |
sudo apt-get install tcl8.6 tclx tcl-tls
./utils/gen-test-certs.sh
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: |
./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: |
./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: |
./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: |
./runtest-cluster ${{github.event.inputs.cluster_test_args}}
test-ubuntu-io-threads:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'iothreads')
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
make
make REDIS_CFLAGS='-Werror'
- name: testprep
run: sudo apt-get install tcl8.6 tclx
- name: test
run: |
sudo apt-get install tcl8.6 tcl-tls
./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags network --dump-logs ${{github.event.inputs.test_args}}
- name: cluster tests
run: |
./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster --config io-threads 4 --config io-threads-do-reads yes ${{github.event.inputs.cluster_test_args}}
test-valgrind:
test-valgrind-test:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'valgrind') && !contains(github.event.inputs.skiptests, 'redis')
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make valgrind REDIS_CFLAGS='-Werror -DREDIS_TEST'
- name: test
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 valgrind -y
./runtest --valgrind --verbose --clients 1 --dump-logs
sudo apt-get install tcl8.6 tclx valgrind -y
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}}
test-valgrind-misc:
runs-on: ubuntu-latest
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'valgrind') && !(contains(github.event.inputs.skiptests, 'modules') && contains(github.event.inputs.skiptests, 'unittest'))
timeout-minutes: 14400
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make valgrind REDIS_CFLAGS='-Werror -DREDIS_TEST'
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 tclx valgrind -y
- name: module api test
run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}}
- name: unittest
if: true && !contains(github.event.inputs.skiptests, 'unittest')
run: |
valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all
if grep -q 0x err.txt; then cat err.txt; exit 1; fi
test-valgrind-no-malloc-usable-size:
test-valgrind-no-malloc-usable-size-test:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'valgrind') && !contains(github.event.inputs.skiptests, 'redis')
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE"
- name: test
run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE -DREDIS_TEST" REDIS_CFLAGS='-Werror'
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 valgrind -y
./runtest --valgrind --verbose --clients 1 --dump-logs
sudo apt-get install tcl8.6 tclx valgrind -y
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}}
test-valgrind-no-malloc-usable-size-misc:
runs-on: ubuntu-latest
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'valgrind') && !(contains(github.event.inputs.skiptests, 'modules') && contains(github.event.inputs.skiptests, 'unittest'))
timeout-minutes: 14400
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make valgrind CFLAGS="-DNO_MALLOC_USABLE_SIZE -DREDIS_TEST" REDIS_CFLAGS='-Werror'
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 tclx valgrind -y
- name: module api test
run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}}
- name: unittest
if: true && !contains(github.event.inputs.skiptests, 'unittest')
run: |
valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all
if grep -q 0x err.txt; then cat err.txt; exit 1; fi
test-sanitizer-address:
runs-on: ubuntu-latest
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'sanitizer')
timeout-minutes: 14400
strategy:
matrix:
compiler: [ gcc, clang ]
env:
CC: ${{ matrix.compiler }}
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make SANITIZER=address REDIS_CFLAGS='-DREDIS_TEST -Werror'
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 tclx -y
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
- name: unittest
if: true && !contains(github.event.inputs.skiptests, 'unittest')
run: ./src/redis-server test all
test-sanitizer-undefined:
runs-on: ubuntu-latest
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'sanitizer')
timeout-minutes: 14400
strategy:
matrix:
compiler: [ gcc, clang ]
env:
CC: ${{ matrix.compiler }}
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make SANITIZER=undefined REDIS_CFLAGS='-DREDIS_TEST -Werror' LUA_DEBUG=yes # we (ab)use this flow to also check Lua C API violations
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 tclx -y
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
- name: unittest
if: true && !contains(github.event.inputs.skiptests, 'unittest')
run: ./src/redis-server test all --accurate
test-centos7-jemalloc:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'centos')
container: centos:7
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
yum -y install gcc make
make
make REDIS_CFLAGS='-Werror'
- name: testprep
run: yum -y install which tcl tclx
- name: test
run: |
yum -y install which tcl
./runtest --accurate --verbose --dump-logs
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
run: ./runtest-moduleapi --verbose
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
run: ./runtest-sentinel
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
run: ./runtest-cluster
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
test-centos7-tls:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'tls')
container: centos:7
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
yum -y install centos-release-scl epel-release
yum -y install devtoolset-7 openssl-devel openssl
scl enable devtoolset-7 "make BUILD_TLS=yes"
- name: test
scl enable devtoolset-7 "make BUILD_TLS=yes REDIS_CFLAGS='-Werror'"
- name: testprep
run: |
yum -y install tcl tcltls
yum -y install tcl tcltls tclx
./utils/gen-test-certs.sh
./runtest --accurate --verbose --tls --dump-logs
./runtest --accurate --verbose --dump-logs
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: |
./runtest --accurate --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: |
./runtest-moduleapi --verbose --tls
./runtest-moduleapi --verbose
./runtest-moduleapi --verbose --dump-logs --tls --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: |
./runtest-sentinel --tls
./runtest-sentinel
./runtest-sentinel --tls ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: |
./runtest-cluster --tls
./runtest-cluster
./runtest-cluster --tls ${{github.event.inputs.cluster_test_args}}
test-centos7-tls-no-tls:
runs-on: ubuntu-latest
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'tls')
container: centos:7
timeout-minutes: 14400
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
yum -y install centos-release-scl epel-release
yum -y install devtoolset-7 openssl-devel openssl
scl enable devtoolset-7 "make BUILD_TLS=yes REDIS_CFLAGS='-Werror'"
- name: testprep
run: |
yum -y install tcl tcltls tclx
./utils/gen-test-certs.sh
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: |
./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: |
./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: |
./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: |
./runtest-cluster ${{github.event.inputs.cluster_test_args}}
test-macos-latest:
runs-on: macos-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'macos') && !(contains(github.event.inputs.skiptests, 'redis') && contains(github.event.inputs.skiptests, 'modules'))
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: make
run: make
- name: test
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
./runtest --accurate --verbose --no-latency --dump-logs
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make REDIS_CFLAGS='-Werror'
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --no-latency --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
run: ./runtest-moduleapi --verbose
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --no-latency --dump-logs ${{github.event.inputs.test_args}}
test-macos-latest-sentinel:
runs-on: macos-latest
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'macos') && !contains(github.event.inputs.skiptests, 'sentinel')
timeout-minutes: 14400
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make REDIS_CFLAGS='-Werror'
- name: sentinel tests
run: ./runtest-sentinel
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
test-macos-latest-cluster:
runs-on: macos-latest
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'macos') && !contains(github.event.inputs.skiptests, 'cluster')
timeout-minutes: 14400
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make REDIS_CFLAGS='-Werror'
- name: cluster tests
run: ./runtest-cluster
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
test-freebsd:
runs-on: macos-latest
if: github.repository == 'redis/redis'
runs-on: macos-12
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'freebsd') && !(contains(github.event.inputs.skiptests, 'redis') && contains(github.event.inputs.skiptests, 'modules'))
timeout-minutes: 14400
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: test
uses: vmactions/freebsd-vm@v0.1.2
uses: vmactions/freebsd-vm@v0.3.0
with:
usesh: true
sync: rsync
prepare: pkg install -y bash gmake lang/tcl86
copyback: false
prepare: pkg install -y bash gmake lang/tcl86 lang/tclx
run: >
gmake &&
./runtest --accurate --verbose --no-latency --dump-logs &&
MAKE=gmake ./runtest-moduleapi --verbose &&
./runtest-sentinel &&
./runtest-cluster
gmake || exit 1 ;
if echo "${{github.event.inputs.skiptests}}" | grep -vq redis ; then ./runtest --verbose --timeout 2400 --no-latency --dump-logs ${{github.event.inputs.test_args}} || exit 1 ; fi ;
if echo "${{github.event.inputs.skiptests}}" | grep -vq modules ; then MAKE=gmake ./runtest-moduleapi --verbose --timeout 2400 --no-latency --dump-logs ${{github.event.inputs.test_args}} || exit 1 ; fi ;
test-freebsd-sentinel:
runs-on: macos-12
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'freebsd') && !contains(github.event.inputs.skiptests, 'sentinel')
timeout-minutes: 14400
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: test
uses: vmactions/freebsd-vm@v0.3.0
with:
usesh: true
sync: rsync
copyback: false
prepare: pkg install -y bash gmake lang/tcl86 lang/tclx
run: >
gmake || exit 1 ;
if echo "${{github.event.inputs.skiptests}}" | grep -vq sentinel ; then ./runtest-sentinel ${{github.event.inputs.cluster_test_args}} || exit 1 ; fi ;
test-freebsd-cluster:
runs-on: macos-12
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'freebsd') && !contains(github.event.inputs.skiptests, 'cluster')
timeout-minutes: 14400
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: test
uses: vmactions/freebsd-vm@v0.3.0
with:
usesh: true
sync: rsync
copyback: false
prepare: pkg install -y bash gmake lang/tcl86 lang/tclx
run: >
gmake || exit 1 ;
if echo "${{github.event.inputs.skiptests}}" | grep -vq cluster ; then ./runtest-cluster ${{github.event.inputs.cluster_test_args}} || exit 1 ; fi ;
test-alpine-jemalloc:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'alpine')
container: alpine:latest
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
apk add build-base
make REDIS_CFLAGS='-Werror'
- name: testprep
run: apk add tcl procps tclx
- name: test
run: |
apk add tcl procps
./runtest --accurate --verbose --dump-logs
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
run: ./runtest-moduleapi --verbose
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
run: ./runtest-sentinel
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
run: ./runtest-cluster
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
test-alpine-libc-malloc:
runs-on: ubuntu-latest
if: github.repository == 'redis/redis'
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'alpine')
container: alpine:latest
steps:
- uses: actions/checkout@v2
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
apk add build-base
make REDIS_CFLAGS='-Werror' USE_JEMALLOC=no CFLAGS=-DUSE_MALLOC_USABLE_SIZE
- name: testprep
run: apk add tcl procps tclx
- name: test
run: |
apk add tcl procps
./runtest --accurate --verbose --dump-logs
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
run: ./runtest-moduleapi --verbose
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
run: ./runtest-sentinel
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
run: ./runtest-cluster
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
.github/workflows/external.yml
+82
View File
@@ -0,0 +1,82 @@
name: External Server Tests
on:
pull_request:
push:
schedule:
- cron: '0 0 * * *'
jobs:
test-external-standalone:
runs-on: ubuntu-latest
if: github.event_name != 'schedule' || github.repository == 'redis/redis'
timeout-minutes: 14400
steps:
- uses: actions/checkout@v3
- name: Build
run: make REDIS_CFLAGS=-Werror
- name: Start redis-server
run: |
./src/redis-server --daemonize yes --save "" --logfile external-redis.log \
--enable-protected-configs yes --enable-debug-command yes --enable-module-command yes
- name: Run external test
run: |
./runtest \
--host 127.0.0.1 --port 6379 \
--tags -slow
- name: Archive redis log
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: test-external-redis-log
path: external-redis.log
test-external-cluster:
runs-on: ubuntu-latest
if: github.event_name != 'schedule' || github.repository == 'redis/redis'
timeout-minutes: 14400
steps:
- uses: actions/checkout@v3
- name: Build
run: make REDIS_CFLAGS=-Werror
- name: Start redis-server
run: |
./src/redis-server --cluster-enabled yes --daemonize yes --save "" --logfile external-redis.log \
--enable-protected-configs yes --enable-debug-command yes --enable-module-command yes
- name: Create a single node cluster
run: ./src/redis-cli cluster addslots $(for slot in {0..16383}; do echo $slot; done); sleep 5
- name: Run external test
run: |
./runtest \
--host 127.0.0.1 --port 6379 \
--cluster-mode \
--tags -slow
- name: Archive redis log
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: test-external-cluster-log
path: external-redis.log
test-external-nodebug:
runs-on: ubuntu-latest
if: github.event_name != 'schedule' || github.repository == 'redis/redis'
timeout-minutes: 14400
steps:
- uses: actions/checkout@v3
- name: Build
run: make REDIS_CFLAGS=-Werror
- name: Start redis-server
run: |
./src/redis-server --daemonize yes --save "" --logfile external-redis.log
- name: Run external test
run: |
./runtest \
--host 127.0.0.1 --port 6379 \
--tags "-slow -needs:debug"
- name: Archive redis log
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: test-external-redis-log
path: external-redis.log
.github/workflows/spell-check.yml
+32
View File
@@ -0,0 +1,32 @@
# A CI action that using codespell to check spell.
# .github/.codespellrc is a config file.
# .github/wordlist.txt is a list of words that will ignore word checks.
# More details please check the following link:
# https://github.com/codespell-project/codespell
name: Spellcheck
on:
push:
pull_request:
jobs:
build:
name: Spellcheck
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: pip cache
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: ${{ runner.os }}-pip-
- name: Install prerequisites
run: sudo pip install -r ./.codespell/requirements.txt
- name: Spell check
run: codespell --config=./.codespell/.codespellrc
.gitignore
+3 -1
View File
@@ -16,7 +16,8 @@ doc-tools
release
misc/*
src/release.h
appendonly.aof
appendonly.aof*
appendonlydir
SHORT_TERM_TODO
release.h
src/transfer.sh
@@ -27,6 +28,7 @@ src/nodes.conf
deps/lua/src/lua
deps/lua/src/luac
deps/lua/src/liblua.a
deps/hdr_histogram/libhdrhistogram.a
tests/tls/*
.make-*
.prerequisites
00-RELEASENOTES
+897 -11
View File
@@ -1,16 +1,902 @@
Hello! This file is just a placeholder, since this is the "unstable" branch
of Redis, the place where all the development happens.
Redis 7.0 release notes
=======================
There is no release notes for this branch, it gets forked into another branch
every time there is a partial feature freeze in order to eventually create
a new stable release.
--------------------------------------------------------------------------------
Upgrade urgency levels:
Usually "unstable" is stable enough for you to use it in development environments
however you should never use it in production environments. It is possible
to download the latest stable release here:
LOW: No need to upgrade unless there are new features you want to use.
MODERATE: Program an upgrade of the server, but it's not urgent.
HIGH: There is a critical bug that may affect a subset of users. Upgrade!
CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
SECURITY: There are security fixes in the release.
--------------------------------------------------------------------------------
http://download.redis.io/releases/redis-stable.tar.gz
================================================================================
Redis 7.0.6 Released Mon Dec 12 12:00:00 IST 2022
================================================================================
More information is available at https://redis.io
Upgrade urgency: MODERATE, Contains fixes for a few non-critical or unlikely bugs,
and some dramatic optimizations to Geo, EVAL, and Sorted sets commands.
Potentially Breaking Bug Fixes for new Redis 7.0 features
=======================================================
* RM_ResetDataset module API should not clear the functions (#11268)
* RM_Call module API used with the "C" flag to run scripts, would now cause
the commands in the script to check ACL with the designated user (#10966)
Performance and resource utilization improvements
=================================================
* Geo commands speedups (#11535, #11522, #11552, #11579)
* Fix EVAL command performance regression from Redis 7.0 (#11521, #11541)
* Reduce EXPIRE commands performance regression from Redis 7.0 (#11602)
* Optimize commands returning double values, mainly affecting zset commands (#11093)
* Optimize Lua parsing of some command responses (#11556)
* Optimize client memory usage tracking operation while client eviction is disabled (#11348)
Platform / toolchain support related improvements
=================================================
* Fix compilation on Solaris (#11327)
Module API changes
==================
* RM_SetContextUser, RM_SetModuleUserACLString, RM_GetModuleUserACLString (#10966)
* Fix crash in CLIENT_CHANGE event, when the selected database is not 0 (#11500)
Changes in CLI tools
====================
* redis-benchmark avoid aborting on NOPERM from CONFIG GET (#11096)
Bug Fixes
=========
* Avoid hang of diskless replication fork child when parent crashes (#11463)
* Fix crash with module API of list iterator and RM_ListDelete (#11383)
* Fix TLS error handling to avoid connection drops on timeouts (#11563)
* Fix runtime changes to cluster-announce-*-port to take effect on the local node too (#10745)
* Fix sentinel function that compares hostnames if failed resolve (#11419)
* Fix MIGRATE with AUTH set to "keys" is getting wrong key names leading to MOVED or ACL errors (#11253)
Fixes for issues in previous releases of Redis 7.0
--------------------------------------------------
* Fix command line startup --sentinel problem (#11591)
* Fis missing FCALL commands in monitor (#11510)
* Fix CLUSTER SHARDS showing empty hostname (#11297)
* Replica that asks for rdb-only could have missed the EOF and hang (#11296)
================================================================================
Redis 7.0.5 Released Wed Sep 21 20:00:00 IST 2022
================================================================================
Upgrade urgency: SECURITY, contains fixes to security issues.
Security Fixes:
* (CVE-2022-35951) Executing a XAUTOCLAIM command on a stream key in a specific
state, with a specially crafted COUNT argument, may cause an integer overflow,
a subsequent heap overflow, and potentially lead to remote code execution.
The problem affects Redis versions 7.0.0 or newer
[reported by Xion (SeungHyun Lee) of KAIST GoN].
Module API changes
==================
* Fix RM_Call execution of scripts when used with M/W/S flags to properly
handle script flags (#11159)
* Fix RM_SetAbsExpire and RM_GetAbsExpire API registration (#11025, #8564)
Bug Fixes
=========
* Fix a hang when eviction is combined with lazy-free and maxmemory-eviction-tenacity is set to 100 (#11237)
* Fix a crash when a replica may attempt to set itself as its master as a result of a manual failover (#11263)
* Fix a bug where a cluster-enabled replica node may permanently set its master's hostname to '?' (#10696)
* Fix a crash when a Lua script returns a meta-table (#11032)
Fixes for issues in previous releases of Redis 7.0
--------------------------------------------------
* Fix redis-cli to do DNS lookup before sending CLUSTER MEET (#11151)
* Fix crash when a key is lazy expired during cluster key migration (#11176)
* Fix AOF rewrite to fsync the old AOF file when a new one is created (#11004)
* Fix some crashes involving a list containing entries larger than 1GB (#11242)
* Correctly handle scripts with a non-read-only shebang on a cluster replica (#11223)
* Fix memory leak when unloading a module (#11147)
* Fix bug with scripts ignoring client tracking NOLOOP (#11052)
* Fix client-side tracking breaking protocol when FLUSHDB / FLUSHALL / SWAPDB is used inside MULTI-EXEC (#11038)
* Fix ACL: BITFIELD with GET and also SET / INCRBY can be executed with read-only key permission (#11086)
* Fix missing sections for INFO ALL when also requesting a module info section (#11291)
================================================================================
Redis 7.0.4 Released Monday Jul 18 12:00:00 IST 2022
================================================================================
Upgrade urgency: SECURITY, contains fixes to security issues.
Security Fixes:
* (CVE-2022-31144) A specially crafted XAUTOCLAIM command on a stream
key in a specific state may result with heap overflow, and potentially
remote code execution. The problem affects Redis versions 7.0.0 or newer.
================================================================================
Redis 7.0.3 Released Monday Jul 11 12:00:00 IST 2022
================================================================================
Upgrade urgency: MODERATE, specifically if you're using a previous release of
Redis 7.0, contains fixes for bugs in previous 7.0 releases.
Performance and resource utilization improvements
=================================================
* Optimize zset conversion on large ZRANGESTORE (#10789)
* Optimize the performance of sending PING on large clusters (#10624)
* Allow for faster restart of Redis in cluster mode (#10912)
INFO fields and introspection changes
=====================================
* Add missing sharded pubsub keychannel count to CLIENT LIST (#10895)
* Add missing pubsubshard_channels field in INFO STATS (#10929)
Module API changes
==================
* Add RM_StringToULongLong and RM_CreateStringFromULongLong (#10889)
* Add RM_SetClientNameById and RM_GetClientNameById (#10839)
Changes in CLI tools
====================
* Add missing cluster-port support to redis-cli --cluster (#10344)
Other General Improvements
==========================
* Account sharded pubsub channels memory consumption (#10925)
* Allow ECHO in loading and stale modes (#10853)
* Cluster: Throw -TRYAGAIN instead of -ASK on migrating nodes for multi-key
commands when the node only has some of the keys (#9526)
Bug Fixes
=========
* TLS: Notify clients on connection shutdown (#10931)
* Fsync directory while persisting AOF manifest, RDB file, and config file (#10737)
* Script that made modification will not break with unexpected NOREPLICAS error (#10855)
* Cluster: Fix a bug where nodes may not acknowledge a CLUSTER FAILOVER TAKEOVER
after a replica reboots (#10798)
* Cluster: Fix crash during handshake and cluster shards call (#10942)
Fixes for issues in previous releases of Redis 7.0
--------------------------------------------------
* TLS: Fix issues with large replies (#10909)
* Correctly report the startup warning for vm.overcommit_memory (#10841)
* redis-server command line allow passing config name and value in the same argument (#10866)
* Support --save command line argument with no value for backwards compatibility (#10866)
* Fix CLUSTER RESET command regression requiring an argument (#10898)
================================================================================
Redis 7.0.2 Released Sunday Jun 12 12:00:00 IST 2022
================================================================================
Upgrade urgency: MODERATE, specifically if you're using a previous release of
Redis 7.0, contains fixes for bugs in previous 7.0 releases.
Bug Fixes
=========
* Fixed SET and BITFIELD commands being wrongly marked movablekeys (#10837)
Regression in 7.0 possibly resulting in excessive roundtrip from cluster clients.
* Fix crash when /proc/sys/vm/overcommit_memory is inaccessible (#10848)
Regression in 7.0.1 resulting in crash on startup on some configurations.
================================================================================
Redis 7.0.1 Released Wed Jun 8 12:00:00 IST 2022
================================================================================
Upgrade urgency: MODERATE, specifically if you're using a previous release of
Redis 7.0, contains some behavior changes for new 7.0 features and important
fixes for bugs in previous 7.0 releases.
Improvements
============
* Add warning for suspected slow system clocksource setting
Add --check-system command line option. (#10636)
* Allow read-only scripts (*_RO commands, and ones with `no-writes` flag)
during CLIENT PAUSE WRITE (#10744)
* Add `readonly` flag in COMMAND command for EVAL_RO, EVALSHA_RO and FCALL_RO (#10728)
* redis-server command line arguments now accept one string with spaces
for multi-arg configs (#10660)
Potentially Breaking Changes
============================
* Omitting a config option value in command line argument no longer works (#10660)
* Hide the `may_replicate` flag from the COMMAND command response (#10744)
Potentially Breaking Changes for new Redis 7.0 features
-------------------------------------------------------
* Protocol: Sharded pubsub publish emits `smessage` instead of `message` (#10792)
* CLUSTER SHARDS returns slots as RESP integers, not strings (#10683)
* Block PFCOUNT and PUBLISH in read-only scripts (*_RO commands, and no-writes) (#10744)
* Scripts that declare the `no-writes` flag are implicitly `allow-oom` too (#10699)
Changes in CLI tools
====================
* redis-cli --bigkeys, --memkeys, --hotkeys, --scan. Finish nicely after Ctrl+C (#10736)
Platform / toolchain support related improvements
=================================================
* Support tcp-keepalive config interval on MacOs (#10667)
* Support RSS metrics on Haiku OS (#10687)
INFO fields and introspection changes
=====================================
* Add isolated network metrics for replication. (#10062, #10810)
Module API changes
==================
* Add two more new checks to RM_Call script mode (#10786)
* Add new RM_Call flag to let Redis automatically refuse `deny-oom` commands (#10786)
* Add module API RM_MallocUsableSize (#10795)
* Add missing REDISMODULE_NOTIFY_NEW (#10688)
* Fix cursor type in RedisModuleScanCursor to handle more than 2^31 elements (#10698)
* Fix RM_Yield bugs and RM_Call("EVAL") OOM check bug (#10786)
* Fix bugs in enum configs with overlapping bit flags (#10661)
Bug Fixes
=========
* FLUSHALL correctly resets rdb_changes_since_last_save INFO field (#10691)
* FLUSHDB is now propagated to replicas / AOF, even if the db is empty (#10691)
* Replica fail and retry the PSYNC if the master is unresponsive (#10726)
* Fix ZRANGESTORE crash when zset_max_listpack_entries is 0 (#10767)
Fixes for issues in previous releases of Redis 7.0
--------------------------------------------------
* CONFIG REWRITE could cause a config change to be dropped for aliased configs (#10811)
* CONFIG REWRITE would omit rename-command and include lines (#10761)
NOTE: Affected users who used Redis 7.0.0 to rewrite their configuration file
should review and fix the file.
* Fix broken protocol after MISCONF (persistence) error (#10786)
* Fix --save command line regression (#10690)
* Fix possible regression around TLS config changes. re-load files even if the
file name didn't change. (#10713)
* Re-add SENTINEL SLAVES command, missing in redis 7.0 (#10723)
* BZMPOP gets unblocked by non-key args and returns them (#10764)
* Fix possible memory leak in XADD and XTRIM (#10753)
================================================================================
Redis 7.0.0 GA Released Wed Apr 27 12:00:00 IST 2022
================================================================================
Upgrade urgency: SECURITY, contains fixes to security issues.
Security Fixes:
* (CVE-2022-24736) An attacker attempting to load a specially crafted Lua script
can cause NULL pointer dereference which will result with a crash of the
redis-server process. This issue affects all versions of Redis.
[reported by Aviv Yahav].
* (CVE-2022-24735) By exploiting weaknesses in the Lua script execution
environment, an attacker with access to Redis can inject Lua code that will
execute with the (potentially higher) privileges of another Redis user.
[reported by Aviv Yahav].
New Features
============
* Keyspace event for new keys (#10512)
Command replies that have been extended
---------------------------------------
* COMMAND DOCS shows deprecated_since field in command args (#10545)
* COMMAND DOCS shows module name where applicable (#10544)
Potentially Breaking Changes
============================
* Replicas panic when they fail writing persistence (#10504)
* Prevent cross slot operations in functions and scripts with shebang (#10615)
* Rephrased some error responses about invalid commands or args (#10612)
* Lua scripts do not have access to the print() function (#10651)
Performance and resource utilization improvements
=================================================
* Speed optimization in streams (#10574)
* Speed optimization in command execution pipeline (#10502)
* Speed optimization in listpack encoded sorted (#10486)
* Speed optimization in latency tracking at INFO (relevant for 7.0 RCs) (#10606)
* Speed optimization when there are many replicas (relevant for 7.0 RCs) (#10588)
New configuration options
=========================
* Allow ignoring disk persistence errors on replicas (#10504)
* Allow abort with panic when replica fails to execute a command sent by the master (#10504)
* Allow configuring shutdown flags of SIGTERM and SIGINT (#10594)
* Allow attaching an operating system-specific identifier to Redis sockets (#10349)
Module API changes
==================
* Add argument specifying ACL reason for module log entry (#10559)
Breaking API compatibility with 7.0 RCs
* Add the deprecated_since field in command args of COMMAND DOCS (#10545)
Breaking API/ABI compatibility with 7.0 RCs
* Add module API flag for using enum configs as bit flags (#10643)
* Add RM_PublishMessageShard (#10543)
* Add RM_MallocSizeString, RM_MallocSizeDict (#10542)
* Add RM_TryAlloc (#10541)
Bug Fixes
=========
* Replica report disk persistence errors in PING (#10603)
* Fixes around rejecting commands on replicas and AOF when they must be respected (#10603)
* Durability fixes for appendfsync=always policy (#9678)
Fixes for issues in previous release candidates of Redis 7.0
------------------------------------------------------------
* Fix possible crash on CONFIG REWRITE (#10598)
* Fix regression not aborting transaction on errors (#10612)
* Fix auto-aof-rewrite-percentage based AOFRW trigger after restart (#10550)
* Fix bugs when AOF enabled after startup, in case of failure before the first rewrite completes (#10616)
* Fix RM_Yield module API bug processing future commands of the current client (#10573)
================================================================================
Redis 7.0 RC3 Released Tue Apr 5 12:00:00 IST 2022
================================================================================
Upgrade urgency LOW: This is another Release Candidate of Redis 7.0.
New Features
============
New administrative and introspection commands and command arguments
-------------------------------------------------------------------
* CLUSTER SHARDS command deprecates CLUSTER SLOTS (#10293)
Potentially Breaking Changes
============================
* CONFIG GET response returned in a non-deterministic order.
It's possible that a client was relying on configs order (#10323)
* SORT / SORT_RO commands reject keys access patterns in GET and BY if ACL
doesn't grant the command full keyspace access (#10340)
* FUNCTION LOAD command introduced in 7.0-RC1 was stripped of the ENGINE, and
NAME arguments which are now part of the script itself. The DESCRIPTION
argument was completely removed (#10500)
* Set disable-thp config to be immutable (#10409)
Performance and resource utilization improvements
=================================================
* Optimize performance and memory usage on replicas (#10413)
* A faster and more robust code of zslRandomLevel using RAND_MAX (#5539)
Changes in CLI tools
====================
* redis-cli: Use exit code 1 on error (#10468)
* redis-cli: Do DNS lookup before sending CLUSTER MEET (#10436)
* redis-benchmark: Fix --cluster with IPv6. (#10393)
* redis-cli: Better --json Unicode support and --quoted-json (#10286)
INFO fields and introspection changes
=====================================
* MEMORY STATS: Show cluster.links memory usage (#10302)
Module API changes
==================
* APIs for exposing module configs to config file and CONFIG command (#10285)
* Add an event notifying about configuration changes (#10311)
* Add API for redacting command arguments from SLOWLOG and MONITOR (#10425)
* RM_Call: new flags for script mode compatibility, no writes, and error replies (#10372)
Bug Fixes
=========
* Sentinel: Fix no reconnect after auth-pass is changed (#10400)
* Cluster: Fix race condition: Turn into replica on SETSLOT (#10489, #10381)
* XREADGROUP: Unblock client when the stream key is deleted (#10306)
Fixes for issue in previous release candidates of Redis 7.0
-----------------------------------------------------------
* ACL DRYRUN does not validate the verified command args. (#10405)
* ACL DRYRUN returns the tested common permission error (#10359)
* Incorrect parsing of hostname information from nodes.conf (#10435)
* BITSET and BITFIELD SET should propagate even if just length changed (#10459)
* SHUTDOWN, Fix a possible crash when the shutdown was aborted (#10440)
* Script should not allow may-replicate commands when client pause write (#10364)
* Optimization tracking memory usage from i/o threads. (#10401)
* Initialize help when using redis-cli help or redis-cli ? (#10382)
* Dismiss COW of client output buffer now that it's dynamic (#10371)
* Fix memory corruption when EVAL fails before being processed (#10519)
================================================================================
Redis 7.0 RC2 Released Mon Feb 28 12:00:00 IST 2022
================================================================================
Upgrade urgency LOW: This is another Release Candidate of Redis 7.0.
New Features
============
* Add stream consumer group lag tracking and reporting (#9127)
* Add API for functions and eval Lua scripts to check ACL explicitly (#10220)
New user commands or command arguments
--------------------------------------
* COMMAND GETKEYSANDFLAGS sub-command (#10237)
* INFO command can take multiple section arguments (#6891)
* XGROUP CREATE and SETID: new ENTRIESREAD optional argument (#9127)
* XSETID new ENTRIESADDED and MAXDELETEDID optional arguments (#9127)
Command replies that have been extended
---------------------------------------
* XINFO reports consumer group lag and a few other fields (#9127)
* XAUTOCLAIM returns a new element with a list of deletes IDs (#10227)
Potentially Breaking Changes
============================
* X[AUTO]CLAIM skips deleted entries instead of replying with Nil, and deletes
them from the pending entry list (#10227)
* Fix messed up error codes returned from EVAL scripts (#10218, #10329)
* COMMAND INFO, Renames key-spec "CHANNEL" flag to be "NOT_KEY" (#10299)
Performance and resource utilization improvements
=================================================
* Reduce system calls and small packets for client replies (#9934)
* Reduce memory usage of stale clients (#9822)
* Fix regression in Z[REV]RANGE commands (by-rank) introduced in Redis 6.2 (#10337)
Changes in CLI tools
===================
* Adapt redis-check-aof tool for Multi Part AOF (#10061)
* Enable redis-benchmark to use RESP3 protocol mode (#10335)
Platform / toolchain support related improvements
=================================================
* Fix OpenSSL 3.0.x related issues (#10291)
INFO fields and introspection changes
=====================================
* COMMAND INFO key-specs has new variable_flags flag (#10237, #10148)
* INFO stats: add aof_rewrites and rdb_snapshots counters (#10178)
* INFO stats: add reply_buffer_shrinks and reply_buffer_expends (#9822)
* INFO modules: add no-implicit-signal-modified module option (#10284)
Module API changes
==================
* Add RM_SetCommandInfo API to set command metadata for the new COMMAND
introspection features and ACL key permissions (#10108)
* Add RM_KeyAtPosWithFlags and RM_GetCommandKeysWithFlags APIs (#10237)
* Add getchannels-api command flag and RM_IsChannelsPositionRequest,
RM_ChannelAtPosWithFlags APIs (#10299)
* Change RM_ACLCheckChannelPermissions and RM_ACLCheckKeyPermissions APIs
(released in RC1) to take different flags (#10299)
* Fix RM_SetModuleOptions flag collision. Bug in 7.0 RC1 header file, modules
that used OPTIONS_HANDLE_REPL_ASYNC_LOAD will mess up key invalidations (#10284)
Bug Fixes
=========
* Modules: Fix thread safety violation when a module thread adds an error reply,
broken in 6.2 (#10278)
* Lua: Fix Eval scripts active defrag, broken 7.0 in RC1 (#10271)
* Fix geo search bounding box check causing missing results (#10018)
* Lua: Add checks for min-slave-* configs when evaluating Lua scripts and
Functions (#10160)
* Modules: Prevent crashes and memory leaks when MODULE UNLOAD is used on module
with a pending timer (#10187)
* Fix error stats and failed command stats for blocked clients (#10309)
* Lua/Modules: Fix missing and duplicate error stats for scripts and modules (#10329, #10278)
* Check target node is a primary during cluster setslot (#10277)
* Fix key deletion not to invalidate WATCH when used on a logically expired key (#10256)
* Sentinel: return an error if configuration save fails (#10151)
* Sentinel: fix a free-after-use issue re-registering Sentinels (#10333)
================================================================================
Redis 7.0 RC1 Released Mon Jan 31 12:00:00 IST 2022
================================================================================
Upgrade urgency LOW: This is the first Release Candidate of Redis 7.0.
Redis Release Candidate (RC) versions are early versions that are made available
for early adopters in the community to test them. We do not consider
them suitable for production environments.
Introduction to the Redis 7.0 release
=====================================
Redis 7.0 includes several new user-facing features, significant performance
optimizations, and many other improvements. It also includes changes that
potentially break backwards compatibility with older versions. We urge users to
review the release notes carefully before upgrading.
In particular, users should be aware of the following changes:
1. Redis 7 stores AOF as multiple files in a folder; see Multi-Part AOF below.
2. Redis 7 uses a new version 10 format for RDB files, which is incompatible
with older versions.
3. Redis 7 converts ziplist encoded keys to listpacks on the fly when loading
an older RDB format. Conversion applies to loading a file from disk or
replicating from a Redis master and will slightly increase loading time.
4. See sections about breaking changes mentioned below.
Here is a comprehensive list of changes in this release compared to 6.2.6.
Each one includes the PR number that added it so that you can get more details
at https://github.com/redis/redis/pull/<number>
New Features
============
* Redis Functions: A new way to extend Redis with server-side scripts (#8693)
see https://redis.io/topics/functions-intro
* ACL: Fine-grained key-based permissions and allow users to support multiple
sets of command rules with selectors (#9974)
see https://redis.io/topics/acl#key-permissions and https://redis.io/topics/acl#selectors.
* Cluster: Sharded (node-specific) Pub/Sub support (#8621)
see https://redis.io/topics/pubsub#sharded-pubsub
* First-class handling of sub-commands in most contexts (affecting ACL
categories, INFO commandstats, etc.) (#9504, #10147)
* Command metadata and documentation (#10104)
see https://redis.io/commands/command-docs, https://redis.io/topics/command-tips
* Command key-specs. A better way for clients to locate key arguments and their
read/write purpose (#8324, #10122, #10167)
see https://redis.io/topics/key-specs
* Multi-Part AOF mechanism to avoid AOF rewrite overheads (#9788)
* Cluster: Support for hostnames, instead of IP addresses only (#9530)
* Improved management of memory consumed by network buffers, and an option to
drop clients when total memory exceeds a limit (#8687)
* Cluster: A mechanism for disconnecting cluster bus connections to prevent
uncontrolled buffer growth (#9774)
* AOF: Timestamp annotations and support for point-in-time recovery (#9326)
* Lua: support Function flags in EVAL scripts (#10126)
see https://redis.io/topics/eval-intro#eval-flags
* Lua: Support RESP3 reply for Verbatim and Big-Number types (#9202)
* Lua: Get Redis version via redis.REDIS_VERSION, redis.REDIS_VERSION_NUM (#10066)
New user commands or command arguments
--------------------------------------
* ZMPOP, BZMPOP commands (#9484)
* LMPOP, BLMPOP commands (#9373)
* SINTERCARD, ZINTERCARD commands (#8946, #9425)
* SPUBLISH, SSUBSCRIBE, SUNSUBSCRIBE, PUBSUB SHARDCHANNELS/SHARDNUMSUB (#8621)
* EXPIRETIME and PEXPIRETIME commands (#8474)
* EXPIRE command group supports NX/XX/GT/LT options (#2795)
* SET command supports combining NX and GET flags (#8906)
* BITPOS, BITCOUNT accepts BIT index (#9324)
* EVAL_RO, EVALSHA_RO command variants, to run on read-only replicas (#8820)
* SORT_RO command, to run on read-only replicas (#9299)
* SHUTDOWN arguments: NOW, FORCE, ABORT (#9872)
* FUNCTION *, FCALL, FCALL_RO - https://redis.io/commands/function-load
* CONFIG SET/GET can handle multiple configs atomically, in one call (#9748, #9914)
* QUIT promoted to be a proper command, HOST: and POST demoted (#9798)
* XADD supports auto sequence number via <ms>-* (#9217)
New administrative and introspection commands and command arguments
-------------------------------------------------------------------
* COMMAND DOCS (#9656, #10056, #10104)
* COMMAND LIST (#9504)
* COMMAND INFO accepts sub-commands as args, and no args too (#9504, #10056)
* LATENCY HISTOGRAM (#9462)
* CLUSTER LINKS (#9774)
* CLUSTER DELSLOTSRANGE and CLUSTER ADDSLOTSRANGE (#9445)
* CLIENT NO-EVICT (#8687)
* ACL DRYRUN (#9974)
* SLOWLOG GET supports passing in -1 to get all entries (#9018)
Command replies that have been extended
---------------------------------------
* COMMAND and COMMAND INFO extended with tips, key-specs and sub-commands
see https://redis.io/commands/command
* ACL CAT, COMMAND LIST list sub-commands (#10127)
* MODULE LIST reply includes path and args (#4848)
* OBJECT ENCODING returns listpack instead of ziplist (#8887, #9366)
* CLUSTER SLOTS hostname support (#9530)
* COMMAND command: Added the `blocking` and `module` flags (#10104, #9656)
Potentially Breaking Changes
============================
* Modifying the bind parameter to a non-default value will no longer implicitly
disable protected-mode (#9034)
* Remove EVAL script verbatim replication, propagation, and deterministic
execution logic (#9812)
This has been deprecated and off by default since Redis 6 and is no longer
supported.
* ACL: pub/sub channels are blocked by default (acl-pubsub-default=resetchannels) (#10181)
* SCRIPT LOAD and SCRIPT FLUSH are no longer propagated to replicas / AOF (#9812)
* ACL: Declarations of duplicate ACL users in startup files and command line
arguments will result in an error, whereas previously the last declaration
would overwrite the others. (#9330)
* Replication: TTLs are always replicated as absolute (not relative) millisecond
timestamps (#8474)
* Fixes in handling multi-key commands with expired keys on writable replicas (#9572)
* CONFIG SET maxmemory returns before starting eviction (#10019)
* AOF: The new Multi-Part mechanism stores data as a set of multiple files in a
designated folder (#9788)
* Remove STRALGO command, preserve LCS a standalone command which only works on
keys (#9799)
* Remove gopher protocol support (#9057)
* MODULE and DEBUG commands disabled (protected) by default, for better security (#9920)
* Snapshot-creating and other admin commands in MULTI/EXEC transactions are now
rejected (#10015)
* PING is now rejected with -MASTERDOWN when replica-serve-stale-data=no (#9757)
* ACL GETUSER reply now uses ACL syntax for `keys` and `channels` (#9974)
* COMMAND reply drops `random` and `sort-for-scripts` flags, which are now part
of command tips (#10104)
* LPOP/RPOP with count against non-existing list return null array (#10095)
* INFO commandstats now shows the stats per sub-command (#9504)
* ZPOPMIN/ZPOPMAX used to produce wrong replies when count is 0 with non-zset (#9711)
* LPOP/RPOP used to produce wrong replies when count is 0 (#9692)
* CONFIG GET bind now returns the current value in effect, even if the implicit
default is in use (#9034)
* CONFIG REWRITE now rewrites the list of modules to load (#4848)
* Config: repl-diskless-sync is now set to yes by default (#10092)
* When shutting down, Redis can optionally wait for replicas to catch up on the
replication link (#9872)
* Most CONFIG SET, REWRITE, RESETSTAT commands are now allowed during loading (#9878)
* READONLY and READWRITE commands are now allowed when loading and on stale
replicas (#7425)
* Fix ACL category for SELECT, WAIT, ROLE, LASTSAVE, READONLY, READWRITE, ASKING (#9208)
* RESET is now allowed even when on unauthenticated connections (#9798)
* SCRIPT LOAD is now allowed on stale replicas (#10126)
Security improvements
=====================
* Sensitive configs and commands blocked (protected) by default (#9920)
* Improve bind and protected-mode config handling (#9034)
* Sentinel: avoid logging auth-pass value (#9652)
* redis-cli: sensitive commands bypass the history file (#8895)
Performance and resource utilization improvements
=================================================
* Significant memory saving and latency improvements in cluster mode (#9356)
* Significant memory savings in case of many hash or zset keys (#9228)
* Replication backlog and replicas use one global shared replication buffer (#9166)
* Significant reduction of copy-on-write memory overheads (#8974)
* Free unused capacity in the cluster send buffer (#9255)
* Memory efficiency, make full use of client struct memory for reply buffers (#8968)
* Replace ziplist with listpack in Hash, List, Zset (#8887, #9366, #9740)
* Add support for list type to store elements larger than 4GB (#9357)
* Reuse temporary client objects for blocked clients by module (#9940)
* Remove command argument count limit, dynamically grow argv buffer (#9528)
* Optimize list type operations to seek from the nearest end (#9454)
* Improvements in fsync to avoid large writes to disk (#9409)
* BITSET and BITFIELD SET only propagated when the value actually changed (#9403)
* Improve latency when a client is unblocked by module timer (#9593)
Other General Improvements
==========================
* Make partial sync possible after master reboot (#8015)
* Always create a base AOF file when redis starts from empty (#10102)
* Replica keep serving data during repl-diskless-load=swapdb for better
availability (#9323)
Changes in CLI tools
====================
* redis-cli --json, and -2 options (#9954)
* redis-cli --scan, add sleep interval option (#3751)
* redis-cli --replica optimization, skip RDB generation (#10044)
* redis-cli --functions-rdb, generate RDB with Functions only (#9968)
* redis-cli -X, take an arbitrary arg from stdin, extend --cluster call take -x (#9980)
* redis-benchmark -x takes an argument from stdin (#9130)
* redis-benchmark, Added URI support (#9314)
* redis-cli monitor and pubsub can be aborted with Ctrl+C, keeping the cli alive (#9347)
Platform / toolchain support related improvements
=================================================
* Upgrade jemalloc 5.2.1 (#9623)
* Fix RSS metrics on NetBSD and OpenBSD (#10116, #10149)
* Check somaxconn system settings on macOS, FreeBSD and OpenBSD (#9972)
* Better fsync on MacOS, improve power failure safety (#9545)
New configuration options
=========================
* CONFIG SET/GET can handle multiple configs in one call (#9748, #9914)
* Support glob pattern matching for config include files (#8980)
* appenddirname, folder where multi-part AOF files are stored (#9788)
* shutdown-timeout, default 10 seconds (#9872)
* maxmemory-clients, allows limiting the total memory usage by all clients (#8687)
* cluster-port, can control the bind port of cluster bus (#9389)
* bind-source-addr, configuration argument control IP of outgoing connections (#9142)
* busy-reply-threshold, alias for the old lua-time-limit (#9963)
* repl-diskless-sync-max-replicas, allows faster replication in some cases (#10092)
* latency-tracking, enabled by default, and latency-tracking-info-percentiles (#9462)
* cluster-announce-hostnameand cluster-preferred-endpoint-type (#9530)
* cluster-allow-pubsubshard-when-down (#8621)
* cluster-link-sendbuf-limit (#9774)
* list-max-listpack-*, hash-max-listpack-*, zset-max-listpack-* as aliases for
the old ziplist configs (#8887, #9366, #9740)
INFO fields and introspection changes
=====================================
* INFO: latencystats section (#9462)
* INFO: total_active_defrag_time and current_active_defrag_time (#9377)
* INFO: total_eviction_exceeded_time and current_eviction_exceeded_time (#9031)
* INFO: evicted_clients (#8687)
* INFO: mem_cluster_links, total_cluster_links_buffer_limit_exceeded (#9774)
* INFO: current_cow_peak (#8974)
* INFO: Remove aof_rewrite_buffer_length (#9788)
* MEMORY STATS: Report slot to keys map size in in cluster mode (#10017)
* INFO MEMORY: changes to separate memory usage of Functions and EVAL (#9780)
* INFO MEMORY: Add mem_total_replication_buffers, change meaning of
mem_clients_slaves (#9166)
* CLIENT LIST: tot-mem, multi-mem (#8687)
* CLIENT LIST, INFO: Show RESP version (#9508)
* SENTINEL INFO: tilt_mode_since (#9000)
* LATENCY: Track module-acquire-GIL latency (#9608)
Module API changes
==================
* Add API for replying with RESP3 types (#8521, #9639, #9632)
* Add API for parsing RESP3 replies from RM_Call (#9202)
* Add RM_Call '0' and '3' flags to control RESP version to be used (#9202)
* Add Support for validating ACL explicitly (#9309, #9974)
* Add missing list type functionality APIs (#8439)
* Add API for yielding to Redis events during long busy jobs (#9963)
* Add API for registering other file descriptors to the Redis event loop (#10001)
* Enhance mem_usage/free_effort/unlink/copy and IO callbacks to have key name
and DB index (#8999)
* Enhance mem_usage callback to get the requested sample size (#9612)
* RM_GetContextFlags: CTX_FLAGS_ASYNC_LOADING, CTX_FLAGS_RESP3 (#9323, #9202)
* Mark APIs as non-experimental (#9983)
* RM_CreateSubcommand (#9504)
* RM_KeyExists (#9600)
* RM_TrimStringAllocation (#9540)
* RM_LoadDataTypeFromStringEncver (#9537)
* RM_MonotonicMicroseconds (#10101)
* Add ReplAsyncLoad event and deprecate the ReplBackup event (#9323)
* Add RM_SetModuleOptions OPTIONS_HANDLE_REPL_ASYNC_LOAD flag (#9323)
Bug Fixes
=========
* Fix COMMAND GETKEYS on EVAL without keys (#9733)
* Improve MEMORY USAGE with allocator overheads (#9095)
* Unpause clients after manual failover ends instead of waiting for timed (#9676)
* Lua: fix crash on a script call with many arguments, a regression in v6.2.6 (#9809)
* Lua: Use all characters to calculate string hash to prevent hash collisions (#9449)
* Prevent LCS from allocating temp memory over proto-max-bulk-len (#9817)
* Tracking: Make invalidation messages always after command's reply (#9422)
* Cluster: Hide empty replicas from CLUSTER SLOTS responses (#9287)
* CLIENT KILL killed all clients when used with ID of 0 (#9853)
* Fix bugs around lists with list-compress-depth (#9849, #9779)
* Fix one in a blue moon LRU bug in RESTORE, RDB loading, and module API (#9279)
* Reset lazyfreed_objects info field with RESETSTAT, test for stream lazyfree (#8934)
* Fix RDB and list node compression for handling values larger than 4GB (#9776)
* Fix a crash when adding elements larger than 2GB to a Set or Hash (#9916)
* Diskless replication could not count as a change and skip next database SAVE (#9323)
* Fix excessive stream trimming due to an overflow (#10068)
* Safe and organized exit when receiving SIGTERM while loading (#10003)
* Improve EXPIRE TTL overflow detection (#9839)
* Add missed error counting for INFO errorstats (#9646)
* DECRBY LLONG_MIN caused negation overflow (#9577)
* Delay discarding cached master when full synchronization (#9398)
* Fix Stream keyspace notification and persistence triggers in consumer
creation and deletion (#9263)
* Fix rank overflow in zset with more than 2B entries (#9249)
* Avoid starting in check-aof / check-rdb / sentinel modes if only the folder
name contains that name (#9215, #9176)
* create the log file only after done parsing the entire config file (#6741)
* redis-cli: Fix SCAN sleep interval for --bigkeys, --memkeys, --hotkeys (#9624)
* redis-cli: Fix prompt to show the right DB num and transaction state after
RESET (#9096)
* Module API: fix possible propagation bugs in case a module calls CONFIG SET
maxmemory outside a command (#10019, #9890)
* Module API: carry through client RESP version to module blocked clients (#9634)
* Module API: release clients blocked on module commands in cluster resharding
and down state (#9483)
* Sentinel: Fix availability after master reboot (#9438)
* Sentinel: Fix memory leak with TLS (#9753)
* Sentinel: Fix possible failover due to duplicate zero-port (#9240)
* Sentinel: Fix issues with hostname support (#10146)
* Sentinel: Fix election failures on certain container environments (#10197)
Thanks to all the users and developers who made this release possible.
We'll follow up with more RC releases, until the code looks production ready
and we don't get reports of serious issues for a while.
A special thank you for the amount of work put into this release by:
- Guy Benoish
- Meir Shpilraien
- Oran Agra
- Chen Yang
- Zhu Binbin
- Yoav Steinberg
- sundb
- Madelyn Olson
- Yossi Gottlieb
- Viktor Söderqvist
- Wang Yuan
- Harkrishn Patro
- Nick Chun
- Ozan Tezcan
- Wen Hui
- Huang Zhw
- Nan Yan
- Filipe Oliveira
- Eduardo Semprebon
- Yaacov Hazan
- Itamar Haber
- Zhao Zhao
- Itay Perry
- Moti Cohen
- Ning Sun
- zhugezy
- Ran Shidlansik
- menwen
- Andy Pan
Happy hacking!
CONDUCT → CODE_OF_CONDUCT.md
+1 -1
View File
@@ -93,4 +93,4 @@ Community Impact Guidelines were inspired by Mozilla’s code of conduct
enforcement ladder.
For answers to common questions about this code of conduct, see the FAQ at
https://www.contributor-covenant.org/faq. Translations are available at
https://www.contributor-covenant.org/translations.
https://www.contributor-covenant.org/translations.
CONTRIBUTING → CONTRIBUTING.md
+6 -24
View File
@@ -18,31 +18,13 @@ all the support in the mailing list.
There is also an active community of Redis users at Stack Overflow:
http://stackoverflow.com/questions/tagged/redis
https://stackoverflow.com/questions/tagged/redis
# Reporting Security Bugs
Issues and pull requests for documentation belong on the redis-doc repo:
*If you are reporting a security bug*, please contact the core team privately
by emailing redis@redis.io. Your report will be acknowledged by a core team
member and once the report has been reviewed you will receive a more detailed
response including next steps.
https://github.com/redis/redis-doc
If you do not receive a reply you can escalate to the Redis Google Group,
linked above. Because this group is a public space please do not disclose the
issue in detail, only say that you are trying to reach the core team for a
security issue.
Redis follows a responsible disclosure process:
1. Reports are reviewed and analyzed privately
2. Patches are prepared for supported versions of Redis
3. Vendor lists are notified with an embargo date to reduce the public impact
4. We push a fix release and your bug can be posted publicly with credit in
release notes and the version history (and our thanks!)
Issues and pull requests for documentation belong on the redis-doc repo:
https://github.com/redis/redis-doc
If you are reporting a security bug or vulnerability, see SECURITY.md.
# How to provide a patch for a new feature
@@ -56,10 +38,10 @@ Here you'll see if there is consensus about your idea.
2. If in step 1 you get an acknowledgment from the project leaders, use the
following procedure to submit a patch:
a. Fork Redis on github ( http://help.github.com/fork-a-repo/ )
a. Fork Redis on github ( https://docs.github.com/en/github/getting-started-with-github/fork-a-repo )
b. Create a topic branch (git checkout -b my_branch)
c. Push to your branch (git push origin my_branch)
d. Initiate a pull request on github ( https://help.github.com/articles/creating-a-pull-request/ )
d. Initiate a pull request on github ( https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request )
e. Done :)
3. Keep in mind that we are very overloaded, so issues and PRs sometimes wait
README.md
+27 -11
View File
@@ -16,7 +16,7 @@ Another good example is to think of Redis as a more complex version of memcached
If you want to know more, this is a list of selected starting points:
* Introduction to Redis data types. https://redis.io/topics/data-types-intro
* Try Redis directly inside your browser. http://try.redis.io
* Try Redis directly inside your browser. https://try.redis.io
* The full list of Redis commands. https://redis.io/commands
* There is much more inside the official Redis documentation. https://redis.io/documentation
@@ -223,11 +223,12 @@ public discussion groups, you agree to release your code under the terms
of the BSD license that you can find in the [COPYING][1] file included in the Redis
source distribution.
Please see the [CONTRIBUTING][2] file in this source distribution for more
information, including details on our process for security bugs/vulnerabilities.
Please see the [CONTRIBUTING.md][2] file in this source distribution for more
information. For security bugs and vulnerabilities, please see [SECURITY.md][3].
[1]: https://github.com/redis/redis/blob/unstable/COPYING
[2]: https://github.com/redis/redis/blob/unstable/CONTRIBUTING
[2]: https://github.com/redis/redis/blob/unstable/CONTRIBUTING.md
[3]: https://github.com/redis/redis/blob/unstable/SECURITY.md
Redis internals
===
@@ -303,8 +304,8 @@ struct client {
redisDb *db;
int flags;
list *reply;
char buf[PROTO_REPLY_CHUNK_BYTES];
// ... many other fields ...
char buf[PROTO_REPLY_CHUNK_BYTES];
}
```
The client structure defines a *connected client*:
@@ -361,6 +362,12 @@ Inside server.c you can find code that handles other vital things of the Redis s
* `performEvictions()` is called when a new write command should be performed but Redis is out of memory according to the `maxmemory` directive.
* The global variable `redisCommandTable` defines all the Redis commands, specifying the name of the command, the function implementing the command, the number of arguments required, and other properties of each command.
commands.c
---
This file is auto generated by utils/generate-command-code.py, the content is based on the JSON files in the src/commands folder.
These are meant to be the single source of truth about the Redis commands, and all the metadata about them.
These JSON files are not meant to be used directly by anyone directly, instead that metadata can be obtained via the COMMAND command.
networking.c
---
@@ -379,8 +386,8 @@ aof.c and rdb.c
As you can guess from the names, these files implement the RDB and AOF
persistence for Redis. Redis uses a persistence model based on the `fork()`
system call in order to create a thread with the same (shared) memory
content of the main Redis thread. This secondary thread dumps the content
system call in order to create a process with the same (shared) memory
content of the main Redis process. This secondary process dumps the content
of the memory on disk. This is used by `rdb.c` to create the snapshots
on disk and by `aof.c` in order to perform the AOF rewrite when the
append only file gets too big.
@@ -442,18 +449,27 @@ This file also implements both the `SYNC` and `PSYNC` commands that are
used in order to perform the first synchronization between masters and
replicas, or to continue the replication after a disconnection.
Script
---
The script unit is compose of 3 units
* `script.c` - integration of scripts with Redis (commands execution, set replication/resp, ..)
* `script_lua.c` - responsible to execute Lua code, uses script.c to interact with Redis from within the Lua code.
* `function_lua.c` - contains the Lua engine implementation, uses script_lua.c to execute the Lua code.
* `functions.c` - Contains Redis Functions implementation (FUNCTION command), uses functions_lua.c if the function it wants to invoke needs the Lua engine.
* `eval.c` - Contains the `eval` implementation using `script_lua.c` to invoke the Lua code.
Other C files
---
* `t_hash.c`, `t_list.c`, `t_set.c`, `t_string.c`, `t_zset.c` and `t_stream.c` contains the implementation of the Redis data types. They implement both an API to access a given data type, and the client command implementations for these data types.
* `ae.c` implements the Redis event loop, it's a self contained library which is simple to read and understand.
* `sds.c` is the Redis string library, check http://github.com/antirez/sds for more information.
* `sds.c` is the Redis string library, check https://github.com/antirez/sds for more information.
* `anet.c` is a library to use POSIX networking in a simpler way compared to the raw interface exposed by the kernel.
* `dict.c` is an implementation of a non-blocking hash table which rehashes incrementally.
* `scripting.c` implements Lua scripting. It is completely self-contained and isolated from the rest of the Redis implementation and is simple enough to understand if you are familiar with the Lua API.
* `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][3].
* `cluster.c` implements the Redis Cluster. Probably a good read only after being very familiar with the rest of the Redis code base. If you want to read `cluster.c` make sure to read the [Redis Cluster specification][4].
[3]: https://redis.io/topics/cluster-spec
[4]: https://redis.io/topics/cluster-spec
Anatomy of a Redis command
---
SECURITY.md
+43
View File
@@ -0,0 +1,43 @@
# Security Policy
## Supported Versions
Redis is generally backwards compatible with very few exceptions, so we
recommend users to always use the latest version to experience stability,
performance and security.
We generally backport security issues to a single previous major version,
unless this is not possible or feasible with a reasonable effort.
| Version | Supported |
| ------- | ------------------ |
| 7.0.x | :white_check_mark: |
| 6.2.x | :white_check_mark: |
| 6.0.x | :white_check_mark: |
| < 6.0 | :x: |
## Reporting a Vulnerability
If you believe you’ve discovered a serious vulnerability, please contact the
Redis core team at redis@redis.io. We will evaluate your report and if
necessary issue a fix and an advisory. If the issue was previously undisclosed,
we’ll also mention your name in the credits.
## Responsible Disclosure
In some cases, we may apply a responsible disclosure process to reported or
otherwise discovered vulnerabilities. We will usually do that for a critical
vulnerability, and only if we have a good reason to believe information about
it is not yet public.
This process involves providing an early notification about the vulnerability,
its impact and mitigations to a short list of vendors under a time-limited
embargo on public disclosure.
Vendors on the list are individuals or organizations that maintain Redis
distributions or provide Redis as a service, who have third party users who
will benefit from the vendor’s ability to prepare for a new version or deploy a
fix early.
If you believe you should be on the list, please contact us and we will
consider your request based on the above criteria.
deps/Makefile
Vendored
+13 -2
View File
@@ -2,6 +2,8 @@
uname_S:= $(shell sh -c 'uname -s 2>/dev/null || echo not')
LUA_DEBUG?=no
CCCOLOR="\033[34m"
LINKCOLOR="\033[34;1m"
SRCCOLOR="\033[33m"
@@ -69,8 +71,13 @@ ifeq ($(uname_S),SunOS)
LUA_CFLAGS= -D__C99FEATURES__=1
endif
LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS)
LUA_CFLAGS+= -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' -DLUA_USE_MKSTEMP $(CFLAGS)
LUA_LDFLAGS+= $(LDFLAGS)
ifeq ($(LUA_DEBUG),yes)
LUA_CFLAGS+= -O0 -g -DLUA_USE_APICHECK
else
LUA_CFLAGS+= -O2
endif
# lua's Makefile defines AR="ar rcu", which is unusual, and makes it more
# challenging to cross-compile lua (and redis). These defines make it easier
# to fit redis into cross-compilation environments, which typically set AR.
@@ -86,9 +93,13 @@ lua: .make-prerequisites
JEMALLOC_CFLAGS= -std=gnu99 -Wall -pipe -g3 -O3 -funroll-loops $(CFLAGS)
JEMALLOC_LDFLAGS= $(LDFLAGS)
ifneq ($(DEB_HOST_GNU_TYPE),)
JEMALLOC_CONFIGURE_OPTS += --host=$(DEB_HOST_GNU_TYPE)
endif
jemalloc: .make-prerequisites
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
cd jemalloc && ./configure --with-version=5.2.1-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" $(JEMALLOC_CONFIGURE_OPTS)
cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a
.PHONY: jemalloc
deps/README.md
Vendored
+25
View File
@@ -5,6 +5,7 @@ should be provided by the operating system.
* **hiredis** is the official C client library for Redis. It is used by redis-cli, redis-benchmark and Redis Sentinel. It is part of the Redis official ecosystem but is developed externally from the Redis repository, so we just upgrade it as needed.
* **linenoise** is a readline replacement. It is developed by the same authors of Redis but is managed as a separated project and updated as needed.
* **lua** is Lua 5.1 with minor changes for security and additional libraries.
* **hdr_histogram** Used for per-command latency tracking histograms.
How to upgrade the above dependencies
===
@@ -41,6 +42,22 @@ the following additional steps:
changed, otherwise you could just copy the old implementation if you are
upgrading just to a similar version of Jemalloc.
#### Updating/upgrading jemalloc
The jemalloc directory is pulled as a subtree from the upstream jemalloc github repo. To update it you should run from the project root:
1. `git subtree pull --prefix deps/jemalloc https://github.com/jemalloc/jemalloc.git <version-tag> --squash`<br>
This should hopefully merge the local changes into the new version.
2. In case any conflicts arise (due to our changes) you'll need to resolve them and commit.
3. Reconfigure jemalloc:<br>
```sh
rm deps/jemalloc/VERSION deps/jemalloc/configure
cd deps/jemalloc
./autogen.sh --with-version=<version-tag>-0-g0
```
4. Update jemalloc's version in `deps/Makefile`: search for "`--with-version=<old-version-tag>-0-g0`" and update it accordingly.
5. Commit the changes (VERSION,configure,Makefile).
Hiredis
---
@@ -79,4 +96,12 @@ and our version:
2. We have the implementation source code, and directly link to the following external libraries: `lua_cjson.o`, `lua_struct.o`, `lua_cmsgpack.o` and `lua_bit.o`.
3. There is a security fix in `ldo.c`, line 498: The check for `LUA_SIGNATURE[0]` is removed in order to avoid direct bytecode execution.
Hdr_Histogram
---
Updated source can be found here: https://github.com/HdrHistogram/HdrHistogram_c
We use a customized version based on master branch commit e4448cf6d1cd08fff519812d3b1e58bd5a94ac42.
1. Compare all changes under /hdr_histogram directory to upstream master commit e4448cf6d1cd08fff519812d3b1e58bd5a94ac42
2. Copy updated files from newer version onto files in /hdr_histogram.
3. Apply the changes from 1 above to the updated files.
deps/hdr_histogram/Makefile
Vendored
+10 -3
View File
@@ -1,20 +1,27 @@
STD=
STD= -std=c99
WARN= -Wall
OPT= -Os
R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS)
R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) -DHDR_MALLOC_INCLUDE=\"hdr_redis_malloc.h\"
R_LDFLAGS= $(LDFLAGS)
DEBUG= -g
R_CC=$(CC) $(R_CFLAGS)
R_LD=$(CC) $(R_LDFLAGS)
hdr_histogram.o: hdr_histogram.h hdr_histogram.c
AR= ar
ARFLAGS= rcs
libhdrhistogram.a: hdr_histogram.o
$(AR) $(ARFLAGS) $@ $+
hdr_histogram.o: hdr_histogram.h hdr_histogram.c
.c.o:
$(R_CC) -c $<
clean:
rm -f *.o
rm -f *.a
deps/hdr_histogram/README.md
Vendored
+73 -5
View File
@@ -1,10 +1,78 @@
HdrHistogram_c v0.11.0
HdrHistogram_c: 'C' port of High Dynamic Range (HDR) Histogram
HdrHistogram
----------------------------------------------
This port contains a subset of the 'C' version of High Dynamic Range (HDR) Histogram available at [github.com/HdrHistogram/HdrHistogram_c](https://github.com/HdrHistogram/HdrHistogram_c).
[![Gitter chat](https://badges.gitter.im/HdrHistogram/HdrHistogram.png)](https://gitter.im/HdrHistogram/HdrHistogram)
This port contains a subset of the functionality supported by the Java
implementation. The current supported features are:
The code present on `hdr_histogram.c`, `hdr_histogram.h`, and `hdr_atomic.c` was Written by Gil Tene, Michael Barker,
and Matt Warren, and released to the public domain, as explained at
http://creativecommons.org/publicdomain/zero/1.0/.
* Standard histogram with 64 bit counts (32/16 bit counts not supported)
* All iterator types (all values, recorded, percentiles, linear, logarithmic)
* Histogram serialisation (encoding version 1.2, decoding 1.0-1.2)
* Reader/writer phaser and interval recorder
Features not supported, but planned
* Auto-resizing of histograms
Features unlikely to be implemented
* Double histograms
* Atomic/Concurrent histograms
* 16/32 bit histograms
# Simple Tutorial
## Recording values
```C
#include <hdr_histogram.h>
struct hdr_histogram* histogram;
// Initialise the histogram
hdr_init(
1, // Minimum value
INT64_C(3600000000), // Maximum value
3, // Number of significant figures
&histogram) // Pointer to initialise
// Record value
hdr_record_value(
histogram, // Histogram to record to
value) // Value to record
// Record value n times
hdr_record_values(
histogram, // Histogram to record to
value, // Value to record
10) // Record value 10 times
// Record value with correction for co-ordinated omission.
hdr_record_corrected_value(
histogram, // Histogram to record to
value, // Value to record
1000) // Record with expected interval of 1000.
// Print out the values of the histogram
hdr_percentiles_print(
histogram,
stdout, // File to write to
5, // Granularity of printed values
1.0, // Multiplier for results
CLASSIC); // Format CLASSIC/CSV supported.
```
## More examples
For more detailed examples of recording and logging results look at the
[hdr_decoder](examples/hdr_decoder.c)
and [hiccup](examples/hiccup.c)
examples. You can run hiccup and decoder
and pipe the results of one into the other.
```
$ ./examples/hiccup | ./examples/hdr_decoder
```
deps/hdr_histogram/hdr_histogram.c
Vendored
+107 -33
View File
@@ -14,8 +14,15 @@
#include <inttypes.h>
#include "hdr_histogram.h"
#include "hdr_tests.h"
#include "hdr_atomic.h"
#ifndef HDR_MALLOC_INCLUDE
#define HDR_MALLOC_INCLUDE "hdr_malloc.h"
#endif
#include HDR_MALLOC_INCLUDE
/* ###### ####### ## ## ## ## ######## ###### */
/* ## ## ## ## ## ## ### ## ## ## ## */
/* ## ## ## ## ## #### ## ## ## */
@@ -215,6 +222,15 @@ int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_
return INT64_C(1) << (h->unit_magnitude + adjusted_bucket);
}
static int64_t size_of_equivalent_value_range_given_bucket_indices(
const struct hdr_histogram *h,
int32_t bucket_index,
int32_t sub_bucket_index)
{
const int32_t adjusted_bucket = (sub_bucket_index >= h->sub_bucket_count) ? (bucket_index + 1) : bucket_index;
return INT64_C(1) << (h->unit_magnitude + adjusted_bucket);
}
static int64_t lowest_equivalent_value(const struct hdr_histogram* h, int64_t value)
{
int32_t bucket_index = get_bucket_index(h, value);
@@ -222,6 +238,14 @@ static int64_t lowest_equivalent_value(const struct hdr_histogram* h, int64_t va
return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude);
}
static int64_t lowest_equivalent_value_given_bucket_indices(
const struct hdr_histogram *h,
int32_t bucket_index,
int32_t sub_bucket_index)
{
return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude);
}
int64_t hdr_next_non_equivalent_value(const struct hdr_histogram *h, int64_t value)
{
return lowest_equivalent_value(h, value) + hdr_size_of_equivalent_value_range(h, value);
@@ -317,7 +341,7 @@ static int32_t buckets_needed_to_cover_value(int64_t value, int32_t sub_bucket_c
/* ## ## ######## ## ## ####### ## ## ## */
int hdr_calculate_bucket_config(
int64_t lowest_trackable_value,
int64_t lowest_discernible_value,
int64_t highest_trackable_value,
int significant_figures,
struct hdr_histogram_bucket_config* cfg)
@@ -325,14 +349,14 @@ int hdr_calculate_bucket_config(
int32_t sub_bucket_count_magnitude;
int64_t largest_value_with_single_unit_resolution;
if (lowest_trackable_value < 1 ||
if (lowest_discernible_value < 1 ||
significant_figures < 1 || 5 < significant_figures ||
lowest_trackable_value * 2 > highest_trackable_value)
lowest_discernible_value * 2 > highest_trackable_value)
{
return EINVAL;
}
cfg->lowest_trackable_value = lowest_trackable_value;
cfg->lowest_discernible_value = lowest_discernible_value;
cfg->significant_figures = significant_figures;
cfg->highest_trackable_value = highest_trackable_value;
@@ -340,8 +364,13 @@ int hdr_calculate_bucket_config(
sub_bucket_count_magnitude = (int32_t) ceil(log((double)largest_value_with_single_unit_resolution) / log(2));
cfg->sub_bucket_half_count_magnitude = ((sub_bucket_count_magnitude > 1) ? sub_bucket_count_magnitude : 1) - 1;
cfg->unit_magnitude = (int32_t) floor(log((double)lowest_trackable_value) / log(2));
double unit_magnitude = log((double)lowest_discernible_value) / log(2);
if (INT32_MAX < unit_magnitude)
{
return EINVAL;
}
cfg->unit_magnitude = (int32_t) unit_magnitude;
cfg->sub_bucket_count = (int32_t) pow(2, (cfg->sub_bucket_half_count_magnitude + 1));
cfg->sub_bucket_half_count = cfg->sub_bucket_count / 2;
cfg->sub_bucket_mask = ((int64_t) cfg->sub_bucket_count - 1) << cfg->unit_magnitude;
@@ -359,7 +388,7 @@ int hdr_calculate_bucket_config(
void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg)
{
h->lowest_trackable_value = cfg->lowest_trackable_value;
h->lowest_discernible_value = cfg->lowest_discernible_value;
h->highest_trackable_value = cfg->highest_trackable_value;
h->unit_magnitude = (int32_t)cfg->unit_magnitude;
h->significant_figures = (int32_t)cfg->significant_figures;
@@ -377,7 +406,7 @@ void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_
}
int hdr_init(
int64_t lowest_trackable_value,
int64_t lowest_discernible_value,
int64_t highest_trackable_value,
int significant_figures,
struct hdr_histogram** result)
@@ -386,22 +415,22 @@ int hdr_init(
struct hdr_histogram_bucket_config cfg;
struct hdr_histogram* histogram;
int r = hdr_calculate_bucket_config(lowest_trackable_value, highest_trackable_value, significant_figures, &cfg);
int r = hdr_calculate_bucket_config(lowest_discernible_value, highest_trackable_value, significant_figures, &cfg);
if (r)
{
return r;
}
counts = (int64_t*) calloc((size_t) cfg.counts_len, sizeof(int64_t));
counts = (int64_t*) hdr_calloc((size_t) cfg.counts_len, sizeof(int64_t));
if (!counts)
{
return ENOMEM;
}
histogram = (struct hdr_histogram*) calloc(1, sizeof(struct hdr_histogram));
histogram = (struct hdr_histogram*) hdr_calloc(1, sizeof(struct hdr_histogram));
if (!histogram)
{
free(counts);
hdr_free(counts);
return ENOMEM;
}
@@ -416,8 +445,8 @@ int hdr_init(
void hdr_close(struct hdr_histogram* h)
{
if (h) {
free(h->counts);
free(h);
hdr_free(h->counts);
hdr_free(h);
}
}
@@ -637,31 +666,71 @@ int64_t hdr_min(const struct hdr_histogram* h)
return non_zero_min(h);
}
int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile)
static int64_t get_value_from_idx_up_to_count(const struct hdr_histogram* h, int64_t count_at_percentile)
{
struct hdr_iter iter;
int64_t total = 0;
double requested_percentile = percentile < 100.0 ? percentile : 100.0;
int64_t count_at_percentile =
(int64_t) (((requested_percentile / 100) * h->total_count) + 0.5);
count_at_percentile = count_at_percentile > 1 ? count_at_percentile : 1;
int64_t count_to_idx = 0;
hdr_iter_init(&iter, h);
while (hdr_iter_next(&iter))
count_at_percentile = 0 < count_at_percentile ? count_at_percentile : 1;
for (int32_t idx = 0; idx < h->counts_len; idx++)
{
total += iter.count;
if (total >= count_at_percentile)
count_to_idx += h->counts[idx];
if (count_to_idx >= count_at_percentile)
{
int64_t value_from_index = iter.value;
return highest_equivalent_value(h, value_from_index);
return hdr_value_at_index(h, idx);
}
}
return 0;
}
int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile)
{
double requested_percentile = percentile < 100.0 ? percentile : 100.0;
int64_t count_at_percentile =
(int64_t) (((requested_percentile / 100) * h->total_count) + 0.5);
int64_t value_from_idx = get_value_from_idx_up_to_count(h, count_at_percentile);
if (percentile == 0.0)
{
return lowest_equivalent_value(h, value_from_idx);
}
return highest_equivalent_value(h, value_from_idx);
}
int hdr_value_at_percentiles(const struct hdr_histogram *h, const double *percentiles, int64_t *values, size_t length)
{
if (NULL == percentiles || NULL == values)
{
return EINVAL;
}
struct hdr_iter iter;
const int64_t total_count = h->total_count;
// to avoid allocations we use the values array for intermediate computation
// i.e. to store the expected cumulative count at each percentile
for (size_t i = 0; i < length; i++)
{
const double requested_percentile = percentiles[i] < 100.0 ? percentiles[i] : 100.0;
const int64_t count_at_percentile =
(int64_t) (((requested_percentile / 100) * total_count) + 0.5);
values[i] = count_at_percentile > 1 ? count_at_percentile : 1;
}
hdr_iter_init(&iter, h);
int64_t total = 0;
size_t at_pos = 0;
while (hdr_iter_next(&iter) && at_pos < length)
{
total += iter.count;
while (at_pos < length && total >= values[at_pos])
{
values[at_pos] = highest_equivalent_value(h, iter.value);
at_pos++;
}
}
return 0;
}
double hdr_mean(const struct hdr_histogram* h)
{
struct hdr_iter iter;
@@ -751,11 +820,16 @@ static bool move_next(struct hdr_iter* iter)
iter->count = counts_get_normalised(iter->h, iter->counts_index);
iter->cumulative_count += iter->count;
iter->value = hdr_value_at_index(iter->h, iter->counts_index);
iter->highest_equivalent_value = highest_equivalent_value(iter->h, iter->value);
iter->lowest_equivalent_value = lowest_equivalent_value(iter->h, iter->value);
iter->median_equivalent_value = hdr_median_equivalent_value(iter->h, iter->value);
const int64_t value = hdr_value_at_index(iter->h, iter->counts_index);
const int32_t bucket_index = get_bucket_index(iter->h, value);
const int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, iter->h->unit_magnitude);
const int64_t leq = lowest_equivalent_value_given_bucket_indices(iter->h, bucket_index, sub_bucket_index);
const int64_t size_of_equivalent_value_range = size_of_equivalent_value_range_given_bucket_indices(
iter->h, bucket_index, sub_bucket_index);
iter->lowest_equivalent_value = leq;
iter->value = value;
iter->highest_equivalent_value = leq + size_of_equivalent_value_range - 1;
iter->median_equivalent_value = leq + (size_of_equivalent_value_range >> 1);
return true;
}
deps/hdr_histogram/hdr_histogram.h
Vendored
+28 -16
View File
@@ -16,7 +16,7 @@
struct hdr_histogram
{
int64_t lowest_trackable_value;
int64_t lowest_discernible_value;
int64_t highest_trackable_value;
int32_t unit_magnitude;
int32_t significant_figures;
@@ -45,8 +45,8 @@ extern "C" {
* involved math on the input parameters this function it is tricky to stack allocate.
* The histogram should be released with hdr_close
*
* @param lowest_trackable_value The smallest possible value to be put into the
* histogram.
* @param lowest_discernible_value The smallest possible value that is distinguishable from 0.
* Must be a positive integer that is >= 1. May be internally rounded down to nearest power of 2.
* @param highest_trackable_value The largest possible value to be put into the
* histogram.
* @param significant_figures The level of precision for this histogram, i.e. the number
@@ -54,12 +54,12 @@ extern "C" {
* the results from the histogram will be accurate up to the first three digits. Must
* be a value between 1 and 5 (inclusive).
* @param result Output parameter to capture allocated histogram.
* @return 0 on success, EINVAL if lowest_trackable_value is < 1 or the
* @return 0 on success, EINVAL if lowest_discernible_value is < 1 or the
* significant_figure value is outside of the allowed range, ENOMEM if malloc
* failed.
*/
int hdr_init(
int64_t lowest_trackable_value,
int64_t lowest_discernible_value,
int64_t highest_trackable_value,
int significant_figures,
struct hdr_histogram** result);
@@ -159,10 +159,10 @@ bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t co
* Record a value in the histogram and backfill based on an expected interval.
*
* Records a value in the histogram, will round this value of to a precision at or better
* than the significant_figure specified at contruction time. This is specifically used
* than the significant_figure specified at construction time. This is specifically used
* for recording latency. If the value is larger than the expected_interval then the
* latency recording system has experienced co-ordinated omission. This method fills in the
* values that would have occured had the client providing the load not been blocked.
* values that would have occurred had the client providing the load not been blocked.
* @param h "This" pointer
* @param value Value to add to the histogram
@@ -170,16 +170,16 @@ bool hdr_record_values_atomic(struct hdr_histogram* h, int64_t value, int64_t co
* @return false if the value is larger than the highest_trackable_value and can't be recorded,
* true otherwise.
*/
bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval);
bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expected_interval);
/**
* Record a value in the histogram and backfill based on an expected interval.
*
* Records a value in the histogram, will round this value of to a precision at or better
* than the significant_figure specified at contruction time. This is specifically used
* than the significant_figure specified at construction time. This is specifically used
* for recording latency. If the value is larger than the expected_interval then the
* latency recording system has experienced co-ordinated omission. This method fills in the
* values that would have occured had the client providing the load not been blocked.
* values that would have occurred had the client providing the load not been blocked.
*
* Will record this value atomically, however the whole structure may appear inconsistent
* when read concurrently with this update. Do NOT mix calls to this method with calls
@@ -191,7 +191,7 @@ bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t
* @return false if the value is larger than the highest_trackable_value and can't be recorded,
* true otherwise.
*/
bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval);
bool hdr_record_corrected_value_atomic(struct hdr_histogram* h, int64_t value, int64_t expected_interval);
/**
* Record a value in the histogram 'count' times. Applies the same correcting logic
@@ -226,7 +226,7 @@ bool hdr_record_corrected_values_atomic(struct hdr_histogram* h, int64_t value,
/**
* Adds all of the values from 'from' to 'this' histogram. Will return the
* number of values that are dropped when copying. Values will be dropped
* if they around outside of h.lowest_trackable_value and
* if they around outside of h.lowest_discernible_value and
* h.highest_trackable_value.
*
* @param h "This" pointer
@@ -238,7 +238,7 @@ int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from);
/**
* Adds all of the values from 'from' to 'this' histogram. Will return the
* number of values that are dropped when copying. Values will be dropped
* if they around outside of h.lowest_trackable_value and
* if they around outside of h.lowest_discernible_value and
* h.highest_trackable_value.
*
* @param h "This" pointer
@@ -272,6 +272,18 @@ int64_t hdr_max(const struct hdr_histogram* h);
*/
int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile);
/**
* Get the values at the given percentiles.
*
* @param h "This" pointer.
* @param percentiles The ordered percentiles array to get the values for.
* @param length Number of elements in the arrays.
* @param values Destination array containing the values at the given percentiles.
* The values array should be allocated by the caller.
* @return 0 on success, ENOMEM if the provided destination array is null.
*/
int hdr_value_at_percentiles(const struct hdr_histogram *h, const double *percentiles, int64_t *values, size_t length);
/**
* Gets the standard deviation for the values in the histogram.
*
@@ -470,7 +482,7 @@ int hdr_percentiles_print(
*/
struct hdr_histogram_bucket_config
{
int64_t lowest_trackable_value;
int64_t lowest_discernible_value;
int64_t highest_trackable_value;
int64_t unit_magnitude;
int64_t significant_figures;
@@ -483,7 +495,7 @@ struct hdr_histogram_bucket_config
};
int hdr_calculate_bucket_config(
int64_t lowest_trackable_value,
int64_t lowest_discernible_value,
int64_t highest_trackable_value,
int significant_figures,
struct hdr_histogram_bucket_config* cfg);
@@ -497,7 +509,7 @@ int64_t hdr_next_non_equivalent_value(const struct hdr_histogram* h, int64_t val
int64_t hdr_median_equivalent_value(const struct hdr_histogram* h, int64_t value);
/**
* Used to reset counters after importing data manuallying into the histogram, used by the logging code
* Used to reset counters after importing data manually into the histogram, used by the logging code
* and other custom serialisation tools.
*/
void hdr_reset_internal_counters(struct hdr_histogram* h);
deps/hdr_histogram/hdr_redis_malloc.h
Vendored
+13
View File
@@ -0,0 +1,13 @@
#ifndef HDR_MALLOC_H__
#define HDR_MALLOC_H__
void *zmalloc(size_t size);
void *zcalloc_num(size_t num, size_t size);
void *zrealloc(void *ptr, size_t size);
void zfree(void *ptr);
#define hdr_malloc zmalloc
#define hdr_calloc zcalloc_num
#define hdr_realloc zrealloc
#define hdr_free zfree
#endif
deps/hdr_histogram/hdr_tests.h
Vendored
+22
View File
@@ -0,0 +1,22 @@
#ifndef HDR_TESTS_H
#define HDR_TESTS_H
/* These are functions used in tests and are not intended for normal usage. */
#include "hdr_histogram.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t counts_index_for(const struct hdr_histogram* h, int64_t value);
int hdr_encode_compressed(struct hdr_histogram* h, uint8_t** compressed_histogram, size_t* compressed_len);
int hdr_decode_compressed(uint8_t* buffer, size_t length, struct hdr_histogram** histogram);
void hdr_base64_decode_block(const char* input, uint8_t* output);
void hdr_base64_encode_block(const uint8_t* input, char* output);
#ifdef __cplusplus
}
#endif
#endif
deps/hiredis/.github/workflows/build.yml
Vendored
+205
View File
@@ -0,0 +1,205 @@
name: Build and test
on: [push, pull_request]
jobs:
ubuntu:
name: Ubuntu
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: Install dependencies
run: |
sudo add-apt-repository -y ppa:chris-lea/redis-server
sudo apt-get update
sudo apt-get install -y redis-server valgrind libevent-dev
- name: Build using cmake
env:
EXTRA_CMAKE_OPTS: -DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON -DENABLE_ASYNC_TESTS:BOOL=ON
CFLAGS: -Werror
CXXFLAGS: -Werror
run: mkdir build-ubuntu && cd build-ubuntu && cmake ..
- name: Build using makefile
run: USE_SSL=1 TEST_ASYNC=1 make
- name: Run tests
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
run: $GITHUB_WORKSPACE/test.sh
# - name: Run tests under valgrind
# env:
# SKIPS_AS_FAILS: 1
# TEST_PREFIX: valgrind --error-exitcode=99 --track-origins=yes --leak-check=full
# run: $GITHUB_WORKSPACE/test.sh
centos7:
name: CentOS 7
runs-on: ubuntu-latest
container: centos:7
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: Install dependencies
run: |
yum -y install http://rpms.remirepo.net/enterprise/remi-release-7.rpm
yum -y --enablerepo=remi install redis
yum -y install gcc gcc-c++ make openssl openssl-devel cmake3 valgrind libevent-devel
- name: Build using cmake
env:
EXTRA_CMAKE_OPTS: -DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON -DENABLE_ASYNC_TESTS:BOOL=ON
CFLAGS: -Werror
CXXFLAGS: -Werror
run: mkdir build-centos7 && cd build-centos7 && cmake3 ..
- name: Build using Makefile
run: USE_SSL=1 TEST_ASYNC=1 make
- name: Run tests
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
run: $GITHUB_WORKSPACE/test.sh
- name: Run tests under valgrind
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
TEST_PREFIX: valgrind --error-exitcode=99 --track-origins=yes --leak-check=full
run: $GITHUB_WORKSPACE/test.sh
centos8:
name: RockyLinux 8
runs-on: ubuntu-latest
container: rockylinux:8
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: Install dependencies
run: |
dnf -y install https://rpms.remirepo.net/enterprise/remi-release-8.rpm
dnf -y module install redis:remi-6.0
dnf -y group install "Development Tools"
dnf -y install openssl-devel cmake valgrind libevent-devel
- name: Build using cmake
env:
EXTRA_CMAKE_OPTS: -DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON -DENABLE_ASYNC_TESTS:BOOL=ON
CFLAGS: -Werror
CXXFLAGS: -Werror
run: mkdir build-centos8 && cd build-centos8 && cmake ..
- name: Build using Makefile
run: USE_SSL=1 TEST_ASYNC=1 make
- name: Run tests
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
run: $GITHUB_WORKSPACE/test.sh
- name: Run tests under valgrind
env:
SKIPS_AS_FAILS: 1
TEST_SSL: 1
TEST_PREFIX: valgrind --error-exitcode=99 --track-origins=yes --leak-check=full
run: $GITHUB_WORKSPACE/test.sh
freebsd:
runs-on: macos-10.15
name: FreeBSD
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: Build in FreeBSD
uses: vmactions/freebsd-vm@v0.1.5
with:
prepare: pkg install -y gmake cmake
run: |
mkdir build && cd build && cmake .. && make && cd ..
gmake
macos:
name: macOS
runs-on: macos-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: Install dependencies
run: |
brew install openssl redis
- name: Build hiredis
run: USE_SSL=1 make
- name: Run tests
env:
TEST_SSL: 1
run: $GITHUB_WORKSPACE/test.sh
windows:
name: Windows
runs-on: windows-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: Install dependencies
run: |
choco install -y ninja memurai-developer
- uses: ilammy/msvc-dev-cmd@v1
- name: Build hiredis
run: |
mkdir build && cd build
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DENABLE_EXAMPLES=ON
ninja -v
- name: Run tests
run: |
./build/hiredis-test.exe
- name: Setup cygwin
uses: egor-tensin/setup-cygwin@v3
with:
platform: x64
packages: make git gcc-core
- name: Build in cygwin
env:
HIREDIS_PATH: ${{ github.workspace }}
run: |
build_hiredis() {
cd $(cygpath -u $HIREDIS_PATH)
git clean -xfd
make
}
build_hiredis
shell: C:\tools\cygwin\bin\bash.exe --login --norc -eo pipefail -o igncr '{0}'
deps/hiredis/.travis.yml
Vendored
+3 -9
View File
@@ -17,11 +17,11 @@ branches:
- /^release\/.*$/
install:
- if [ "$BITS" == "64" ]; then
- if [ "$TRAVIS_COMPILER" != "mingw" ]; then
wget https://github.com/redis/redis/archive/6.0.6.tar.gz;
tar -xzvf 6.0.6.tar.gz;
pushd redis-6.0.6 && BUILD_TLS=yes make && export PATH=$PWD/src:$PATH && popd;
fi
fi;
before_script:
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then
@@ -33,8 +33,6 @@ before_script:
addons:
apt:
sources:
- sourceline: 'ppa:chris-lea/redis-server'
packages:
- libc6-dbg
- libc6-dev
@@ -46,17 +44,13 @@ addons:
- libssl-dev
- libssl-dev:i386
- valgrind
- redis
env:
- BITS="32"
- BITS="64"
script:
- EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON";
if [ "$BITS" == "64" ]; then
EXTRA_CMAKE_OPTS="$EXTRA_CMAKE_OPTS -DENABLE_SSL_TESTS:BOOL=ON";
fi;
- EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DENABLE_SSL:BOOL=ON -DENABLE_SSL_TESTS:BOOL=ON";
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
if [ "$BITS" == "32" ]; then
CFLAGS="-m32 -Werror";
deps/hiredis/CHANGELOG.md
Vendored
+19
View File
@@ -1,3 +1,22 @@
## [1.0.2](https://github.com/redis/hiredis/tree/v1.0.2) - (2021-10-07)
Announcing Hiredis v1.0.2, which fixes CVE-2021-32765 but returns the SONAME to the correct value of `1.0.0`.
- [Revert SONAME bump](https://github.com/redis/hiredis/commit/d4e6f109a064690cde64765c654e679fea1d3548)
([Michael Grunder](https://github.com/michael-grunder))
## [1.0.1](https://github.com/redis/hiredis/tree/v1.0.1) - (2021-10-04)
<span style="color:red">This release erroneously bumped the SONAME, please use [1.0.2](https://github.com/redis/hiredis/tree/v1.0.2)</span>
Announcing Hiredis v1.0.1, a security release fixing CVE-2021-32765
- Fix for [CVE-2021-32765](https://github.com/redis/hiredis/security/advisories/GHSA-hfm9-39pp-55p2)
[commit](https://github.com/redis/hiredis/commit/76a7b10005c70babee357a7d0f2becf28ec7ed1e)
([Yossi Gottlieb](https://github.com/yossigo))
_Thanks to [Yossi Gottlieb](https://github.com/yossigo) for the security fix and to [Microsoft Security Vulnerability Research](https://www.microsoft.com/en-us/msrc/msvr) for finding the bug._ :sparkling_heart:
## [1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) - (2020-08-03)
Announcing Hiredis v1.0.0, which adds support for RESP3, SSL connections, allocator injection, and better Windows support! :tada:
deps/hiredis/CMakeLists.txt
Vendored
+95 -15
View File
@@ -1,10 +1,9 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0)
INCLUDE(GNUInstallDirs)
PROJECT(hiredis)
OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF)
OPTION(DISABLE_TESTS "If tests should be compiled or not" OFF)
OPTION(ENABLE_SSL_TESTS, "Should we test SSL connections" OFF)
OPTION(ENABLE_SSL_TESTS "Should we test SSL connections" OFF)
OPTION(ENABLE_ASYNC_TESTS "Should we run all asynchronous API tests" OFF)
MACRO(getVersionBit name)
SET(VERSION_REGEX "^#define ${name} (.+)$")
@@ -20,7 +19,13 @@ getVersionBit(HIREDIS_SONAME)
SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}")
MESSAGE("Detected version: ${VERSION}")
PROJECT(hiredis VERSION "${VERSION}")
PROJECT(hiredis LANGUAGES "C" VERSION "${VERSION}")
INCLUDE(GNUInstallDirs)
# Hiredis requires C99
SET(CMAKE_C_STANDARD 99)
SET(CMAKE_POSITION_INDEPENDENT_CODE ON)
SET(CMAKE_DEBUG_POSTFIX d)
SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples")
@@ -41,30 +46,84 @@ IF(WIN32)
ENDIF()
ADD_LIBRARY(hiredis SHARED ${hiredis_sources})
ADD_LIBRARY(hiredis_static STATIC ${hiredis_sources})
ADD_LIBRARY(hiredis::hiredis ALIAS hiredis)
ADD_LIBRARY(hiredis::hiredis_static ALIAS hiredis_static)
SET_TARGET_PROPERTIES(hiredis
PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE
VERSION "${HIREDIS_SONAME}")
SET_TARGET_PROPERTIES(hiredis_static
PROPERTIES COMPILE_PDB_NAME hiredis_static)
SET_TARGET_PROPERTIES(hiredis_static
PROPERTIES COMPILE_PDB_NAME_DEBUG hiredis_static${CMAKE_DEBUG_POSTFIX})
IF(WIN32 OR MINGW)
TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32)
TARGET_LINK_LIBRARIES(hiredis PUBLIC ws2_32 crypt32)
TARGET_LINK_LIBRARIES(hiredis_static PUBLIC ws2_32 crypt32)
ELSEIF(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
TARGET_LINK_LIBRARIES(hiredis PUBLIC m)
TARGET_LINK_LIBRARIES(hiredis_static PUBLIC m)
ELSEIF(CMAKE_SYSTEM_NAME MATCHES "SunOS")
TARGET_LINK_LIBRARIES(hiredis PUBLIC socket)
TARGET_LINK_LIBRARIES(hiredis_static PUBLIC socket)
ENDIF()
TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC $<INSTALL_INTERFACE:.> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
TARGET_INCLUDE_DIRECTORIES(hiredis_static PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY)
INSTALL(TARGETS hiredis
set(CPACK_PACKAGE_VENDOR "Redis")
set(CPACK_PACKAGE_DESCRIPTION "\
Hiredis is a minimalistic C client library for the Redis database.
It is minimalistic because it just adds minimal support for the protocol, \
but at the same time it uses a high level printf-alike API in order to make \
it much higher level than otherwise suggested by its minimal code base and the \
lack of explicit bindings for every Redis command.
Apart from supporting sending commands and receiving replies, it comes with a \
reply parser that is decoupled from the I/O layer. It is a stream parser designed \
for easy reusability, which can for instance be used in higher level language bindings \
for efficient reply parsing.
Hiredis only supports the binary-safe Redis protocol, so you can use it with any Redis \
version >= 1.2.0.
The library comes with multiple APIs. There is the synchronous API, the asynchronous API \
and the reply parsing API.")
set(CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/redis/hiredis")
set(CPACK_PACKAGE_CONTACT "michael dot grunder at gmail dot com")
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
set(CPACK_RPM_PACKAGE_AUTOREQPROV ON)
include(CPack)
INSTALL(TARGETS hiredis hiredis_static
EXPORT hiredis-targets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
if (MSVC)
INSTALL(FILES $<TARGET_PDB_FILE:hiredis>
DESTINATION ${CMAKE_INSTALL_BINDIR}
CONFIGURATIONS Debug RelWithDebInfo)
INSTALL(FILES $<TARGET_FILE_DIR:hiredis_static>/$<TARGET_FILE_BASE_NAME:hiredis_static>.pdb
DESTINATION ${CMAKE_INSTALL_LIBDIR}
CONFIGURATIONS Debug RelWithDebInfo)
endif()
# For NuGet packages
INSTALL(FILES hiredis.targets
DESTINATION build/native)
INSTALL(FILES hiredis.h read.h sds.h async.h alloc.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
INSTALL(DIRECTORY adapters
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
@@ -95,10 +154,12 @@ IF(ENABLE_SSL)
ENDIF()
ENDIF()
FIND_PACKAGE(OpenSSL REQUIRED)
SET(hiredis_ssl_sources
SET(hiredis_ssl_sources
ssl.c)
ADD_LIBRARY(hiredis_ssl SHARED
${hiredis_ssl_sources})
ADD_LIBRARY(hiredis_ssl_static STATIC
${hiredis_ssl_sources})
IF (APPLE)
SET_PROPERTY(TARGET hiredis_ssl PROPERTY LINK_FLAGS "-Wl,-undefined -Wl,dynamic_lookup")
@@ -108,23 +169,39 @@ IF(ENABLE_SSL)
PROPERTIES
WINDOWS_EXPORT_ALL_SYMBOLS TRUE
VERSION "${HIREDIS_SONAME}")
SET_TARGET_PROPERTIES(hiredis_ssl_static
PROPERTIES COMPILE_PDB_NAME hiredis_ssl_static)
SET_TARGET_PROPERTIES(hiredis_ssl_static
PROPERTIES COMPILE_PDB_NAME_DEBUG hiredis_ssl_static${CMAKE_DEBUG_POSTFIX})
TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}")
TARGET_INCLUDE_DIRECTORIES(hiredis_ssl_static PRIVATE "${OPENSSL_INCLUDE_DIR}")
TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES})
IF (WIN32 OR MINGW)
TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE hiredis)
TARGET_LINK_LIBRARIES(hiredis_ssl_static PUBLIC hiredis_static)
ENDIF()
CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY)
INSTALL(TARGETS hiredis_ssl
INSTALL(TARGETS hiredis_ssl hiredis_ssl_static
EXPORT hiredis_ssl-targets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
if (MSVC)
INSTALL(FILES $<TARGET_PDB_FILE:hiredis_ssl>
DESTINATION ${CMAKE_INSTALL_BINDIR}
CONFIGURATIONS Debug RelWithDebInfo)
INSTALL(FILES $<TARGET_FILE_DIR:hiredis_ssl_static>/$<TARGET_FILE_BASE_NAME:hiredis_ssl_static>.pdb
DESTINATION ${CMAKE_INSTALL_LIBDIR}
CONFIGURATIONS Debug RelWithDebInfo)
endif()
INSTALL(FILES hiredis_ssl.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
@@ -149,11 +226,14 @@ ENDIF()
IF(NOT DISABLE_TESTS)
ENABLE_TESTING()
ADD_EXECUTABLE(hiredis-test test.c)
TARGET_LINK_LIBRARIES(hiredis-test hiredis)
IF(ENABLE_SSL_TESTS)
ADD_DEFINITIONS(-DHIREDIS_TEST_SSL=1)
TARGET_LINK_LIBRARIES(hiredis-test hiredis hiredis_ssl)
ELSE()
TARGET_LINK_LIBRARIES(hiredis-test hiredis)
TARGET_LINK_LIBRARIES(hiredis-test hiredis_ssl)
ENDIF()
IF(ENABLE_ASYNC_TESTS)
ADD_DEFINITIONS(-DHIREDIS_TEST_ASYNC=1)
TARGET_LINK_LIBRARIES(hiredis-test event)
ENDIF()
ADD_TEST(NAME hiredis-test
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh)
deps/hiredis/Makefile
Vendored
+71 -41
View File
@@ -4,16 +4,10 @@
# This file is released under the BSD license, see the COPYING file
OBJ=alloc.o net.o hiredis.o sds.o async.o read.o sockcompat.o
SSL_OBJ=ssl.o
EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib hiredis-example-push
ifeq ($(USE_SSL),1)
EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl
endif
TESTS=hiredis-test
LIBNAME=libhiredis
PKGCONFNAME=hiredis.pc
SSL_LIBNAME=libhiredis_ssl
SSL_PKGCONFNAME=hiredis_ssl.pc
HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}')
HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}')
@@ -60,28 +54,66 @@ DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME)
STLIBNAME=$(LIBNAME).$(STLIBSUFFIX)
STLIB_MAKE_CMD=$(AR) rcs
#################### SSL variables start ####################
SSL_OBJ=ssl.o
SSL_LIBNAME=libhiredis_ssl
SSL_PKGCONFNAME=hiredis_ssl.pc
SSL_INSTALLNAME=install-ssl
SSL_DYLIB_MINOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME)
SSL_DYLIB_MAJOR_NAME=$(SSL_LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR)
SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX)
SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX)
SSL_DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(SSL_DYLIB_MINOR_NAME)
USE_SSL?=0
ifeq ($(USE_SSL),1)
# This is required for test.c only
CFLAGS+=-DHIREDIS_TEST_SSL
EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl
SSL_STLIB=$(SSL_STLIBNAME)
SSL_DYLIB=$(SSL_DYLIBNAME)
SSL_PKGCONF=$(SSL_PKGCONFNAME)
SSL_INSTALL=$(SSL_INSTALLNAME)
else
SSL_STLIB=
SSL_DYLIB=
SSL_PKGCONF=
SSL_INSTALL=
endif
##################### SSL variables end #####################
# Platform-specific overrides
uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
USE_SSL?=0
# This is required for test.c only
ifeq ($(USE_SSL),1)
CFLAGS+=-DHIREDIS_TEST_SSL
ifeq ($(TEST_ASYNC),1)
export CFLAGS+=-DHIREDIS_TEST_ASYNC
endif
ifeq ($(uname_S),Linux)
SSL_LDFLAGS=-lssl -lcrypto
ifeq ($(USE_SSL),1)
ifeq ($(uname_S),Linux)
ifdef OPENSSL_PREFIX
CFLAGS+=-I$(OPENSSL_PREFIX)/include
SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto
else
SSL_LDFLAGS=-lssl -lcrypto
endif
else
OPENSSL_PREFIX?=/usr/local/opt/openssl
CFLAGS+=-I$(OPENSSL_PREFIX)/include
SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto
endif
endif
ifeq ($(uname_S),FreeBSD)
LDFLAGS+=-lm
IS_GCC=$(shell sh -c '$(CC) --version 2>/dev/null |egrep -i -c "gcc"')
ifeq ($(IS_GCC),1)
REAL_CFLAGS+=-pedantic
endif
else
OPENSSL_PREFIX?=/usr/local/opt/openssl
CFLAGS+=-I$(OPENSSL_PREFIX)/include
SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto
REAL_CFLAGS+=-pedantic
endif
ifeq ($(uname_S),SunOS)
@@ -103,10 +135,13 @@ ifeq ($(uname_S),Darwin)
DYLIB_PLUGIN=-Wl,-undefined -Wl,dynamic_lookup
endif
all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME)
ifeq ($(USE_SSL),1)
all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME)
endif
all: dynamic static hiredis-test pkgconfig
dynamic: $(DYLIBNAME) $(SSL_DYLIB)
static: $(STLIBNAME) $(SSL_STLIB)
pkgconfig: $(PKGCONFNAME) $(SSL_PKGCONF)
# Deps (use make dep to generate this)
alloc.o: alloc.c fmacros.h alloc.h
@@ -117,7 +152,6 @@ net.o: net.c fmacros.h net.h hiredis.h read.h sds.h alloc.h sockcompat.h win32.h
read.o: read.c fmacros.h alloc.h read.h sds.h win32.h
sds.o: sds.c sds.h sdsalloc.h alloc.h
sockcompat.o: sockcompat.c sockcompat.h
ssl.o: ssl.c hiredis.h read.h sds.h alloc.h async.h win32.h async_private.h
test.o: test.c fmacros.h hiredis.h read.h sds.h alloc.h net.h sockcompat.h win32.h
$(DYLIBNAME): $(OBJ)
@@ -126,18 +160,15 @@ $(DYLIBNAME): $(OBJ)
$(STLIBNAME): $(OBJ)
$(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ)
#################### SSL building rules start ####################
$(SSL_DYLIBNAME): $(SSL_OBJ)
$(SSL_DYLIB_MAKE_CMD) $(DYLIB_PLUGIN) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(LDFLAGS) $(SSL_LDFLAGS)
$(SSL_STLIBNAME): $(SSL_OBJ)
$(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ)
dynamic: $(DYLIBNAME)
static: $(STLIBNAME)
ifeq ($(USE_SSL),1)
dynamic: $(SSL_DYLIBNAME)
static: $(SSL_STLIBNAME)
endif
$(SSL_OBJ): ssl.c hiredis.h read.h sds.h alloc.h async.h win32.h async_private.h
#################### SSL building rules end ####################
# Binaries:
hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME)
@@ -161,7 +192,6 @@ hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME)
hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS)
ifndef AE_DIR
hiredis-example-ae:
@echo "Please specify AE_DIR (e.g. <redis repository>/src)"
@@ -172,10 +202,11 @@ hiredis-example-ae: examples/example-ae.c adapters/ae.h $(STLIBNAME)
endif
ifndef LIBUV_DIR
hiredis-example-libuv:
@echo "Please specify LIBUV_DIR (e.g. ../libuv/)"
@false
# dynamic link libuv.so
hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< -luv -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS)
else
# use user provided static lib
hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME)
$(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS)
endif
@@ -201,10 +232,13 @@ hiredis-example-push: examples/example-push.c $(STLIBNAME)
examples: $(EXAMPLES)
TEST_LIBS = $(STLIBNAME)
TEST_LIBS = $(STLIBNAME) $(SSL_STLIB)
TEST_LDFLAGS = $(SSL_LDFLAGS)
ifeq ($(USE_SSL),1)
TEST_LIBS += $(SSL_STLIBNAME)
TEST_LDFLAGS = $(SSL_LDFLAGS) -lssl -lcrypto -lpthread
TEST_LDFLAGS += -pthread
endif
ifeq ($(TEST_ASYNC),1)
TEST_LDFLAGS += -levent
endif
hiredis-test: test.o $(TEST_LIBS)
@@ -220,7 +254,7 @@ check: hiredis-test
TEST_SSL=$(USE_SSL) ./test.sh
.c.o:
$(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $<
$(CC) -std=c99 -c $(REAL_CFLAGS) $<
clean:
rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov
@@ -257,7 +291,7 @@ $(SSL_PKGCONFNAME): hiredis_ssl.h
@echo Libs: -L\$${libdir} -lhiredis_ssl >> $@
@echo Libs.private: -lssl -lcrypto >> $@
install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME)
install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) $(SSL_INSTALL)
mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH)
$(INSTALL) hiredis.h async.h read.h sds.h alloc.h $(INSTALL_INCLUDE_PATH)
$(INSTALL) adapters/*.h $(INSTALL_INCLUDE_PATH)/adapters
@@ -267,9 +301,6 @@ install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME)
mkdir -p $(INSTALL_PKGCONF_PATH)
$(INSTALL) $(PKGCONFNAME) $(INSTALL_PKGCONF_PATH)
ifeq ($(USE_SSL),1)
install: install-ssl
install-ssl: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME)
mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_LIBRARY_PATH)
$(INSTALL) hiredis_ssl.h $(INSTALL_INCLUDE_PATH)
@@ -278,7 +309,6 @@ install-ssl: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME)
$(INSTALL) $(SSL_STLIBNAME) $(INSTALL_LIBRARY_PATH)
mkdir -p $(INSTALL_PKGCONF_PATH)
$(INSTALL) $(SSL_PKGCONFNAME) $(INSTALL_PKGCONF_PATH)
endif
32bit:
@echo ""
@@ -294,12 +324,12 @@ gprof:
$(MAKE) CFLAGS="-pg" LDFLAGS="-pg"
gcov:
$(MAKE) CFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS="-fprofile-arcs"
$(MAKE) CFLAGS+="-fprofile-arcs -ftest-coverage" LDFLAGS="-fprofile-arcs"
coverage: gcov
make check
mkdir -p tmp/lcov
lcov -d . -c -o tmp/lcov/hiredis.info
lcov -d . -c --exclude '/usr*' -o tmp/lcov/hiredis.info
genhtml --legend -o tmp/lcov/report tmp/lcov/hiredis.info
noopt:
deps/hiredis/README.md
Vendored
+27 -14
View File
@@ -1,10 +1,11 @@
[![Build Status](https://travis-ci.org/redis/hiredis.png)](https://travis-ci.org/redis/hiredis)
[![Build Status](https://github.com/redis/hiredis/actions/workflows/build.yml/badge.svg)](https://github.com/redis/hiredis/actions/workflows/build.yml)
**This Readme reflects the latest changed in the master branch. See [v1.0.0](https://github.com/redis/hiredis/tree/v1.0.0) for the Readme and documentation for the latest release ([API/ABI history](https://abi-laboratory.pro/?view=timeline&l=hiredis)).**
# HIREDIS
Hiredis is a minimalistic C client library for the [Redis](http://redis.io/) database.
Hiredis is a minimalistic C client library for the [Redis](https://redis.io/) database.
It is minimalistic because it just adds minimal support for the protocol, but
at the same time it uses a high level printf-alike API in order to make it
@@ -22,6 +23,12 @@ Redis version >= 1.2.0.
The library comes with multiple APIs. There is the
*synchronous API*, the *asynchronous API* and the *reply parsing API*.
## Upgrading to `1.0.2`
<span style="color:red">NOTE: v1.0.1 erroneously bumped SONAME, which is why it is skipped here.</span>
Version 1.0.2 is simply 1.0.0 with a fix for [CVE-2021-32765](https://github.com/redis/hiredis/security/advisories/GHSA-hfm9-39pp-55p2). They are otherwise identical.
## Upgrading to `1.0.0`
Version 1.0.0 marks the first stable release of Hiredis.
@@ -169,7 +176,7 @@ Hiredis also supports every new `RESP3` data type which are as follows. For mor
* **`REDIS_REPLY_MAP`**:
* An array with the added invariant that there will always be an even number of elements.
The MAP is functionally equivelant to `REDIS_REPLY_ARRAY` except for the previously mentioned invariant.
The MAP is functionally equivalent to `REDIS_REPLY_ARRAY` except for the previously mentioned invariant.
* **`REDIS_REPLY_SET`**:
* An array response where each entry is unique.
@@ -189,7 +196,7 @@ Hiredis also supports every new `RESP3` data type which are as follows. For mor
* **`REDIS_REPLY_VERB`**:
* A verbatim string, intended to be presented to the user without modification.
The string payload is stored in the `str` memeber, and type data is stored in the `vtype` member (e.g. `txt` for raw text or `md` for markdown).
The string payload is stored in the `str` member, and type data is stored in the `vtype` member (e.g. `txt` for raw text or `md` for markdown).
Replies should be freed using the `freeReplyObject()` function.
Note that this function will take care of freeing sub-reply objects
@@ -261,9 +268,9 @@ a single call to `read(2)`):
redisReply *reply;
redisAppendCommand(context,"SET foo bar");
redisAppendCommand(context,"GET foo");
redisGetReply(context,(void *)&reply); // reply for SET
redisGetReply(context,(void**)&reply); // reply for SET
freeReplyObject(reply);
redisGetReply(context,(void *)&reply); // reply for GET
redisGetReply(context,(void**)&reply); // reply for GET
freeReplyObject(reply);
```
This API can also be used to implement a blocking subscriber:
@@ -517,7 +524,7 @@ initialize OpenSSL and create a context. You can do that in two ways:
/* An Hiredis SSL context. It holds SSL configuration and can be reused across
* many contexts.
*/
redisSSLContext *ssl;
redisSSLContext *ssl_context;
/* An error variable to indicate what went wrong, if the context fails to
* initialize.
@@ -532,17 +539,23 @@ redisSSLContextError ssl_error;
redisInitOpenSSL();
/* Create SSL context */
ssl = redisCreateSSLContext(
ssl_context = redisCreateSSLContext(
"cacertbundle.crt", /* File name of trusted CA/ca bundle file, optional */
"/path/to/certs", /* Path of trusted certificates, optional */
"client_cert.pem", /* File name of client certificate file, optional */
"client_key.pem", /* File name of client private key, optional */
"redis.mydomain.com", /* Server name to request (SNI), optional */
&ssl_error
) != REDIS_OK) {
printf("SSL error: %s\n", redisSSLContextGetError(ssl_error);
/* Abort... */
}
&ssl_error);
if(ssl_context == NULL || ssl_error != 0) {
/* Handle error and abort... */
/* e.g.
printf("SSL error: %s\n",
(ssl_error != 0) ?
redisSSLContextGetError(ssl_error) : "Unknown error");
// Abort
*/
}
/* Create Redis context and establish connection */
c = redisConnect("localhost", 6443);
@@ -551,7 +564,7 @@ if (c == NULL || c->err) {
}
/* Negotiate SSL/TLS */
if (redisInitiateSSLWithContext(c, ssl) != REDIS_OK) {
if (redisInitiateSSLWithContext(c, ssl_context) != REDIS_OK) {
/* Handle error, in c->err / c->errstr */
}
```
deps/hiredis/adapters/libev.h
Vendored
+18 -9
View File
@@ -46,7 +46,7 @@ typedef struct redisLibevEvents {
static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) {
#if EV_MULTIPLICITY
((void)loop);
((void)EV_A);
#endif
((void)revents);
@@ -56,7 +56,7 @@ static void redisLibevReadEvent(EV_P_ ev_io *watcher, int revents) {
static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) {
#if EV_MULTIPLICITY
((void)loop);
((void)EV_A);
#endif
((void)revents);
@@ -66,8 +66,9 @@ static void redisLibevWriteEvent(EV_P_ ev_io *watcher, int revents) {
static void redisLibevAddRead(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
((void)loop);
#endif
if (!e->reading) {
e->reading = 1;
ev_io_start(EV_A_ &e->rev);
@@ -76,8 +77,9 @@ static void redisLibevAddRead(void *privdata) {
static void redisLibevDelRead(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
((void)loop);
#endif
if (e->reading) {
e->reading = 0;
ev_io_stop(EV_A_ &e->rev);
@@ -86,8 +88,9 @@ static void redisLibevDelRead(void *privdata) {
static void redisLibevAddWrite(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
((void)loop);
#endif
if (!e->writing) {
e->writing = 1;
ev_io_start(EV_A_ &e->wev);
@@ -96,8 +99,9 @@ static void redisLibevAddWrite(void *privdata) {
static void redisLibevDelWrite(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
((void)loop);
#endif
if (e->writing) {
e->writing = 0;
ev_io_stop(EV_A_ &e->wev);
@@ -106,8 +110,9 @@ static void redisLibevDelWrite(void *privdata) {
static void redisLibevStopTimer(void *privdata) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
((void)loop);
#endif
ev_timer_stop(EV_A_ &e->timer);
}
@@ -120,6 +125,9 @@ static void redisLibevCleanup(void *privdata) {
}
static void redisLibevTimeout(EV_P_ ev_timer *timer, int revents) {
#if EV_MULTIPLICITY
((void)EV_A);
#endif
((void)revents);
redisLibevEvents *e = (redisLibevEvents*)timer->data;
redisAsyncHandleTimeout(e->context);
@@ -127,8 +135,9 @@ static void redisLibevTimeout(EV_P_ ev_timer *timer, int revents) {
static void redisLibevSetTimeout(void *privdata, struct timeval tv) {
redisLibevEvents *e = (redisLibevEvents*)privdata;
#if EV_MULTIPLICITY
struct ev_loop *loop = e->loop;
((void)loop);
#endif
if (!ev_is_active(&e->timer)) {
ev_init(&e->timer, redisLibevTimeout);
@@ -154,7 +163,7 @@ static int redisLibevAttach(EV_P_ redisAsyncContext *ac) {
e->context = ac;
#if EV_MULTIPLICITY
e->loop = loop;
e->loop = EV_A;
#else
e->loop = NULL;
#endif
deps/hiredis/adapters/libevent.h
Vendored
+1 -1
View File
@@ -50,7 +50,7 @@ static void redisLibeventDestroy(redisLibeventEvents *e) {
hi_free(e);
}
static void redisLibeventHandler(int fd, short event, void *arg) {
static void redisLibeventHandler(evutil_socket_t fd, short event, void *arg) {
((void)fd);
redisLibeventEvents *e = (redisLibeventEvents*)arg;
e->state |= REDIS_LIBEVENT_ENTERED;
deps/hiredis/adapters/libuv.h
Vendored
+105 -59
View File
@@ -7,111 +7,157 @@
#include <string.h>
typedef struct redisLibuvEvents {
redisAsyncContext* context;
uv_poll_t handle;
int events;
redisAsyncContext* context;
uv_poll_t handle;
uv_timer_t timer;
int events;
} redisLibuvEvents;
static void redisLibuvPoll(uv_poll_t* handle, int status, int events) {
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
int ev = (status ? p->events : events);
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
int ev = (status ? p->events : events);
if (p->context != NULL && (ev & UV_READABLE)) {
redisAsyncHandleRead(p->context);
}
if (p->context != NULL && (ev & UV_WRITABLE)) {
redisAsyncHandleWrite(p->context);
}
if (p->context != NULL && (ev & UV_READABLE)) {
redisAsyncHandleRead(p->context);
}
if (p->context != NULL && (ev & UV_WRITABLE)) {
redisAsyncHandleWrite(p->context);
}
}
static void redisLibuvAddRead(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->events |= UV_READABLE;
p->events |= UV_READABLE;
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
}
static void redisLibuvDelRead(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->events &= ~UV_READABLE;
p->events &= ~UV_READABLE;
if (p->events) {
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
} else {
uv_poll_stop(&p->handle);
}
if (p->events) {
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
} else {
uv_poll_stop(&p->handle);
}
}
static void redisLibuvAddWrite(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->events |= UV_WRITABLE;
p->events |= UV_WRITABLE;
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
}
static void redisLibuvDelWrite(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->events &= ~UV_WRITABLE;
p->events &= ~UV_WRITABLE;
if (p->events) {
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
} else {
uv_poll_stop(&p->handle);
}
if (p->events) {
uv_poll_start(&p->handle, p->events, redisLibuvPoll);
} else {
uv_poll_stop(&p->handle);
}
}
static void on_close(uv_handle_t* handle) {
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
hi_free(p);
static void on_timer_close(uv_handle_t *handle) {
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
p->timer.data = NULL;
if (!p->handle.data) {
// both timer and handle are closed
hi_free(p);
}
// else, wait for `on_handle_close`
}
static void on_handle_close(uv_handle_t *handle) {
redisLibuvEvents* p = (redisLibuvEvents*)handle->data;
p->handle.data = NULL;
if (!p->timer.data) {
// timer never started, or timer already destroyed
hi_free(p);
}
// else, wait for `on_timer_close`
}
// libuv removed `status` parameter since v0.11.23
// see: https://github.com/libuv/libuv/blob/v0.11.23/include/uv.h
#if (UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR < 11) || \
(UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR == 11 && UV_VERSION_PATCH < 23)
static void redisLibuvTimeout(uv_timer_t *timer, int status) {
(void)status; // unused
#else
static void redisLibuvTimeout(uv_timer_t *timer) {
#endif
redisLibuvEvents *e = (redisLibuvEvents*)timer->data;
redisAsyncHandleTimeout(e->context);
}
static void redisLibuvSetTimeout(void *privdata, struct timeval tv) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
uint64_t millsec = tv.tv_sec * 1000 + tv.tv_usec / 1000.0;
if (!p->timer.data) {
// timer is uninitialized
if (uv_timer_init(p->handle.loop, &p->timer) != 0) {
return;
}
p->timer.data = p;
}
// updates the timeout if the timer has already started
// or start the timer
uv_timer_start(&p->timer, redisLibuvTimeout, millsec, 0);
}
static void redisLibuvCleanup(void *privdata) {
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
redisLibuvEvents* p = (redisLibuvEvents*)privdata;
p->context = NULL; // indicate that context might no longer exist
uv_close((uv_handle_t*)&p->handle, on_close);
p->context = NULL; // indicate that context might no longer exist
if (p->timer.data) {
uv_close((uv_handle_t*)&p->timer, on_timer_close);
}
uv_close((uv_handle_t*)&p->handle, on_handle_close);
}
static int redisLibuvAttach(redisAsyncContext* ac, uv_loop_t* loop) {
redisContext *c = &(ac->c);
redisContext *c = &(ac->c);
if (ac->ev.data != NULL) {
return REDIS_ERR;
}
if (ac->ev.data != NULL) {
return REDIS_ERR;
}
ac->ev.addRead = redisLibuvAddRead;
ac->ev.delRead = redisLibuvDelRead;
ac->ev.addWrite = redisLibuvAddWrite;
ac->ev.delWrite = redisLibuvDelWrite;
ac->ev.cleanup = redisLibuvCleanup;
ac->ev.addRead = redisLibuvAddRead;
ac->ev.delRead = redisLibuvDelRead;
ac->ev.addWrite = redisLibuvAddWrite;
ac->ev.delWrite = redisLibuvDelWrite;
ac->ev.cleanup = redisLibuvCleanup;
ac->ev.scheduleTimer = redisLibuvSetTimeout;
redisLibuvEvents* p = (redisLibuvEvents*)hi_malloc(sizeof(*p));
if (p == NULL)
return REDIS_ERR;
redisLibuvEvents* p = (redisLibuvEvents*)hi_malloc(sizeof(*p));
if (p == NULL)
return REDIS_ERR;
memset(p, 0, sizeof(*p));
memset(p, 0, sizeof(*p));
if (uv_poll_init_socket(loop, &p->handle, c->fd) != 0) {
return REDIS_ERR;
}
if (uv_poll_init_socket(loop, &p->handle, c->fd) != 0) {
return REDIS_ERR;
}
ac->ev.data = p;
p->handle.data = p;
p->context = ac;
ac->ev.data = p;
p->handle.data = p;
p->context = ac;
return REDIS_OK;
return REDIS_OK;
}
#endif
deps/hiredis/alloc.c
Vendored
+4
View File
@@ -68,6 +68,10 @@ void *hi_malloc(size_t size) {
}
void *hi_calloc(size_t nmemb, size_t size) {
/* Overflow check as the user can specify any arbitrary allocator */
if (SIZE_MAX / size < nmemb)
return NULL;
return hiredisAllocFns.callocFn(nmemb, size);
}
deps/hiredis/alloc.h
Vendored
+5
View File
@@ -32,6 +32,7 @@
#define HIREDIS_ALLOC_H
#include <stddef.h> /* for size_t */
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
@@ -59,6 +60,10 @@ static inline void *hi_malloc(size_t size) {
}
static inline void *hi_calloc(size_t nmemb, size_t size) {
/* Overflow check as the user can specify any arbitrary allocator */
if (SIZE_MAX / size < nmemb)
return NULL;
return hiredisAllocFns.callocFn(nmemb, size);
}
deps/hiredis/async.c
Vendored
+75 -53
View File
@@ -47,6 +47,11 @@
#include "async_private.h"
#ifdef NDEBUG
#undef assert
#define assert(e) (void)(e)
#endif
/* Forward declarations of hiredis.c functions */
int __redisAppendCommand(redisContext *c, const char *cmd, size_t len);
void __redisSetError(redisContext *c, int type, const char *str);
@@ -139,8 +144,8 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) {
ac->replies.head = NULL;
ac->replies.tail = NULL;
ac->sub.invalid.head = NULL;
ac->sub.invalid.tail = NULL;
ac->sub.replies.head = NULL;
ac->sub.replies.tail = NULL;
ac->sub.channels = channels;
ac->sub.patterns = patterns;
@@ -301,36 +306,28 @@ static void __redisRunPushCallback(redisAsyncContext *ac, redisReply *reply) {
static void __redisAsyncFree(redisAsyncContext *ac) {
redisContext *c = &(ac->c);
redisCallback cb;
dictIterator *it;
dictIterator it;
dictEntry *de;
/* Execute pending callbacks with NULL reply. */
while (__redisShiftCallback(&ac->replies,&cb) == REDIS_OK)
__redisRunCallback(ac,&cb,NULL);
/* Execute callbacks for invalid commands */
while (__redisShiftCallback(&ac->sub.invalid,&cb) == REDIS_OK)
while (__redisShiftCallback(&ac->sub.replies,&cb) == REDIS_OK)
__redisRunCallback(ac,&cb,NULL);
/* Run subscription callbacks with NULL reply */
if (ac->sub.channels) {
it = dictGetIterator(ac->sub.channels);
if (it != NULL) {
while ((de = dictNext(it)) != NULL)
__redisRunCallback(ac,dictGetEntryVal(de),NULL);
dictReleaseIterator(it);
}
dictInitIterator(&it,ac->sub.channels);
while ((de = dictNext(&it)) != NULL)
__redisRunCallback(ac,dictGetEntryVal(de),NULL);
dictRelease(ac->sub.channels);
}
if (ac->sub.patterns) {
it = dictGetIterator(ac->sub.patterns);
if (it != NULL) {
while ((de = dictNext(it)) != NULL)
__redisRunCallback(ac,dictGetEntryVal(de),NULL);
dictReleaseIterator(it);
}
dictInitIterator(&it,ac->sub.patterns);
while ((de = dictNext(&it)) != NULL)
__redisRunCallback(ac,dictGetEntryVal(de),NULL);
dictRelease(ac->sub.patterns);
}
@@ -420,10 +417,11 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply,
char *stype;
hisds sname;
/* Custom reply functions are not supported for pub/sub. This will fail
* very hard when they are used... */
if (reply->type == REDIS_REPLY_ARRAY || reply->type == REDIS_REPLY_PUSH) {
assert(reply->elements >= 2);
/* Match reply with the expected format of a pushed message.
* The type and number of elements (3 to 4) are specified at:
* https://redis.io/topics/pubsub#format-of-pushed-messages */
if ((reply->type == REDIS_REPLY_ARRAY && !(c->flags & REDIS_SUPPORTS_PUSH) && reply->elements >= 3) ||
reply->type == REDIS_REPLY_PUSH) {
assert(reply->element[0]->type == REDIS_REPLY_STRING);
stype = reply->element[0]->str;
pvariant = (tolower(stype[0]) == 'p') ? 1 : 0;
@@ -462,14 +460,21 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply,
/* Unset subscribed flag only when no pipelined pending subscribe. */
if (reply->element[2]->integer == 0
&& dictSize(ac->sub.channels) == 0
&& dictSize(ac->sub.patterns) == 0)
&& dictSize(ac->sub.patterns) == 0) {
c->flags &= ~REDIS_SUBSCRIBED;
/* Move ongoing regular command callbacks. */
redisCallback cb;
while (__redisShiftCallback(&ac->sub.replies,&cb) == REDIS_OK) {
__redisPushCallback(&ac->replies,&cb);
}
}
}
}
hi_sdsfree(sname);
} else {
/* Shift callback for invalid commands. */
__redisShiftCallback(&ac->sub.invalid,dstcb);
/* Shift callback for pending command in subscribed context. */
__redisShiftCallback(&ac->sub.replies,dstcb);
}
return REDIS_OK;
oom:
@@ -497,13 +502,12 @@ static int redisIsSubscribeReply(redisReply *reply) {
len = reply->element[0]->len - off;
return !strncasecmp(str, "subscribe", len) ||
!strncasecmp(str, "message", len);
!strncasecmp(str, "message", len) ||
!strncasecmp(str, "unsubscribe", len);
}
void redisProcessCallbacks(redisAsyncContext *ac) {
redisContext *c = &(ac->c);
redisCallback cb = {NULL, NULL, 0, NULL};
void *reply = NULL;
int status;
@@ -516,17 +520,14 @@ void redisProcessCallbacks(redisAsyncContext *ac) {
__redisAsyncDisconnect(ac);
return;
}
/* If monitor mode, repush callback */
if(c->flags & REDIS_MONITORING) {
__redisPushCallback(&ac->replies,&cb);
}
/* When the connection is not being disconnected, simply stop
* trying to get replies and wait for the next loop tick. */
break;
}
/* Keep track of push message support for subscribe handling */
if (redisIsPushReply(reply)) c->flags |= REDIS_SUPPORTS_PUSH;
/* Send any non-subscribe related PUSH messages to our PUSH handler
* while allowing subscribe related PUSH messages to pass through.
* This allows existing code to be backward compatible and work in
@@ -539,6 +540,7 @@ void redisProcessCallbacks(redisAsyncContext *ac) {
/* Even if the context is subscribed, pending regular
* callbacks will get a reply before pub/sub messages arrive. */
redisCallback cb = {NULL, NULL, 0, NULL};
if (__redisShiftCallback(&ac->replies,&cb) != REDIS_OK) {
/*
* A spontaneous reply in a not-subscribed context can be the error
@@ -562,15 +564,17 @@ void redisProcessCallbacks(redisAsyncContext *ac) {
__redisAsyncDisconnect(ac);
return;
}
/* No more regular callbacks and no errors, the context *must* be subscribed or monitoring. */
assert((c->flags & REDIS_SUBSCRIBED || c->flags & REDIS_MONITORING));
if(c->flags & REDIS_SUBSCRIBED)
/* No more regular callbacks and no errors, the context *must* be subscribed. */
assert(c->flags & REDIS_SUBSCRIBED);
if (c->flags & REDIS_SUBSCRIBED)
__redisGetSubscribeCallback(ac,reply,&cb);
}
if (cb.fn != NULL) {
__redisRunCallback(ac,&cb,reply);
c->reader->fn->freeObject(reply);
if (!(c->flags & REDIS_NO_AUTO_FREE_REPLIES)){
c->reader->fn->freeObject(reply);
}
/* Proceed with free'ing when redisAsyncFree() was called. */
if (c->flags & REDIS_FREEING) {
@@ -584,6 +588,11 @@ void redisProcessCallbacks(redisAsyncContext *ac) {
* doesn't know what the server will spit out over the wire. */
c->reader->fn->freeObject(reply);
}
/* If in monitor mode, repush the callback */
if (c->flags & REDIS_MONITORING) {
__redisPushCallback(&ac->replies,&cb);
}
}
/* Disconnect when there was an error reading the reply */
@@ -605,7 +614,8 @@ static int __redisAsyncHandleConnect(redisAsyncContext *ac) {
if (redisCheckConnectDone(c, &completed) == REDIS_ERR) {
/* Error! */
redisCheckSocketError(c);
if (redisCheckSocketError(c) == REDIS_ERR)
__redisAsyncCopyError(ac);
__redisAsyncHandleConnectFailure(ac);
return REDIS_ERR;
} else if (completed == 1) {
@@ -691,13 +701,22 @@ void redisAsyncHandleTimeout(redisAsyncContext *ac) {
redisContext *c = &(ac->c);
redisCallback cb;
if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) {
/* Nothing to do - just an idle timeout */
return;
if ((c->flags & REDIS_CONNECTED)) {
if (ac->replies.head == NULL && ac->sub.replies.head == NULL) {
/* Nothing to do - just an idle timeout */
return;
}
if (!ac->c.command_timeout ||
(!ac->c.command_timeout->tv_sec && !ac->c.command_timeout->tv_usec)) {
/* A belated connect timeout arriving, ignore */
return;
}
}
if (!c->err) {
__redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout");
__redisAsyncCopyError(ac);
}
if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) {
@@ -796,17 +815,19 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void
/* (P)UNSUBSCRIBE does not have its own response: every channel or
* pattern that is unsubscribed will receive a message. This means we
* should not append a callback function for this command. */
} else if(strncasecmp(cstr,"monitor\r\n",9) == 0) {
/* Set monitor flag and push callback */
c->flags |= REDIS_MONITORING;
__redisPushCallback(&ac->replies,&cb);
} else if (strncasecmp(cstr,"monitor\r\n",9) == 0) {
/* Set monitor flag and push callback */
c->flags |= REDIS_MONITORING;
if (__redisPushCallback(&ac->replies,&cb) != REDIS_OK)
goto oom;
} else {
if (c->flags & REDIS_SUBSCRIBED)
/* This will likely result in an error reply, but it needs to be
* received and passed to the callback. */
__redisPushCallback(&ac->sub.invalid,&cb);
else
__redisPushCallback(&ac->replies,&cb);
if (c->flags & REDIS_SUBSCRIBED) {
if (__redisPushCallback(&ac->sub.replies,&cb) != REDIS_OK)
goto oom;
} else {
if (__redisPushCallback(&ac->replies,&cb) != REDIS_OK)
goto oom;
}
}
__redisAppendCommand(c,cmd,len);
@@ -817,6 +838,7 @@ static int __redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void
return REDIS_OK;
oom:
__redisSetError(&(ac->c), REDIS_ERR_OOM, "Out of memory");
__redisAsyncCopyError(ac);
return REDIS_ERR;
}
@@ -846,7 +868,7 @@ int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata
int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen) {
hisds cmd;
int len;
long long len;
int status;
len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
if (len < 0)
deps/hiredis/async.h
Vendored
+1 -1
View File
@@ -102,7 +102,7 @@ typedef struct redisAsyncContext {
/* Subscription callbacks */
struct {
redisCallbackList invalid;
redisCallbackList replies;
struct dict *channels;
struct dict *patterns;
} sub;
deps/hiredis/dict.c
Vendored
+1 -10
View File
@@ -267,16 +267,11 @@ static dictEntry *dictFind(dict *ht, const void *key) {
return NULL;
}
static dictIterator *dictGetIterator(dict *ht) {
dictIterator *iter = hi_malloc(sizeof(*iter));
if (iter == NULL)
return NULL;
static void dictInitIterator(dictIterator *iter, dict *ht) {
iter->ht = ht;
iter->index = -1;
iter->entry = NULL;
iter->nextEntry = NULL;
return iter;
}
static dictEntry *dictNext(dictIterator *iter) {
@@ -299,10 +294,6 @@ static dictEntry *dictNext(dictIterator *iter) {
return NULL;
}
static void dictReleaseIterator(dictIterator *iter) {
hi_free(iter);
}
/* ------------------------- private functions ------------------------------ */
/* Expand the hash table if needed */
deps/hiredis/dict.h
Vendored
+1 -2
View File
@@ -119,8 +119,7 @@ static int dictReplace(dict *ht, void *key, void *val);
static int dictDelete(dict *ht, const void *key);
static void dictRelease(dict *ht);
static dictEntry * dictFind(dict *ht, const void *key);
static dictIterator *dictGetIterator(dict *ht);
static void dictInitIterator(dictIterator *iter, dict *ht);
static dictEntry *dictNext(dictIterator *iter);
static void dictReleaseIterator(dictIterator *iter);
#endif /* __DICT_H */
deps/hiredis/examples/CMakeLists.txt
Vendored
+1 -1
View File
@@ -21,7 +21,7 @@ ENDIF()
FIND_PATH(LIBEVENT event.h)
if (LIBEVENT)
ADD_EXECUTABLE(example-libevent example-libevent)
ADD_EXECUTABLE(example-libevent example-libevent.c)
TARGET_LINK_LIBRARIES(example-libevent hiredis event)
ENDIF()
deps/hiredis/examples/example-libuv.c
Vendored
+31 -6
View File
@@ -7,18 +7,33 @@
#include <async.h>
#include <adapters/libuv.h>
void debugCallback(redisAsyncContext *c, void *r, void *privdata) {
(void)privdata; //unused
redisReply *reply = r;
if (reply == NULL) {
/* The DEBUG SLEEP command will almost always fail, because we have set a 1 second timeout */
printf("`DEBUG SLEEP` error: %s\n", c->errstr ? c->errstr : "unknown error");
return;
}
/* Disconnect after receiving the reply of DEBUG SLEEP (which will not)*/
redisAsyncDisconnect(c);
}
void getCallback(redisAsyncContext *c, void *r, void *privdata) {
redisReply *reply = r;
if (reply == NULL) return;
printf("argv[%s]: %s\n", (char*)privdata, reply->str);
if (reply == NULL) {
printf("`GET key` error: %s\n", c->errstr ? c->errstr : "unknown error");
return;
}
printf("`GET key` result: argv[%s]: %s\n", (char*)privdata, reply->str);
/* Disconnect after receiving the reply to GET */
redisAsyncDisconnect(c);
/* start another request that demonstrate timeout */
redisAsyncCommand(c, debugCallback, NULL, "DEBUG SLEEP %f", 1.5);
}
void connectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
printf("connect error: %s\n", c->errstr);
return;
}
printf("Connected...\n");
@@ -26,7 +41,7 @@ void connectCallback(const redisAsyncContext *c, int status) {
void disconnectCallback(const redisAsyncContext *c, int status) {
if (status != REDIS_OK) {
printf("Error: %s\n", c->errstr);
printf("disconnect because of error: %s\n", c->errstr);
return;
}
printf("Disconnected...\n");
@@ -49,8 +64,18 @@ int main (int argc, char **argv) {
redisLibuvAttach(c,loop);
redisAsyncSetConnectCallback(c,connectCallback);
redisAsyncSetDisconnectCallback(c,disconnectCallback);
redisAsyncSetTimeout(c, (struct timeval){ .tv_sec = 1, .tv_usec = 0});
/*
In this demo, we first `set key`, then `get key` to demonstrate the basic usage of libuv adapter.
Then in `getCallback`, we start a `debug sleep` command to create 1.5 second long request.
Because we have set a 1 second timeout to the connection, the command will always fail with a
timeout error, which is shown in the `debugCallback`.
*/
redisAsyncCommand(c, NULL, NULL, "SET key %b", argv[argc-1], strlen(argv[argc-1]));
redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key");
uv_run(loop, UV_RUN_DEFAULT);
return 0;
}
deps/hiredis/examples/example-push.c
Vendored
-1
View File
@@ -31,7 +31,6 @@
#include <stdlib.h>
#include <string.h>
#include <hiredis.h>
#include <win32.h>
#define KEY_COUNT 5
deps/hiredis/examples/example-ssl.c
Vendored
+4 -1
View File
@@ -4,7 +4,10 @@
#include <hiredis.h>
#include <hiredis_ssl.h>
#include <win32.h>
#ifdef _MSC_VER
#include <winsock2.h> /* For struct timeval */
#endif
int main(int argc, char **argv) {
unsigned int j;
deps/hiredis/examples/example.c
Vendored
+4 -1
View File
@@ -2,7 +2,10 @@
#include <stdlib.h>
#include <string.h>
#include <hiredis.h>
#include <win32.h>
#ifdef _MSC_VER
#include <winsock2.h> /* For struct timeval */
#endif
int main(int argc, char **argv) {
unsigned int j, isunix = 0;
deps/hiredis/fmacros.h
Vendored
+2
View File
@@ -1,8 +1,10 @@
#ifndef __HIREDIS_FMACRO_H
#define __HIREDIS_FMACRO_H
#ifndef _AIX
#define _XOPEN_SOURCE 600
#define _POSIX_C_SOURCE 200112L
#endif
#if defined(__APPLE__) && defined(__MACH__)
/* Enable TCP_KEEPALIVE */
deps/hiredis/fuzzing/format_command_fuzzer.c
Vendored
+57
View File
@@ -0,0 +1,57 @@
/*
* Copyright (c) 2020, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2020, Pieter Noordhuis <pcnoordhuis at gmail dot com>
* Copyright (c) 2020, Matt Stancliff <matt at genges dot com>,
* Jan-Erik Rediger <janerik at fnordig dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdlib.h>
#include <string.h>
#include "hiredis.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
char *new_str, *cmd;
if (size < 3)
return 0;
new_str = malloc(size+1);
if (new_str == NULL)
return 0;
memcpy(new_str, data, size);
new_str[size] = '\0';
redisFormatCommand(&cmd, new_str);
if (cmd != NULL)
hi_free(cmd);
free(new_str);
return 0;
}
deps/hiredis/hiredis.c
Vendored
+53 -39
View File
@@ -96,6 +96,8 @@ void freeReplyObject(void *reply) {
switch(r->type) {
case REDIS_REPLY_INTEGER:
case REDIS_REPLY_NIL:
case REDIS_REPLY_BOOL:
break; /* Nothing to free */
case REDIS_REPLY_ARRAY:
case REDIS_REPLY_MAP:
@@ -112,6 +114,7 @@ void freeReplyObject(void *reply) {
case REDIS_REPLY_STRING:
case REDIS_REPLY_DOUBLE:
case REDIS_REPLY_VERB:
case REDIS_REPLY_BIGNUM:
hi_free(r->str);
break;
}
@@ -129,7 +132,8 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len
assert(task->type == REDIS_REPLY_ERROR ||
task->type == REDIS_REPLY_STATUS ||
task->type == REDIS_REPLY_STRING ||
task->type == REDIS_REPLY_VERB);
task->type == REDIS_REPLY_VERB ||
task->type == REDIS_REPLY_BIGNUM);
/* Copy string value */
if (task->type == REDIS_REPLY_VERB) {
@@ -235,12 +239,14 @@ static void *createDoubleObject(const redisReadTask *task, double value, char *s
* decimal string conversion artifacts. */
memcpy(r->str, str, len);
r->str[len] = '\0';
r->len = len;
if (task->parent) {
parent = task->parent->obj;
assert(parent->type == REDIS_REPLY_ARRAY ||
parent->type == REDIS_REPLY_MAP ||
parent->type == REDIS_REPLY_SET);
parent->type == REDIS_REPLY_SET ||
parent->type == REDIS_REPLY_PUSH);
parent->element[task->idx] = r;
}
return r;
@@ -277,7 +283,8 @@ static void *createBoolObject(const redisReadTask *task, int bval) {
parent = task->parent->obj;
assert(parent->type == REDIS_REPLY_ARRAY ||
parent->type == REDIS_REPLY_MAP ||
parent->type == REDIS_REPLY_SET);
parent->type == REDIS_REPLY_SET ||
parent->type == REDIS_REPLY_PUSH);
parent->element[task->idx] = r;
}
return r;
@@ -565,13 +572,12 @@ int redisFormatCommand(char **target, const char *format, ...) {
* lengths. If the latter is set to NULL, strlen will be used to compute the
* argument lengths.
*/
int redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv,
const size_t *argvlen)
long long redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv,
const size_t *argvlen)
{
hisds cmd, aux;
unsigned long long totlen;
unsigned long long totlen, len;
int j;
size_t len;
/* Abort on a NULL target */
if (target == NULL)
@@ -602,7 +608,7 @@ int redisFormatSdsCommandArgv(hisds *target, int argc, const char **argv,
cmd = hi_sdscatfmt(cmd, "*%i\r\n", argc);
for (j=0; j < argc; j++) {
len = argvlen ? argvlen[j] : strlen(argv[j]);
cmd = hi_sdscatfmt(cmd, "$%u\r\n", len);
cmd = hi_sdscatfmt(cmd, "$%U\r\n", len);
cmd = hi_sdscatlen(cmd, argv[j], len);
cmd = hi_sdscatlen(cmd, "\r\n", sizeof("\r\n")-1);
}
@@ -622,11 +628,11 @@ void redisFreeSdsCommand(hisds cmd) {
* lengths. If the latter is set to NULL, strlen will be used to compute the
* argument lengths.
*/
int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) {
long long redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen) {
char *cmd = NULL; /* final command */
int pos; /* position in final command */
size_t len;
int totlen, j;
size_t pos; /* position in final command */
size_t len, totlen;
int j;
/* Abort on a NULL target */
if (target == NULL)
@@ -797,6 +803,9 @@ redisContext *redisConnectWithOptions(const redisOptions *options) {
if (options->options & REDIS_OPT_NOAUTOFREE) {
c->flags |= REDIS_NO_AUTO_FREE;
}
if (options->options & REDIS_OPT_NOAUTOFREEREPLIES) {
c->flags |= REDIS_NO_AUTO_FREE_REPLIES;
}
/* Set any user supplied RESP3 PUSH handler or use freeReplyObject
* as a default unless specifically flagged that we don't want one. */
@@ -825,7 +834,7 @@ redisContext *redisConnectWithOptions(const redisOptions *options) {
c->fd = options->endpoint.fd;
c->flags |= REDIS_CONNECTED;
} else {
// Unknown type - FIXME - FREE
redisFree(c);
return NULL;
}
@@ -939,13 +948,11 @@ int redisBufferRead(redisContext *c) {
return REDIS_ERR;
nread = c->funcs->read(c, buf, sizeof(buf));
if (nread > 0) {
if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) {
__redisSetError(c, c->reader->err, c->reader->errstr);
return REDIS_ERR;
} else {
}
} else if (nread < 0) {
if (nread < 0) {
return REDIS_ERR;
}
if (nread > 0 && redisReaderFeed(c->reader, buf, nread) != REDIS_OK) {
__redisSetError(c, c->reader->err, c->reader->errstr);
return REDIS_ERR;
}
return REDIS_OK;
@@ -989,17 +996,6 @@ oom:
return REDIS_ERR;
}
/* Internal helper function to try and get a reply from the reader,
* or set an error in the context otherwise. */
int redisGetReplyFromReader(redisContext *c, void **reply) {
if (redisReaderGetReply(c->reader,reply) == REDIS_ERR) {
__redisSetError(c,c->reader->err,c->reader->errstr);
return REDIS_ERR;
}
return REDIS_OK;
}
/* Internal helper that returns 1 if the reply was a RESP3 PUSH
* message and we handled it with a user-provided callback. */
static int redisHandledPushReply(redisContext *c, void *reply) {
@@ -1011,12 +1007,34 @@ static int redisHandledPushReply(redisContext *c, void *reply) {
return 0;
}
/* Get a reply from our reader or set an error in the context. */
int redisGetReplyFromReader(redisContext *c, void **reply) {
if (redisReaderGetReply(c->reader, reply) == REDIS_ERR) {
__redisSetError(c,c->reader->err,c->reader->errstr);
return REDIS_ERR;
}
return REDIS_OK;
}
/* Internal helper to get the next reply from our reader while handling
* any PUSH messages we encounter along the way. This is separate from
* redisGetReplyFromReader so as to not change its behavior. */
static int redisNextInBandReplyFromReader(redisContext *c, void **reply) {
do {
if (redisGetReplyFromReader(c, reply) == REDIS_ERR)
return REDIS_ERR;
} while (redisHandledPushReply(c, *reply));
return REDIS_OK;
}
int redisGetReply(redisContext *c, void **reply) {
int wdone = 0;
void *aux = NULL;
/* Try to read pending replies */
if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
if (redisNextInBandReplyFromReader(c,&aux) == REDIS_ERR)
return REDIS_ERR;
/* For the blocking context, flush output buffer and read reply */
@@ -1032,12 +1050,8 @@ int redisGetReply(redisContext *c, void **reply) {
if (redisBufferRead(c) == REDIS_ERR)
return REDIS_ERR;
/* We loop here in case the user has specified a RESP3
* PUSH handler (e.g. for client tracking). */
do {
if (redisGetReplyFromReader(c,&aux) == REDIS_ERR)
return REDIS_ERR;
} while (redisHandledPushReply(c, aux));
if (redisNextInBandReplyFromReader(c,&aux) == REDIS_ERR)
return REDIS_ERR;
} while (aux == NULL);
}
@@ -1114,7 +1128,7 @@ int redisAppendCommand(redisContext *c, const char *format, ...) {
int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen) {
hisds cmd;
int len;
long long len;
len = redisFormatSdsCommandArgv(&cmd,argc,argv,argvlen);
if (len == -1) {
deps/hiredis/hiredis.h
Vendored
+18 -6
View File
@@ -47,8 +47,8 @@ typedef long long ssize_t;
#define HIREDIS_MAJOR 1
#define HIREDIS_MINOR 0
#define HIREDIS_PATCH 0
#define HIREDIS_SONAME 1.0.0
#define HIREDIS_PATCH 3
#define HIREDIS_SONAME 1.0.3-dev
/* Connection type can be blocking or non-blocking and is set in the
* least significant bit of the flags field in redisContext. */
@@ -80,12 +80,18 @@ typedef long long ssize_t;
/* Flag that is set when we should set SO_REUSEADDR before calling bind() */
#define REDIS_REUSEADDR 0x80
/* Flag that is set when the async connection supports push replies. */
#define REDIS_SUPPORTS_PUSH 0x100
/**
* Flag that indicates the user does not want the context to
* be automatically freed upon error
*/
#define REDIS_NO_AUTO_FREE 0x200
/* Flag that indicates the user does not want replies to be automatically freed */
#define REDIS_NO_AUTO_FREE_REPLIES 0x400
#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */
/* number of times we retry to connect in the case of EADDRNOTAVAIL and
@@ -112,7 +118,8 @@ typedef struct redisReply {
double dval; /* The double when type is REDIS_REPLY_DOUBLE */
size_t len; /* Length of string */
char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING
REDIS_REPLY_VERB, and REDIS_REPLY_DOUBLE (in additional to dval). */
REDIS_REPLY_VERB, REDIS_REPLY_DOUBLE (in additional to dval),
and REDIS_REPLY_BIGNUM. */
char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null
terminated 3 character content type, such as "txt". */
size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */
@@ -127,8 +134,8 @@ void freeReplyObject(void *reply);
/* Functions to format a command according to the protocol. */
int redisvFormatCommand(char **target, const char *format, va_list ap);
int redisFormatCommand(char **target, const char *format, ...);
int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen);
int redisFormatSdsCommandArgv(hisds *target, int argc, const char ** argv, const size_t *argvlen);
long long redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen);
long long redisFormatSdsCommandArgv(hisds *target, int argc, const char ** argv, const size_t *argvlen);
void redisFreeCommand(char *cmd);
void redisFreeSdsCommand(hisds cmd);
@@ -152,6 +159,11 @@ struct redisSsl;
/* Don't automatically intercept and free RESP3 PUSH replies. */
#define REDIS_OPT_NO_PUSH_AUTOFREE 0x08
/**
* Don't automatically free replies
*/
#define REDIS_OPT_NOAUTOFREEREPLIES 0x10
/* In Unix systems a file descriptor is a regular signed int, with -1
* representing an invalid descriptor. In Windows it is a SOCKET
* (32- or 64-bit unsigned integer depending on the architecture), where
@@ -255,7 +267,7 @@ typedef struct redisContext {
} unix_sock;
/* For non-blocking connect */
struct sockadr *saddr;
struct sockaddr *saddr;
size_t addrlen;
/* Optional data and corresponding destructor users can use to provide
deps/hiredis/hiredis.targets
Vendored
+11
View File
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemDefinitionGroup>
<ClCompile>
<AdditionalIncludeDirectories>$(MSBuildThisFileDirectory)\..\..\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<AdditionalLibraryDirectories>$(MSBuildThisFileDirectory)\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Link>
</ItemDefinitionGroup>
</Project>
deps/hiredis/hiredis_ssl.h
Vendored
+3 -1
View File
@@ -56,7 +56,9 @@ typedef enum {
REDIS_SSL_CTX_CERT_KEY_REQUIRED, /* Client cert and key must both be specified or skipped */
REDIS_SSL_CTX_CA_CERT_LOAD_FAILED, /* Failed to load CA Certificate or CA Path */
REDIS_SSL_CTX_CLIENT_CERT_LOAD_FAILED, /* Failed to load client certificate */
REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED /* Failed to load private key */
REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED, /* Failed to load private key */
REDIS_SSL_CTX_OS_CERTSTORE_OPEN_FAILED, /* Failed to open system certificate store */
REDIS_SSL_CTX_OS_CERT_ADD_FAILED /* Failed to add CA certificates obtained from system to the SSL context */
} redisSSLContextError;
/**
deps/hiredis/read.c
Vendored
+94 -48
View File
@@ -123,29 +123,28 @@ static char *readBytes(redisReader *r, unsigned int bytes) {
/* Find pointer to \r\n. */
static char *seekNewline(char *s, size_t len) {
int pos = 0;
int _len = len-1;
char *ret;
/* Position should be < len-1 because the character at "pos" should be
* followed by a \n. Note that strchr cannot be used because it doesn't
* allow to search a limited length and the buffer that is being searched
* might not have a trailing NULL character. */
while (pos < _len) {
while(pos < _len && s[pos] != '\r') pos++;
if (pos==_len) {
/* Not found. */
return NULL;
} else {
if (s[pos+1] == '\n') {
/* Found. */
return s+pos;
} else {
/* Continue searching. */
pos++;
}
/* We cannot match with fewer than 2 bytes */
if (len < 2)
return NULL;
/* Search up to len - 1 characters */
len--;
/* Look for the \r */
while ((ret = memchr(s, '\r', len)) != NULL) {
if (ret[1] == '\n') {
/* Found. */
break;
}
/* Continue searching. */
ret++;
len -= ret - s;
s = ret;
}
return NULL;
return ret;
}
/* Convert a string into a long long. Returns REDIS_OK if the string could be
@@ -274,60 +273,104 @@ static int processLineItem(redisReader *r) {
if ((p = readLine(r,&len)) != NULL) {
if (cur->type == REDIS_REPLY_INTEGER) {
long long v;
if (string2ll(p, len, &v) == REDIS_ERR) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"Bad integer value");
return REDIS_ERR;
}
if (r->fn && r->fn->createInteger) {
long long v;
if (string2ll(p, len, &v) == REDIS_ERR) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"Bad integer value");
return REDIS_ERR;
}
obj = r->fn->createInteger(cur,v);
} else {
obj = (void*)REDIS_REPLY_INTEGER;
}
} else if (cur->type == REDIS_REPLY_DOUBLE) {
if (r->fn && r->fn->createDouble) {
char buf[326], *eptr;
double d;
char buf[326], *eptr;
double d;
if ((size_t)len >= sizeof(buf)) {
if ((size_t)len >= sizeof(buf)) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"Double value is too large");
return REDIS_ERR;
}
memcpy(buf,p,len);
buf[len] = '\0';
if (len == 3 && strcasecmp(buf,"inf") == 0) {
d = INFINITY; /* Positive infinite. */
} else if (len == 4 && strcasecmp(buf,"-inf") == 0) {
d = -INFINITY; /* Negative infinite. */
} else {
d = strtod((char*)buf,&eptr);
/* RESP3 only allows "inf", "-inf", and finite values, while
* strtod() allows other variations on infinity, NaN,
* etc. We explicity handle our two allowed infinite cases
* above, so strtod() should only result in finite values. */
if (buf[0] == '\0' || eptr != &buf[len] || !isfinite(d)) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"Double value is too large");
"Bad double value");
return REDIS_ERR;
}
}
memcpy(buf,p,len);
buf[len] = '\0';
if (strcasecmp(buf,",inf") == 0) {
d = INFINITY; /* Positive infinite. */
} else if (strcasecmp(buf,",-inf") == 0) {
d = -INFINITY; /* Negative infinite. */
} else {
d = strtod((char*)buf,&eptr);
if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"Bad double value");
return REDIS_ERR;
}
}
if (r->fn && r->fn->createDouble) {
obj = r->fn->createDouble(cur,d,buf,len);
} else {
obj = (void*)REDIS_REPLY_DOUBLE;
}
} else if (cur->type == REDIS_REPLY_NIL) {
if (len != 0) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"Bad nil value");
return REDIS_ERR;
}
if (r->fn && r->fn->createNil)
obj = r->fn->createNil(cur);
else
obj = (void*)REDIS_REPLY_NIL;
} else if (cur->type == REDIS_REPLY_BOOL) {
int bval = p[0] == 't' || p[0] == 'T';
int bval;
if (len != 1 || !strchr("tTfF", p[0])) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"Bad bool value");
return REDIS_ERR;
}
bval = p[0] == 't' || p[0] == 'T';
if (r->fn && r->fn->createBool)
obj = r->fn->createBool(cur,bval);
else
obj = (void*)REDIS_REPLY_BOOL;
} else if (cur->type == REDIS_REPLY_BIGNUM) {
/* Ensure all characters are decimal digits (with possible leading
* minus sign). */
for (int i = 0; i < len; i++) {
/* XXX Consider: Allow leading '+'? Error on leading '0's? */
if (i == 0 && p[0] == '-') continue;
if (p[i] < '0' || p[i] > '9') {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"Bad bignum value");
return REDIS_ERR;
}
}
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,p,len);
else
obj = (void*)REDIS_REPLY_BIGNUM;
} else {
/* Type will be error or status. */
for (int i = 0; i < len; i++) {
if (p[i] == '\r' || p[i] == '\n') {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"Bad simple string value");
return REDIS_ERR;
}
}
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,p,len);
else
@@ -453,7 +496,6 @@ static int processAggregateItem(redisReader *r) {
long long elements;
int root = 0, len;
/* Set error for nested multi bulks with depth > 7 */
if (r->ridx == r->tasks - 1) {
if (redisReaderGrow(r) == REDIS_ERR)
return REDIS_ERR;
@@ -569,6 +611,9 @@ static int processItem(redisReader *r) {
case '>':
cur->type = REDIS_REPLY_PUSH;
break;
case '(':
cur->type = REDIS_REPLY_BIGNUM;
break;
default:
__redisReaderSetErrorProtocolByte(r,*p);
return REDIS_ERR;
@@ -587,6 +632,7 @@ static int processItem(redisReader *r) {
case REDIS_REPLY_DOUBLE:
case REDIS_REPLY_NIL:
case REDIS_REPLY_BOOL:
case REDIS_REPLY_BIGNUM:
return processLineItem(r);
case REDIS_REPLY_STRING:
case REDIS_REPLY_VERB:
deps/hiredis/sds.c
Vendored
+2 -2
View File
@@ -72,7 +72,7 @@ static inline char hi_sdsReqType(size_t string_size) {
* and 'initlen'.
* If NULL is used for 'init' the string is initialized with zero bytes.
*
* The string is always null-termined (all the hisds strings are, always) so
* The string is always null-terminated (all the hisds strings are, always) so
* even if you create an hisds string with:
*
* mystring = hi_sdsnewlen("abc",3);
@@ -415,7 +415,7 @@ hisds hi_sdscpylen(hisds s, const char *t, size_t len) {
return s;
}
/* Like hi_sdscpylen() but 't' must be a null-termined string so that the length
/* Like hi_sdscpylen() but 't' must be a null-terminated string so that the length
* of the string is obtained with strlen(). */
hisds hi_sdscpy(hisds s, const char *t) {
return hi_sdscpylen(s, t, strlen(t));
deps/hiredis/ssl.c
Vendored
+44 -1
View File
@@ -38,6 +38,7 @@
#include <string.h>
#ifdef _WIN32
#include <windows.h>
#include <wincrypt.h>
#else
#include <pthread.h>
#endif
@@ -182,6 +183,10 @@ const char *redisSSLContextGetError(redisSSLContextError error)
return "Failed to load client certificate";
case REDIS_SSL_CTX_PRIVATE_KEY_LOAD_FAILED:
return "Failed to load private key";
case REDIS_SSL_CTX_OS_CERTSTORE_OPEN_FAILED:
return "Failed to open system certificate store";
case REDIS_SSL_CTX_OS_CERT_ADD_FAILED:
return "Failed to add CA certificates obtained from system to the SSL context";
default:
return "Unknown error code";
}
@@ -214,6 +219,11 @@ redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *
const char *cert_filename, const char *private_key_filename,
const char *server_name, redisSSLContextError *error)
{
#ifdef _WIN32
HCERTSTORE win_store = NULL;
PCCERT_CONTEXT win_ctx = NULL;
#endif
redisSSLContext *ctx = hi_calloc(1, sizeof(redisSSLContext));
if (ctx == NULL)
goto error;
@@ -234,6 +244,31 @@ redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *
}
if (capath || cacert_filename) {
#ifdef _WIN32
if (0 == strcmp(cacert_filename, "wincert")) {
win_store = CertOpenSystemStore(NULL, "Root");
if (!win_store) {
if (error) *error = REDIS_SSL_CTX_OS_CERTSTORE_OPEN_FAILED;
goto error;
}
X509_STORE* store = SSL_CTX_get_cert_store(ctx->ssl_ctx);
while (win_ctx = CertEnumCertificatesInStore(win_store, win_ctx)) {
X509* x509 = NULL;
x509 = d2i_X509(NULL, (const unsigned char**)&win_ctx->pbCertEncoded, win_ctx->cbCertEncoded);
if (x509) {
if ((1 != X509_STORE_add_cert(store, x509)) ||
(1 != SSL_CTX_add_client_CA(ctx->ssl_ctx, x509)))
{
if (error) *error = REDIS_SSL_CTX_OS_CERT_ADD_FAILED;
goto error;
}
X509_free(x509);
}
}
CertFreeCertificateContext(win_ctx);
CertCloseStore(win_store, 0);
} else
#endif
if (!SSL_CTX_load_verify_locations(ctx->ssl_ctx, cacert_filename, capath)) {
if (error) *error = REDIS_SSL_CTX_CA_CERT_LOAD_FAILED;
goto error;
@@ -257,6 +292,10 @@ redisSSLContext *redisCreateSSLContext(const char *cacert_filename, const char *
return ctx;
error:
#ifdef _WIN32
CertFreeCertificateContext(win_ctx);
CertCloseStore(win_store, 0);
#endif
redisFreeSSLContext(ctx);
return NULL;
}
@@ -353,7 +392,11 @@ int redisInitiateSSLWithContext(redisContext *c, redisSSLContext *redis_ssl_ctx)
}
}
return redisSSLConnect(c, ssl);
if (redisSSLConnect(c, ssl) != REDIS_OK) {
goto error;
}
return REDIS_OK;
error:
if (ssl)
deps/hiredis/test.c
Vendored
+630
View File
@@ -11,12 +11,17 @@
#include <signal.h>
#include <errno.h>
#include <limits.h>
#include <math.h>
#include "hiredis.h"
#include "async.h"
#ifdef HIREDIS_TEST_SSL
#include "hiredis_ssl.h"
#endif
#ifdef HIREDIS_TEST_ASYNC
#include "adapters/libevent.h"
#include <event2/event.h>
#endif
#include "net.h"
#include "win32.h"
@@ -58,6 +63,8 @@ struct pushCounters {
int str;
};
static int insecure_calloc_calls;
#ifdef HIREDIS_TEST_SSL
redisSSLContext *_ssl_ctx = NULL;
#endif
@@ -498,6 +505,20 @@ static void test_reply_reader(void) {
freeReplyObject(reply);
redisReaderFree(reader);
test("Multi-bulk never overflows regardless of maxelements: ");
size_t bad_mbulk_len = (SIZE_MAX / sizeof(void *)) + 3;
char bad_mbulk_reply[100];
snprintf(bad_mbulk_reply, sizeof(bad_mbulk_reply), "*%llu\r\n+asdf\r\n",
(unsigned long long) bad_mbulk_len);
reader = redisReaderCreate();
reader->maxelements = 0; /* Don't rely on default limit */
redisReaderFeed(reader, bad_mbulk_reply, strlen(bad_mbulk_reply));
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR && strcasecmp(reader->errstr, "Out of memory") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
#if LLONG_MAX > SIZE_MAX
test("Set error when array > SIZE_MAX: ");
reader = redisReaderCreate();
@@ -583,6 +604,147 @@ static void test_reply_reader(void) {
((redisReply*)reply)->element[1]->integer == 42);
freeReplyObject(reply);
redisReaderFree(reader);
test("Can parse RESP3 doubles: ");
reader = redisReaderCreate();
redisReaderFeed(reader, ",3.14159265358979323846\r\n",25);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_DOUBLE &&
fabs(((redisReply*)reply)->dval - 3.14159265358979323846) < 0.00000001 &&
((redisReply*)reply)->len == 22 &&
strcmp(((redisReply*)reply)->str, "3.14159265358979323846") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Set error on invalid RESP3 double: ");
reader = redisReaderCreate();
redisReaderFeed(reader, ",3.14159\000265358979323846\r\n",26);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Bad double value") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Correctly parses RESP3 double INFINITY: ");
reader = redisReaderCreate();
redisReaderFeed(reader, ",inf\r\n",6);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_DOUBLE &&
isinf(((redisReply*)reply)->dval) &&
((redisReply*)reply)->dval > 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Set error when RESP3 double is NaN: ");
reader = redisReaderCreate();
redisReaderFeed(reader, ",nan\r\n",6);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Bad double value") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Can parse RESP3 nil: ");
reader = redisReaderCreate();
redisReaderFeed(reader, "_\r\n",3);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_NIL);
freeReplyObject(reply);
redisReaderFree(reader);
test("Set error on invalid RESP3 nil: ");
reader = redisReaderCreate();
redisReaderFeed(reader, "_nil\r\n",6);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Bad nil value") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Can parse RESP3 bool (true): ");
reader = redisReaderCreate();
redisReaderFeed(reader, "#t\r\n",4);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_BOOL &&
((redisReply*)reply)->integer);
freeReplyObject(reply);
redisReaderFree(reader);
test("Can parse RESP3 bool (false): ");
reader = redisReaderCreate();
redisReaderFeed(reader, "#f\r\n",4);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_BOOL &&
!((redisReply*)reply)->integer);
freeReplyObject(reply);
redisReaderFree(reader);
test("Set error on invalid RESP3 bool: ");
reader = redisReaderCreate();
redisReaderFeed(reader, "#foobar\r\n",9);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_ERR &&
strcasecmp(reader->errstr,"Bad bool value") == 0);
freeReplyObject(reply);
redisReaderFree(reader);
test("Can parse RESP3 map: ");
reader = redisReaderCreate();
redisReaderFeed(reader, "%2\r\n+first\r\n:123\r\n$6\r\nsecond\r\n#t\r\n",34);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_MAP &&
((redisReply*)reply)->elements == 4 &&
((redisReply*)reply)->element[0]->type == REDIS_REPLY_STATUS &&
((redisReply*)reply)->element[0]->len == 5 &&
!strcmp(((redisReply*)reply)->element[0]->str,"first") &&
((redisReply*)reply)->element[1]->type == REDIS_REPLY_INTEGER &&
((redisReply*)reply)->element[1]->integer == 123 &&
((redisReply*)reply)->element[2]->type == REDIS_REPLY_STRING &&
((redisReply*)reply)->element[2]->len == 6 &&
!strcmp(((redisReply*)reply)->element[2]->str,"second") &&
((redisReply*)reply)->element[3]->type == REDIS_REPLY_BOOL &&
((redisReply*)reply)->element[3]->integer);
freeReplyObject(reply);
redisReaderFree(reader);
test("Can parse RESP3 set: ");
reader = redisReaderCreate();
redisReaderFeed(reader, "~5\r\n+orange\r\n$5\r\napple\r\n#f\r\n:100\r\n:999\r\n",40);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_SET &&
((redisReply*)reply)->elements == 5 &&
((redisReply*)reply)->element[0]->type == REDIS_REPLY_STATUS &&
((redisReply*)reply)->element[0]->len == 6 &&
!strcmp(((redisReply*)reply)->element[0]->str,"orange") &&
((redisReply*)reply)->element[1]->type == REDIS_REPLY_STRING &&
((redisReply*)reply)->element[1]->len == 5 &&
!strcmp(((redisReply*)reply)->element[1]->str,"apple") &&
((redisReply*)reply)->element[2]->type == REDIS_REPLY_BOOL &&
!((redisReply*)reply)->element[2]->integer &&
((redisReply*)reply)->element[3]->type == REDIS_REPLY_INTEGER &&
((redisReply*)reply)->element[3]->integer == 100 &&
((redisReply*)reply)->element[4]->type == REDIS_REPLY_INTEGER &&
((redisReply*)reply)->element[4]->integer == 999);
freeReplyObject(reply);
redisReaderFree(reader);
test("Can parse RESP3 bignum: ");
reader = redisReaderCreate();
redisReaderFeed(reader,"(3492890328409238509324850943850943825024385\r\n",46);
ret = redisReaderGetReply(reader,&reply);
test_cond(ret == REDIS_OK &&
((redisReply*)reply)->type == REDIS_REPLY_BIGNUM &&
((redisReply*)reply)->len == 43 &&
!strcmp(((redisReply*)reply)->str,"3492890328409238509324850943850943825024385"));
freeReplyObject(reply);
redisReaderFree(reader);
}
static void test_free_null(void) {
@@ -609,6 +771,13 @@ static void *hi_calloc_fail(size_t nmemb, size_t size) {
return NULL;
}
static void *hi_calloc_insecure(size_t nmemb, size_t size) {
(void)nmemb;
(void)size;
insecure_calloc_calls++;
return (void*)0xdeadc0de;
}
static void *hi_realloc_fail(void *ptr, size_t size) {
(void)ptr;
(void)size;
@@ -616,6 +785,8 @@ static void *hi_realloc_fail(void *ptr, size_t size) {
}
static void test_allocator_injection(void) {
void *ptr;
hiredisAllocFuncs ha = {
.mallocFn = hi_malloc_fail,
.callocFn = hi_calloc_fail,
@@ -635,6 +806,13 @@ static void test_allocator_injection(void) {
redisReader *reader = redisReaderCreate();
test_cond(reader == NULL);
/* Make sure hiredis itself protects against a non-overflow checking calloc */
test("hiredis calloc wrapper protects against overflow: ");
ha.callocFn = hi_calloc_insecure;
hiredisSetAllocators(&ha);
ptr = hi_calloc((SIZE_MAX / sizeof(void*)) + 3, sizeof(void*));
test_cond(ptr == NULL && insecure_calloc_calls == 0);
// Return allocators to default
hiredisResetAllocators();
}
@@ -1269,6 +1447,440 @@ static void test_throughput(struct config config) {
// redisFree(c);
// }
#ifdef HIREDIS_TEST_ASYNC
struct event_base *base;
typedef struct TestState {
redisOptions *options;
int checkpoint;
int resp3;
int disconnect;
} TestState;
/* Helper to disconnect and stop event loop */
void async_disconnect(redisAsyncContext *ac) {
redisAsyncDisconnect(ac);
event_base_loopbreak(base);
}
/* Testcase timeout, will trigger a failure */
void timeout_cb(int fd, short event, void *arg) {
(void) fd; (void) event; (void) arg;
printf("Timeout in async testing!\n");
exit(1);
}
/* Unexpected call, will trigger a failure */
void unexpected_cb(redisAsyncContext *ac, void *r, void *privdata) {
(void) ac; (void) r;
printf("Unexpected call: %s\n",(char*)privdata);
exit(1);
}
/* Helper function to publish a message via own client. */
void publish_msg(redisOptions *options, const char* channel, const char* msg) {
redisContext *c = redisConnectWithOptions(options);
assert(c != NULL);
redisReply *reply = redisCommand(c,"PUBLISH %s %s",channel,msg);
assert(reply->type == REDIS_REPLY_INTEGER && reply->integer == 1);
freeReplyObject(reply);
disconnect(c, 0);
}
/* Expect a reply of type INTEGER */
void integer_cb(redisAsyncContext *ac, void *r, void *privdata) {
redisReply *reply = r;
TestState *state = privdata;
assert(reply != NULL && reply->type == REDIS_REPLY_INTEGER);
state->checkpoint++;
if (state->disconnect) async_disconnect(ac);
}
/* Subscribe callback for test_pubsub_handling and test_pubsub_handling_resp3:
* - a published message triggers an unsubscribe
* - a command is sent before the unsubscribe response is received. */
void subscribe_cb(redisAsyncContext *ac, void *r, void *privdata) {
redisReply *reply = r;
TestState *state = privdata;
assert(reply != NULL &&
reply->type == (state->resp3 ? REDIS_REPLY_PUSH : REDIS_REPLY_ARRAY) &&
reply->elements == 3);
if (strcmp(reply->element[0]->str,"subscribe") == 0) {
assert(strcmp(reply->element[1]->str,"mychannel") == 0 &&
reply->element[2]->str == NULL);
publish_msg(state->options,"mychannel","Hello!");
} else if (strcmp(reply->element[0]->str,"message") == 0) {
assert(strcmp(reply->element[1]->str,"mychannel") == 0 &&
strcmp(reply->element[2]->str,"Hello!") == 0);
state->checkpoint++;
/* Unsubscribe after receiving the published message. Send unsubscribe
* which should call the callback registered during subscribe */
redisAsyncCommand(ac,unexpected_cb,
(void*)"unsubscribe should call subscribe_cb()",
"unsubscribe");
/* Send a regular command after unsubscribing, then disconnect */
state->disconnect = 1;
redisAsyncCommand(ac,integer_cb,state,"LPUSH mylist foo");
} else if (strcmp(reply->element[0]->str,"unsubscribe") == 0) {
assert(strcmp(reply->element[1]->str,"mychannel") == 0 &&
reply->element[2]->str == NULL);
} else {
printf("Unexpected pubsub command: %s\n", reply->element[0]->str);
exit(1);
}
}
/* Expect a reply of type ARRAY */
void array_cb(redisAsyncContext *ac, void *r, void *privdata) {
redisReply *reply = r;
TestState *state = privdata;
assert(reply != NULL && reply->type == REDIS_REPLY_ARRAY);
state->checkpoint++;
if (state->disconnect) async_disconnect(ac);
}
/* Expect a NULL reply */
void null_cb(redisAsyncContext *ac, void *r, void *privdata) {
(void) ac;
assert(r == NULL);
TestState *state = privdata;
state->checkpoint++;
}
static void test_pubsub_handling(struct config config) {
test("Subscribe, handle published message and unsubscribe: ");
/* Setup event dispatcher with a testcase timeout */
base = event_base_new();
struct event *timeout = evtimer_new(base, timeout_cb, NULL);
assert(timeout != NULL);
evtimer_assign(timeout,base,timeout_cb,NULL);
struct timeval timeout_tv = {.tv_sec = 10};
evtimer_add(timeout, &timeout_tv);
/* Connect */
redisOptions options = get_redis_tcp_options(config);
redisAsyncContext *ac = redisAsyncConnectWithOptions(&options);
assert(ac != NULL && ac->err == 0);
redisLibeventAttach(ac,base);
/* Start subscribe */
TestState state = {.options = &options};
redisAsyncCommand(ac,subscribe_cb,&state,"subscribe mychannel");
/* Make sure non-subscribe commands are handled */
redisAsyncCommand(ac,array_cb,&state,"PING");
/* Start event dispatching loop */
test_cond(event_base_dispatch(base) == 0);
event_free(timeout);
event_base_free(base);
/* Verify test checkpoints */
assert(state.checkpoint == 3);
}
/* Unexpected push message, will trigger a failure */
void unexpected_push_cb(redisAsyncContext *ac, void *r) {
(void) ac; (void) r;
printf("Unexpected call to the PUSH callback!\n");
exit(1);
}
static void test_pubsub_handling_resp3(struct config config) {
test("Subscribe, handle published message and unsubscribe using RESP3: ");
/* Setup event dispatcher with a testcase timeout */
base = event_base_new();
struct event *timeout = evtimer_new(base, timeout_cb, NULL);
assert(timeout != NULL);
evtimer_assign(timeout,base,timeout_cb,NULL);
struct timeval timeout_tv = {.tv_sec = 10};
evtimer_add(timeout, &timeout_tv);
/* Connect */
redisOptions options = get_redis_tcp_options(config);
redisAsyncContext *ac = redisAsyncConnectWithOptions(&options);
assert(ac != NULL && ac->err == 0);
redisLibeventAttach(ac,base);
/* Not expecting any push messages in this test */
redisAsyncSetPushCallback(ac, unexpected_push_cb);
/* Switch protocol */
redisAsyncCommand(ac,NULL,NULL,"HELLO 3");
/* Start subscribe */
TestState state = {.options = &options, .resp3 = 1};
redisAsyncCommand(ac,subscribe_cb,&state,"subscribe mychannel");
/* Make sure non-subscribe commands are handled in RESP3 */
redisAsyncCommand(ac,integer_cb,&state,"LPUSH mylist foo");
redisAsyncCommand(ac,integer_cb,&state,"LPUSH mylist foo");
redisAsyncCommand(ac,integer_cb,&state,"LPUSH mylist foo");
/* Handle an array with 3 elements as a non-subscribe command */
redisAsyncCommand(ac,array_cb,&state,"LRANGE mylist 0 2");
/* Start event dispatching loop */
test_cond(event_base_dispatch(base) == 0);
event_free(timeout);
event_base_free(base);
/* Verify test checkpoints */
assert(state.checkpoint == 6);
}
/* Subscribe callback for test_command_timeout_during_pubsub:
* - a subscribe response triggers a published message
* - the published message triggers a command that times out
* - the command timeout triggers a disconnect */
void subscribe_with_timeout_cb(redisAsyncContext *ac, void *r, void *privdata) {
redisReply *reply = r;
TestState *state = privdata;
/* The non-clean disconnect should trigger the
* subscription callback with a NULL reply. */
if (reply == NULL) {
state->checkpoint++;
event_base_loopbreak(base);
return;
}
assert(reply->type == (state->resp3 ? REDIS_REPLY_PUSH : REDIS_REPLY_ARRAY) &&
reply->elements == 3);
if (strcmp(reply->element[0]->str,"subscribe") == 0) {
assert(strcmp(reply->element[1]->str,"mychannel") == 0 &&
reply->element[2]->str == NULL);
publish_msg(state->options,"mychannel","Hello!");
state->checkpoint++;
} else if (strcmp(reply->element[0]->str,"message") == 0) {
assert(strcmp(reply->element[1]->str,"mychannel") == 0 &&
strcmp(reply->element[2]->str,"Hello!") == 0);
state->checkpoint++;
/* Send a command that will trigger a timeout */
redisAsyncCommand(ac,null_cb,state,"DEBUG SLEEP 3");
redisAsyncCommand(ac,null_cb,state,"LPUSH mylist foo");
} else {
printf("Unexpected pubsub command: %s\n", reply->element[0]->str);
exit(1);
}
}
static void test_command_timeout_during_pubsub(struct config config) {
test("Command timeout during Pub/Sub: ");
/* Setup event dispatcher with a testcase timeout */
base = event_base_new();
struct event *timeout = evtimer_new(base,timeout_cb,NULL);
assert(timeout != NULL);
evtimer_assign(timeout,base,timeout_cb,NULL);
struct timeval timeout_tv = {.tv_sec = 10};
evtimer_add(timeout,&timeout_tv);
/* Connect */
redisOptions options = get_redis_tcp_options(config);
redisAsyncContext *ac = redisAsyncConnectWithOptions(&options);
assert(ac != NULL && ac->err == 0);
redisLibeventAttach(ac,base);
/* Configure a command timout */
struct timeval command_timeout = {.tv_sec = 2};
redisAsyncSetTimeout(ac,command_timeout);
/* Not expecting any push messages in this test */
redisAsyncSetPushCallback(ac,unexpected_push_cb);
/* Switch protocol */
redisAsyncCommand(ac,NULL,NULL,"HELLO 3");
/* Start subscribe */
TestState state = {.options = &options, .resp3 = 1};
redisAsyncCommand(ac,subscribe_with_timeout_cb,&state,"subscribe mychannel");
/* Start event dispatching loop */
assert(event_base_dispatch(base) == 0);
event_free(timeout);
event_base_free(base);
/* Verify test checkpoints */
test_cond(state.checkpoint == 5);
}
/* Subscribe callback for test_pubsub_multiple_channels */
void subscribe_channel_a_cb(redisAsyncContext *ac, void *r, void *privdata) {
redisReply *reply = r;
TestState *state = privdata;
assert(reply != NULL && reply->type == REDIS_REPLY_ARRAY &&
reply->elements == 3);
if (strcmp(reply->element[0]->str,"subscribe") == 0) {
assert(strcmp(reply->element[1]->str,"A") == 0);
publish_msg(state->options,"A","Hello!");
state->checkpoint++;
} else if (strcmp(reply->element[0]->str,"message") == 0) {
assert(strcmp(reply->element[1]->str,"A") == 0 &&
strcmp(reply->element[2]->str,"Hello!") == 0);
state->checkpoint++;
/* Unsubscribe to channels, including a channel X which we don't subscribe to */
redisAsyncCommand(ac,unexpected_cb,
(void*)"unsubscribe should not call unexpected_cb()",
"unsubscribe B X A");
/* Send a regular command after unsubscribing, then disconnect */
state->disconnect = 1;
redisAsyncCommand(ac,integer_cb,state,"LPUSH mylist foo");
} else if (strcmp(reply->element[0]->str,"unsubscribe") == 0) {
assert(strcmp(reply->element[1]->str,"A") == 0);
state->checkpoint++;
} else {
printf("Unexpected pubsub command: %s\n", reply->element[0]->str);
exit(1);
}
}
/* Subscribe callback for test_pubsub_multiple_channels */
void subscribe_channel_b_cb(redisAsyncContext *ac, void *r, void *privdata) {
redisReply *reply = r;
TestState *state = privdata;
assert(reply != NULL && reply->type == REDIS_REPLY_ARRAY &&
reply->elements == 3);
if (strcmp(reply->element[0]->str,"subscribe") == 0) {
assert(strcmp(reply->element[1]->str,"B") == 0);
state->checkpoint++;
} else if (strcmp(reply->element[0]->str,"unsubscribe") == 0) {
assert(strcmp(reply->element[1]->str,"B") == 0);
state->checkpoint++;
} else {
printf("Unexpected pubsub command: %s\n", reply->element[0]->str);
exit(1);
}
}
/* Test handling of multiple channels
* - subscribe to channel A and B
* - a published message on A triggers an unsubscribe of channel B, X and A
* where channel X is not subscribed to.
* - a command sent after unsubscribe triggers a disconnect */
static void test_pubsub_multiple_channels(struct config config) {
test("Subscribe to multiple channels: ");
/* Setup event dispatcher with a testcase timeout */
base = event_base_new();
struct event *timeout = evtimer_new(base,timeout_cb,NULL);
assert(timeout != NULL);
evtimer_assign(timeout,base,timeout_cb,NULL);
struct timeval timeout_tv = {.tv_sec = 10};
evtimer_add(timeout,&timeout_tv);
/* Connect */
redisOptions options = get_redis_tcp_options(config);
redisAsyncContext *ac = redisAsyncConnectWithOptions(&options);
assert(ac != NULL && ac->err == 0);
redisLibeventAttach(ac,base);
/* Not expecting any push messages in this test */
redisAsyncSetPushCallback(ac,unexpected_push_cb);
/* Start subscribing to two channels */
TestState state = {.options = &options};
redisAsyncCommand(ac,subscribe_channel_a_cb,&state,"subscribe A");
redisAsyncCommand(ac,subscribe_channel_b_cb,&state,"subscribe B");
/* Start event dispatching loop */
assert(event_base_dispatch(base) == 0);
event_free(timeout);
event_base_free(base);
/* Verify test checkpoints */
test_cond(state.checkpoint == 6);
}
/* Command callback for test_monitor() */
void monitor_cb(redisAsyncContext *ac, void *r, void *privdata) {
redisReply *reply = r;
TestState *state = privdata;
/* NULL reply is received when BYE triggers a disconnect. */
if (reply == NULL) {
event_base_loopbreak(base);
return;
}
assert(reply != NULL && reply->type == REDIS_REPLY_STATUS);
state->checkpoint++;
if (state->checkpoint == 1) {
/* Response from MONITOR */
redisContext *c = redisConnectWithOptions(state->options);
assert(c != NULL);
redisReply *reply = redisCommand(c,"SET first 1");
assert(reply->type == REDIS_REPLY_STATUS);
freeReplyObject(reply);
redisFree(c);
} else if (state->checkpoint == 2) {
/* Response for monitored command 'SET first 1' */
assert(strstr(reply->str,"first") != NULL);
redisContext *c = redisConnectWithOptions(state->options);
assert(c != NULL);
redisReply *reply = redisCommand(c,"SET second 2");
assert(reply->type == REDIS_REPLY_STATUS);
freeReplyObject(reply);
redisFree(c);
} else if (state->checkpoint == 3) {
/* Response for monitored command 'SET second 2' */
assert(strstr(reply->str,"second") != NULL);
/* Send QUIT to disconnect */
redisAsyncCommand(ac,NULL,NULL,"QUIT");
}
}
/* Test handling of the monitor command
* - sends MONITOR to enable monitoring.
* - sends SET commands via separate clients to be monitored.
* - sends QUIT to stop monitoring and disconnect. */
static void test_monitor(struct config config) {
test("Enable monitoring: ");
/* Setup event dispatcher with a testcase timeout */
base = event_base_new();
struct event *timeout = evtimer_new(base, timeout_cb, NULL);
assert(timeout != NULL);
evtimer_assign(timeout,base,timeout_cb,NULL);
struct timeval timeout_tv = {.tv_sec = 10};
evtimer_add(timeout, &timeout_tv);
/* Connect */
redisOptions options = get_redis_tcp_options(config);
redisAsyncContext *ac = redisAsyncConnectWithOptions(&options);
assert(ac != NULL && ac->err == 0);
redisLibeventAttach(ac,base);
/* Not expecting any push messages in this test */
redisAsyncSetPushCallback(ac,unexpected_push_cb);
/* Start monitor */
TestState state = {.options = &options};
redisAsyncCommand(ac,monitor_cb,&state,"monitor");
/* Start event dispatching loop */
test_cond(event_base_dispatch(base) == 0);
event_free(timeout);
event_base_free(base);
/* Verify test checkpoints */
assert(state.checkpoint == 3);
}
#endif /* HIREDIS_TEST_ASYNC */
int main(int argc, char **argv) {
struct config cfg = {
.tcp = {
@@ -1387,6 +1999,24 @@ int main(int argc, char **argv) {
}
#endif
#ifdef HIREDIS_TEST_ASYNC
printf("\nTesting asynchronous API against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port);
cfg.type = CONN_TCP;
int major;
redisContext *c = do_connect(cfg);
get_redis_version(c, &major, NULL);
disconnect(c, 0);
test_pubsub_handling(cfg);
test_pubsub_multiple_channels(cfg);
test_monitor(cfg);
if (major >= 6) {
test_pubsub_handling_resp3(cfg);
test_command_timeout_during_pubsub(cfg);
}
#endif /* HIREDIS_TEST_ASYNC */
if (test_inherit_fd) {
printf("\nTesting against inherited fd (%s): ", cfg.unix_sock.path);
if (test_unix_socket) {
deps/jemalloc/.appveyor.yml
Vendored
+8 -8
View File
@@ -5,27 +5,27 @@ environment:
- MSYSTEM: MINGW64
CPU: x86_64
MSVC: amd64
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW64
CPU: x86_64
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW32
CPU: i686
MSVC: x86
- MSYSTEM: MINGW64
CPU: x86_64
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW32
CPU: i686
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW64
CPU: x86_64
MSVC: amd64
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW64
CPU: x86_64
- MSYSTEM: MINGW32
CPU: i686
MSVC: x86
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW64
CPU: x86_64
CONFIG_FLAGS: --enable-debug
- MSYSTEM: MINGW32
CPU: i686
CONFIG_FLAGS: --enable-debug
install:
- set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
deps/jemalloc/.cirrus.yml
Vendored
+21
View File
@@ -0,0 +1,21 @@
env:
CIRRUS_CLONE_DEPTH: 1
ARCH: amd64
task:
freebsd_instance:
matrix:
image: freebsd-12-0-release-amd64
image: freebsd-11-2-release-amd64
install_script:
- sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf
- pkg upgrade -y
- pkg install -y autoconf gmake
script:
- autoconf
#- ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS
- ./configure
- export JFLAG=`sysctl -n kern.smp.cpus`
- gmake -j${JFLAG}
- gmake -j${JFLAG} tests
- gmake check
deps/jemalloc/.gitignore
-1
View File
@@ -30,7 +30,6 @@
/include/jemalloc/internal/public_namespace.h
/include/jemalloc/internal/public_symbols.txt
/include/jemalloc/internal/public_unnamespace.h
/include/jemalloc/internal/size_classes.h
/include/jemalloc/jemalloc.h
/include/jemalloc/jemalloc_defs.h
/include/jemalloc/jemalloc_macros.h
deps/jemalloc/.travis.yml
Vendored
+72 -33
View File
@@ -11,7 +11,7 @@ matrix:
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
addons: &gcc_multilib
apt:
packages:
- gcc-multilib
@@ -21,6 +21,10 @@ matrix:
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
@@ -37,20 +41,25 @@ matrix:
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
addons: *gcc_multilib
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
@@ -61,50 +70,39 @@ matrix:
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
addons: *gcc_multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
addons: *gcc_multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
addons: *gcc_multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons: *gcc_multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons: *gcc_multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
addons: *gcc_multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
addons: *gcc_multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
addons: *gcc_multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
- gcc-multilib
addons: *gcc_multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
@@ -115,6 +113,10 @@ matrix:
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
@@ -123,6 +125,10 @@ matrix:
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
@@ -131,6 +137,24 @@ matrix:
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false,dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
@@ -143,10 +167,25 @@ matrix:
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
# Development build
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
# --enable-expermental-smallocx:
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
# Valgrind
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind"
addons:
apt:
packages:
- valgrind
before_script:
- autoconf
- scripts/gen_travis.py > travis_script && diff .travis.yml travis_script
- ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS
- make -j3
- make -j3 tests
deps/jemalloc/COPYING
Vendored
+2 -2
View File
@@ -1,10 +1,10 @@
Unless otherwise specified, files in the jemalloc source distribution are
subject to the following license:
--------------------------------------------------------------------------------
Copyright (C) 2002-2018 Jason Evans <jasone@canonware.com>.
Copyright (C) 2002-present Jason Evans <jasone@canonware.com>.
All rights reserved.
Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
Copyright (C) 2009-2018 Facebook, Inc. All rights reserved.
Copyright (C) 2009-present Facebook, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
deps/jemalloc/ChangeLog
Vendored
+137 -1
View File
@@ -4,7 +4,143 @@ brevity. Much more detail can be found in the git revision history:
https://github.com/jemalloc/jemalloc
* 5.1.0 (May 4th, 2018)
* 5.2.1 (August 5, 2019)
This release is primarily about Windows. A critical virtual memory leak is
resolved on all Windows platforms. The regression was present in all releases
since 5.0.0.
Bug fixes:
- Fix a severe virtual memory leak on Windows. This regression was first
released in 5.0.0. (@Ignition, @j0t, @frederik-h, @davidtgoldblatt,
@interwq)
- Fix size 0 handling in posix_memalign(). This regression was first released
in 5.2.0. (@interwq)
- Fix the prof_log unit test which may observe unexpected backtraces from
compiler optimizations. The test was first added in 5.2.0. (@marxin,
@gnzlbg, @interwq)
- Fix the declaration of the extent_avail tree. This regression was first
released in 5.1.0. (@zoulasc)
- Fix an incorrect reference in jeprof. This functionality was first released
in 3.0.0. (@prehistoric-penguin)
- Fix an assertion on the deallocation fast-path. This regression was first
released in 5.2.0. (@yinan1048576)
- Fix the TLS_MODEL attribute in headers. This regression was first released
in 5.0.0. (@zoulasc, @interwq)
Optimizations and refactors:
- Implement opt.retain on Windows and enable by default on 64-bit. (@interwq,
@davidtgoldblatt)
- Optimize away a branch on the operator delete[] path. (@mgrice)
- Add format annotation to the format generator function. (@zoulasc)
- Refactor and improve the size class header generation. (@yinan1048576)
- Remove best fit. (@djwatson)
- Avoid blocking on background thread locks for stats. (@oranagra, @interwq)
* 5.2.0 (April 2, 2019)
This release includes a few notable improvements, which are summarized below:
1) improved fast-path performance from the optimizations by @djwatson; 2)
reduced virtual memory fragmentation and metadata usage; and 3) bug fixes on
setting the number of background threads. In addition, peak / spike memory
usage is improved with certain allocation patterns. As usual, the release and
prior dev versions have gone through large-scale production testing.
New features:
- Implement oversize_threshold, which uses a dedicated arena for allocations
crossing the specified threshold to reduce fragmentation. (@interwq)
- Add extents usage information to stats. (@tyleretzel)
- Log time information for sampled allocations. (@tyleretzel)
- Support 0 size in sdallocx. (@djwatson)
- Output rate for certain counters in malloc_stats. (@zinoale)
- Add configure option --enable-readlinkat, which allows the use of readlinkat
over readlink. (@davidtgoldblatt)
- Add configure options --{enable,disable}-{static,shared} to allow not
building unwanted libraries. (@Ericson2314)
- Add configure option --disable-libdl to enable fully static builds.
(@interwq)
- Add mallctl interfaces:
+ opt.oversize_threshold (@interwq)
+ stats.arenas.<i>.extent_avail (@tyleretzel)
+ stats.arenas.<i>.extents.<j>.n{dirty,muzzy,retained} (@tyleretzel)
+ stats.arenas.<i>.extents.<j>.{dirty,muzzy,retained}_bytes
(@tyleretzel)
Portability improvements:
- Update MSVC builds. (@maksqwe, @rustyx)
- Workaround a compiler optimizer bug on s390x. (@rkmisra)
- Make use of pthread_set_name_np(3) on FreeBSD. (@trasz)
- Implement malloc_getcpu() to enable percpu_arena for windows. (@santagada)
- Link against -pthread instead of -lpthread. (@paravoid)
- Make background_thread not dependent on libdl. (@interwq)
- Add stringify to fix a linker directive issue on MSVC. (@daverigby)
- Detect and fall back when 8-bit atomics are unavailable. (@interwq)
- Fall back to the default pthread_create if dlsym(3) fails. (@interwq)
Optimizations and refactors:
- Refactor the TSD module. (@davidtgoldblatt)
- Avoid taking extents_muzzy mutex when muzzy is disabled. (@interwq)
- Avoid taking large_mtx for auto arenas on the tcache flush path. (@interwq)
- Optimize ixalloc by avoiding a size lookup. (@interwq)
- Implement opt.oversize_threshold which uses a dedicated arena for requests
crossing the threshold, also eagerly purges the oversize extents. Default
the threshold to 8 MiB. (@interwq)
- Clean compilation with -Wextra. (@gnzlbg, @jasone)
- Refactor the size class module. (@davidtgoldblatt)
- Refactor the stats emitter. (@tyleretzel)
- Optimize pow2_ceil. (@rkmisra)
- Avoid runtime detection of lazy purging on FreeBSD. (@trasz)
- Optimize mmap(2) alignment handling on FreeBSD. (@trasz)
- Improve error handling for THP state initialization. (@jsteemann)
- Rework the malloc() fast path. (@djwatson)
- Rework the free() fast path. (@djwatson)
- Refactor and optimize the tcache fill / flush paths. (@djwatson)
- Optimize sync / lwsync on PowerPC. (@chmeeedalf)
- Bypass extent_dalloc() when retain is enabled. (@interwq)
- Optimize the locking on large deallocation. (@interwq)
- Reduce the number of pages committed from sanity checking in debug build.
(@trasz, @interwq)
- Deprecate OSSpinLock. (@interwq)
- Lower the default number of background threads to 4 (when the feature
is enabled). (@interwq)
- Optimize the trylock spin wait. (@djwatson)
- Use arena index for arena-matching checks. (@interwq)
- Avoid forced decay on thread termination when using background threads.
(@interwq)
- Disable muzzy decay by default. (@djwatson, @interwq)
- Only initialize libgcc unwinder when profiling is enabled. (@paravoid,
@interwq)
Bug fixes (all only relevant to jemalloc 5.x):
- Fix background thread index issues with max_background_threads. (@djwatson,
@interwq)
- Fix stats output for opt.lg_extent_max_active_fit. (@interwq)
- Fix opt.prof_prefix initialization. (@davidtgoldblatt)
- Properly trigger decay on tcache destroy. (@interwq, @amosbird)
- Fix tcache.flush. (@interwq)
- Detect whether explicit extent zero out is necessary with huge pages or
custom extent hooks, which may change the purge semantics. (@interwq)
- Fix a side effect caused by extent_max_active_fit combined with decay-based
purging, where freed extents can accumulate and not be reused for an
extended period of time. (@interwq, @mpghf)
- Fix a missing unlock on extent register error handling. (@zoulasc)
Testing:
- Simplify the Travis script output. (@gnzlbg)
- Update the test scripts for FreeBSD. (@devnexen)
- Add unit tests for the producer-consumer pattern. (@interwq)
- Add Cirrus-CI config for FreeBSD builds. (@jasone)
- Add size-matching sanity checks on tcache flush. (@davidtgoldblatt,
@interwq)
Incompatible changes:
- Remove --with-lg-page-sizes. (@davidtgoldblatt)
Documentation:
- Attempt to build docs by default, however skip doc building when xsltproc
is missing. (@interwq, @cmuellner)
* 5.1.0 (May 4, 2018)
This release is primarily about fine-tuning, ranging from several new features
to numerous notable performance and portability enhancements. The release and
deps/jemalloc/INSTALL.md
Vendored
+5 -7
View File
@@ -221,13 +221,6 @@ any of the following arguments (not a definitive list) to 'configure':
system page size may change between configuration and execution, e.g. when
cross compiling.
* `--with-lg-page-sizes=<lg-page-sizes>`
Specify the comma-separated base 2 logs of the page sizes to support. This
option may be useful when cross compiling in combination with
`--with-lg-page`, but its primary use case is for integration with FreeBSD's
libc, wherein jemalloc is embedded.
* `--with-lg-hugepage=<lg-hugepage>`
Specify the base 2 log of the system huge page size. This option is useful
@@ -276,6 +269,11 @@ any of the following arguments (not a definitive list) to 'configure':
in the same process, which will almost certainly result in confusing runtime
crashes if pointers leak from one implementation to the other.
* `--disable-libdl`
Disable the usage of libdl, namely dlsym(3) which is required by the lazy
lock option. This can allow building static binaries.
The following environment variables (not a definitive list) impact configure's
behavior:
deps/jemalloc/Makefile.in
Vendored
+62 -8
View File
@@ -47,6 +47,7 @@ REV := @rev@
install_suffix := @install_suffix@
ABI := @abi@
XSLTPROC := @XSLTPROC@
XSLROOT := @XSLROOT@
AUTOCONF := @AUTOCONF@
_RPATH = @RPATH@
RPATH = $(if $(1),$(call _RPATH,$(1)))
@@ -55,8 +56,12 @@ cfghdrs_out := @cfghdrs_out@
cfgoutputs_in := $(addprefix $(srcroot),@cfgoutputs_in@)
cfgoutputs_out := @cfgoutputs_out@
enable_autogen := @enable_autogen@
enable_doc := @enable_doc@
enable_shared := @enable_shared@
enable_static := @enable_static@
enable_prof := @enable_prof@
enable_zone_allocator := @enable_zone_allocator@
enable_experimental_smallocx := @enable_experimental_smallocx@
MALLOC_CONF := @JEMALLOC_CPREFIX@MALLOC_CONF
link_whole_archive := @link_whole_archive@
DSO_LDFLAGS = @DSO_LDFLAGS@
@@ -102,7 +107,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/extent_dss.c \
$(srcroot)src/extent_mmap.c \
$(srcroot)src/hash.c \
$(srcroot)src/hooks.c \
$(srcroot)src/hook.c \
$(srcroot)src/large.c \
$(srcroot)src/log.c \
$(srcroot)src/malloc_io.c \
@@ -113,9 +118,12 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/prng.c \
$(srcroot)src/prof.c \
$(srcroot)src/rtree.c \
$(srcroot)src/safety_check.c \
$(srcroot)src/stats.c \
$(srcroot)src/sc.c \
$(srcroot)src/sz.c \
$(srcroot)src/tcache.c \
$(srcroot)src/test_hooks.c \
$(srcroot)src/ticker.c \
$(srcroot)src/tsd.c \
$(srcroot)src/witness.c
@@ -165,14 +173,18 @@ TESTS_UNIT := \
$(srcroot)test/unit/background_thread_enable.c \
$(srcroot)test/unit/base.c \
$(srcroot)test/unit/bitmap.c \
$(srcroot)test/unit/bit_util.c \
$(srcroot)test/unit/binshard.c \
$(srcroot)test/unit/ckh.c \
$(srcroot)test/unit/decay.c \
$(srcroot)test/unit/div.c \
$(srcroot)test/unit/emitter.c \
$(srcroot)test/unit/extent_quantize.c \
$(srcroot)test/unit/extent_util.c \
$(srcroot)test/unit/fork.c \
$(srcroot)test/unit/hash.c \
$(srcroot)test/unit/hooks.c \
$(srcroot)test/unit/hook.c \
$(srcroot)test/unit/huge.c \
$(srcroot)test/unit/junk.c \
$(srcroot)test/unit/junk_alloc.c \
$(srcroot)test/unit/junk_free.c \
@@ -190,6 +202,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/prof_active.c \
$(srcroot)test/unit/prof_gdump.c \
$(srcroot)test/unit/prof_idump.c \
$(srcroot)test/unit/prof_log.c \
$(srcroot)test/unit/prof_reset.c \
$(srcroot)test/unit/prof_tctx.c \
$(srcroot)test/unit/prof_thread_name.c \
@@ -198,13 +211,17 @@ TESTS_UNIT := \
$(srcroot)test/unit/rb.c \
$(srcroot)test/unit/retained.c \
$(srcroot)test/unit/rtree.c \
$(srcroot)test/unit/safety_check.c \
$(srcroot)test/unit/seq.c \
$(srcroot)test/unit/SFMT.c \
$(srcroot)test/unit/sc.c \
$(srcroot)test/unit/size_classes.c \
$(srcroot)test/unit/slab.c \
$(srcroot)test/unit/smoothstep.c \
$(srcroot)test/unit/spin.c \
$(srcroot)test/unit/stats.c \
$(srcroot)test/unit/stats_print.c \
$(srcroot)test/unit/test_hooks.c \
$(srcroot)test/unit/ticker.c \
$(srcroot)test/unit/nstime.c \
$(srcroot)test/unit/tsd.c \
@@ -217,15 +234,21 @@ endif
TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
$(srcroot)test/integration/allocated.c \
$(srcroot)test/integration/extent.c \
$(srcroot)test/integration/malloc.c \
$(srcroot)test/integration/mallocx.c \
$(srcroot)test/integration/MALLOCX_ARENA.c \
$(srcroot)test/integration/overflow.c \
$(srcroot)test/integration/posix_memalign.c \
$(srcroot)test/integration/rallocx.c \
$(srcroot)test/integration/sdallocx.c \
$(srcroot)test/integration/slab_sizes.c \
$(srcroot)test/integration/thread_arena.c \
$(srcroot)test/integration/thread_tcache_enabled.c \
$(srcroot)test/integration/xallocx.c
ifeq (@enable_experimental_smallocx@, 1)
TESTS_INTEGRATION += \
$(srcroot)test/integration/smallocx.c
endif
ifeq (@enable_cxx@, 1)
CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp
@@ -233,7 +256,9 @@ else
CPP_SRCS :=
TESTS_INTEGRATION_CPP :=
endif
TESTS_STRESS := $(srcroot)test/stress/microbench.c
TESTS_STRESS := $(srcroot)test/stress/microbench.c \
$(srcroot)test/stress/hookbench.c
TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_INTEGRATION_CPP) $(TESTS_STRESS)
@@ -274,10 +299,24 @@ all: build_lib
dist: build_doc
$(objroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl
ifneq ($(XSLROOT),)
$(XSLTPROC) -o $@ $(objroot)doc/html.xsl $<
else
ifeq ($(wildcard $(DOCS_HTML)),)
@echo "<p>Missing xsltproc. Doc not built.</p>" > $@
endif
@echo "Missing xsltproc. "$@" not (re)built."
endif
$(objroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl
ifneq ($(XSLROOT),)
$(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $<
else
ifeq ($(wildcard $(DOCS_MAN3)),)
@echo "Missing xsltproc. Doc not built." > $@
endif
@echo "Missing xsltproc. "$@" not (re)built."
endif
build_doc_html: $(DOCS_HTML)
build_doc_man: $(DOCS_MAN3)
@@ -400,7 +439,7 @@ $(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(C_JET_OBJS) $(C_TESTLI
$(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
@mkdir -p $(@D)
$(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -lpthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS)
$(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -pthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS)
$(objroot)test/integration/cpp/%$(EXE): $(objroot)test/integration/cpp/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
@mkdir -p $(@D)
@@ -412,7 +451,12 @@ $(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TE
build_lib_shared: $(DSOS)
build_lib_static: $(STATIC_LIBS)
build_lib: build_lib_shared build_lib_static
ifeq ($(enable_shared), 1)
build_lib: build_lib_shared
endif
ifeq ($(enable_static), 1)
build_lib: build_lib_static
endif
install_bin:
$(INSTALL) -d $(BINDIR)
@@ -449,7 +493,13 @@ install_lib_pc: $(PC)
$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \
done
install_lib: install_lib_shared install_lib_static install_lib_pc
ifeq ($(enable_shared), 1)
install_lib: install_lib_shared
endif
ifeq ($(enable_static), 1)
install_lib: install_lib_static
endif
install_lib: install_lib_pc
install_doc_html:
$(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix)
@@ -465,9 +515,13 @@ install_doc_man:
$(INSTALL) -m 644 $$d $(MANDIR)/man3; \
done
install_doc: install_doc_html install_doc_man
install_doc: build_doc install_doc_html install_doc_man
install: install_bin install_include install_lib install_doc
install: install_bin install_include install_lib
ifeq ($(enable_doc), 1)
install: install_doc
endif
tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE))
tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE)) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%$(EXE))
deps/jemalloc/VERSION
Vendored
+1 -1
View File
@@ -1 +1 @@
5.1.0-0-g0
5.2.1-0-g0
deps/jemalloc/bin/jeprof.in
Vendored
+2 -1
View File
@@ -2909,6 +2909,7 @@ sub RemoveUninterestingFrames {
'@JEMALLOC_PREFIX@xallocx',
'@JEMALLOC_PREFIX@dallocx',
'@JEMALLOC_PREFIX@sdallocx',
'@JEMALLOC_PREFIX@sdallocx_noflags',
'tc_calloc',
'tc_cfree',
'tc_malloc',
@@ -5366,7 +5367,7 @@ sub GetProcedureBoundaries {
my $demangle_flag = "";
my $cppfilt_flag = "";
my $to_devnull = ">$dev_null 2>&1";
if (system(ShellEscape($nm, "--demangle", "image") . $to_devnull) == 0) {
if (system(ShellEscape($nm, "--demangle", $image) . $to_devnull) == 0) {
# In this mode, we do "nm --demangle <foo>"
$demangle_flag = "--demangle";
$cppfilt_flag = "";
deps/jemalloc/configure
Vendored
+623 -231
View File
File diff suppressed because it is too large Load Diff
deps/jemalloc/configure.ac
Vendored
+250 -138
View File
@@ -175,6 +175,9 @@ fi
],
XSLROOT="${DEFAULT_XSLROOT}"
)
if test "x$XSLTPROC" = "xfalse" ; then
XSLROOT=""
fi
AC_SUBST([XSLROOT])
dnl If CFLAGS isn't defined, set CFLAGS to something reasonable. Otherwise,
@@ -242,6 +245,7 @@ if test "x$GCC" = "xyes" ; then
fi
fi
JE_CFLAGS_ADD([-Wall])
JE_CFLAGS_ADD([-Wextra])
JE_CFLAGS_ADD([-Wshorten-64-to-32])
JE_CFLAGS_ADD([-Wsign-compare])
JE_CFLAGS_ADD([-Wundef])
@@ -289,6 +293,7 @@ if test "x$enable_cxx" = "x1" ; then
AX_CXX_COMPILE_STDCXX([14], [noext], [optional])
if test "x${HAVE_CXX14}" = "x1" ; then
JE_CXXFLAGS_ADD([-Wall])
JE_CXXFLAGS_ADD([-Wextra])
JE_CXXFLAGS_ADD([-g3])
SAVED_LIBS="${LIBS}"
@@ -536,6 +541,66 @@ AC_PROG_NM
AC_PROG_AWK
dnl ============================================================================
dnl jemalloc version.
dnl
AC_ARG_WITH([version],
[AS_HELP_STRING([--with-version=<major>.<minor>.<bugfix>-<nrev>-g<gid>],
[Version string])],
[
echo "${with_version}" | grep ['^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$'] 2>&1 1>/dev/null
if test $? -eq 0 ; then
echo "$with_version" > "${objroot}VERSION"
else
echo "${with_version}" | grep ['^VERSION$'] 2>&1 1>/dev/null
if test $? -ne 0 ; then
AC_MSG_ERROR([${with_version} does not match <major>.<minor>.<bugfix>-<nrev>-g<gid> or VERSION])
fi
fi
], [
dnl Set VERSION if source directory is inside a git repository.
if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then
dnl Pattern globs aren't powerful enough to match both single- and
dnl double-digit version numbers, so iterate over patterns to support up
dnl to version 99.99.99 without any accidental matches.
for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \
'[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \
'[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \
'[0-9][0-9].[0-9][0-9].[0-9]' \
'[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do
(test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null
if test $? -eq 0 ; then
mv "${objroot}VERSION.tmp" "${objroot}VERSION"
break
fi
done
fi
rm -f "${objroot}VERSION.tmp"
])
if test ! -e "${objroot}VERSION" ; then
if test ! -e "${srcroot}VERSION" ; then
AC_MSG_RESULT(
[Missing VERSION file, and unable to generate it; creating bogus VERSION])
echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION"
else
cp ${srcroot}VERSION ${objroot}VERSION
fi
fi
jemalloc_version=`cat "${objroot}VERSION"`
jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'`
jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'`
jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'`
jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'`
jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'`
AC_SUBST([jemalloc_version])
AC_SUBST([jemalloc_version_major])
AC_SUBST([jemalloc_version_minor])
AC_SUBST([jemalloc_version_bugfix])
AC_SUBST([jemalloc_version_nrev])
AC_SUBST([jemalloc_version_gid])
dnl Platform-specific settings. abi and RPATH can probably be determined
dnl programmatically, but doing so is error-prone, which makes it generally
dnl not worth the trouble.
@@ -673,6 +738,9 @@ case "${host}" in
libprefix=""
SOREV="${so}"
PIC_CFLAGS=""
if test "${LG_SIZEOF_PTR}" = "3"; then
default_retain="1"
fi
;;
*)
AC_MSG_RESULT([Unsupported operating system: ${host}])
@@ -786,6 +854,18 @@ if test "x${je_cv_format_printf}" = "xyes" ; then
AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_PRINTF], [ ])
fi
dnl Check for format_arg(...) attribute support.
JE_CFLAGS_SAVE()
JE_CFLAGS_ADD([-Werror])
JE_CFLAGS_ADD([-herror_on_warning])
JE_COMPILABLE([format(printf, ...) attribute], [#include <stdlib.h>],
[const char * __attribute__((__format_arg__(1))) foo(const char *format);],
[je_cv_format_arg])
JE_CFLAGS_RESTORE()
if test "x${je_cv_format_arg}" = "xyes" ; then
AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_ARG], [ ])
fi
dnl Support optional additions to rpath.
AC_ARG_WITH([rpath],
[AS_HELP_STRING([--with-rpath=<rpath>], [Colon-separated rpath (ELF systems only)])],
@@ -816,6 +896,49 @@ AC_PROG_RANLIB
AC_PATH_PROG([LD], [ld], [false], [$PATH])
AC_PATH_PROG([AUTOCONF], [autoconf], [false], [$PATH])
dnl Enable documentation
AC_ARG_ENABLE([doc],
[AS_HELP_STRING([--enable-documentation], [Build documentation])],
if test "x$enable_doc" = "xno" ; then
enable_doc="0"
else
enable_doc="1"
fi
,
enable_doc="1"
)
AC_SUBST([enable_doc])
dnl Enable shared libs
AC_ARG_ENABLE([shared],
[AS_HELP_STRING([--enable-shared], [Build shared libaries])],
if test "x$enable_shared" = "xno" ; then
enable_shared="0"
else
enable_shared="1"
fi
,
enable_shared="1"
)
AC_SUBST([enable_shared])
dnl Enable static libs
AC_ARG_ENABLE([static],
[AS_HELP_STRING([--enable-static], [Build static libaries])],
if test "x$enable_static" = "xno" ; then
enable_static="0"
else
enable_static="1"
fi
,
enable_static="1"
)
AC_SUBST([enable_static])
if test "$enable_shared$enable_static" = "00" ; then
AC_MSG_ERROR([Please enable one of shared or static builds])
fi
dnl Perform no name mangling by default.
AC_ARG_WITH([mangling],
[AS_HELP_STRING([--with-mangling=<map>], [Mangle symbols in <map>])],
@@ -848,7 +971,7 @@ AC_ARG_WITH([export],
fi]
)
public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx"
public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx"
dnl Check for additional platform-specific public API functions.
AC_CHECK_FUNC([memalign],
[AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN], [ ])
@@ -966,7 +1089,6 @@ cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_symbols.sh"
cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_namespace.sh"
cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_namespace.sh"
cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_unnamespace.sh"
cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/size_classes.sh"
cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_rename.sh"
cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_mangle.sh"
cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc.sh"
@@ -979,7 +1101,6 @@ cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_symbols_jet.awk"
cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_symbols.txt"
cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_namespace.h"
cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_unnamespace.h"
cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h"
cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_protos_jet.h"
cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_rename.h"
cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle.h"
@@ -991,6 +1112,10 @@ cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.i
cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:include/jemalloc/internal/jemalloc_internal_defs.h.in"
cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:test/include/test/jemalloc_test_defs.h.in"
dnl ============================================================================
dnl jemalloc build options.
dnl
dnl Do not compile with debugging by default.
AC_ARG_ENABLE([debug],
[AS_HELP_STRING([--enable-debug],
@@ -1043,6 +1168,22 @@ if test "x$enable_stats" = "x1" ; then
fi
AC_SUBST([enable_stats])
dnl Do not enable smallocx by default.
AC_ARG_ENABLE([experimental_smallocx],
[AS_HELP_STRING([--enable-experimental-smallocx], [Enable experimental smallocx API])],
[if test "x$enable_experimental_smallocx" = "xno" ; then
enable_experimental_smallocx="0"
else
enable_experimental_smallocx="1"
fi
],
[enable_experimental_smallocx="0"]
)
if test "x$enable_experimental_smallocx" = "x1" ; then
AC_DEFINE([JEMALLOC_EXPERIMENTAL_SMALLOCX_API])
fi
AC_SUBST([enable_experimental_smallocx])
dnl Do not enable profiling by default.
AC_ARG_ENABLE([prof],
[AS_HELP_STRING([--enable-prof], [Enable allocation profiling])],
@@ -1277,6 +1418,38 @@ if test "x$enable_log" = "x1" ; then
fi
AC_SUBST([enable_log])
dnl Do not use readlinkat by default
AC_ARG_ENABLE([readlinkat],
[AS_HELP_STRING([--enable-readlinkat], [Use readlinkat over readlink])],
[if test "x$enable_readlinkat" = "xno" ; then
enable_readlinkat="0"
else
enable_readlinkat="1"
fi
],
[enable_readlinkat="0"]
)
if test "x$enable_readlinkat" = "x1" ; then
AC_DEFINE([JEMALLOC_READLINKAT], [ ])
fi
AC_SUBST([enable_readlinkat])
dnl Avoid extra safety checks by default
AC_ARG_ENABLE([opt-safety-checks],
[AS_HELP_STRING([--enable-opt-safety-checks],
[Perform certain low-overhead checks, even in opt mode])],
[if test "x$enable_opt_safety_checks" = "xno" ; then
enable_opt_safety_checks="0"
else
enable_opt_safety_checks="1"
fi
],
[enable_opt_safety_checks="0"]
)
if test "x$enable_opt_safety_checks" = "x1" ; then
AC_DEFINE([JEMALLOC_OPT_SAFETY_CHECKS], [ ])
fi
AC_SUBST([enable_opt_safety_checks])
JE_COMPILABLE([a program using __builtin_unreachable], [
void foo (void) {
@@ -1333,6 +1506,21 @@ else
fi
fi
JE_COMPILABLE([a program using __builtin_popcountl], [
#include <stdio.h>
#include <strings.h>
#include <string.h>
], [
{
int rv = __builtin_popcountl(0x08);
printf("%d\n", rv);
}
], [je_cv_gcc_builtin_popcountl])
if test "x${je_cv_gcc_builtin_popcountl}" = "xyes" ; then
AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNT], [__builtin_popcount])
AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNTL], [__builtin_popcountl])
fi
AC_ARG_WITH([lg_quantum],
[AS_HELP_STRING([--with-lg-quantum=<lg-quantum>],
[Base 2 log of minimum allocation alignment])],
@@ -1430,70 +1618,20 @@ if test "x${LG_PAGE}" != "xundefined" -a \
fi
AC_DEFINE_UNQUOTED([LG_HUGEPAGE], [${je_cv_lg_hugepage}])
AC_ARG_WITH([lg_page_sizes],
[AS_HELP_STRING([--with-lg-page-sizes=<lg-page-sizes>],
[Base 2 logs of system page sizes to support])],
[LG_PAGE_SIZES="$with_lg_page_sizes"], [LG_PAGE_SIZES="$LG_PAGE"])
dnl ============================================================================
dnl jemalloc configuration.
dnl
AC_ARG_WITH([version],
[AS_HELP_STRING([--with-version=<major>.<minor>.<bugfix>-<nrev>-g<gid>],
[Version string])],
[
echo "${with_version}" | grep ['^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$'] 2>&1 1>/dev/null
if test $? -eq 0 ; then
echo "$with_version" > "${objroot}VERSION"
else
echo "${with_version}" | grep ['^VERSION$'] 2>&1 1>/dev/null
if test $? -ne 0 ; then
AC_MSG_ERROR([${with_version} does not match <major>.<minor>.<bugfix>-<nrev>-g<gid> or VERSION])
fi
fi
], [
dnl Set VERSION if source directory is inside a git repository.
if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then
dnl Pattern globs aren't powerful enough to match both single- and
dnl double-digit version numbers, so iterate over patterns to support up
dnl to version 99.99.99 without any accidental matches.
for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \
'[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \
'[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \
'[0-9][0-9].[0-9][0-9].[0-9]' \
'[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do
(test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null
if test $? -eq 0 ; then
mv "${objroot}VERSION.tmp" "${objroot}VERSION"
break
fi
done
fi
rm -f "${objroot}VERSION.tmp"
])
if test ! -e "${objroot}VERSION" ; then
if test ! -e "${srcroot}VERSION" ; then
AC_MSG_RESULT(
[Missing VERSION file, and unable to generate it; creating bogus VERSION])
echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION"
else
cp ${srcroot}VERSION ${objroot}VERSION
fi
dnl Enable libdl by default.
AC_ARG_ENABLE([libdl],
[AS_HELP_STRING([--disable-libdl],
[Do not use libdl])],
[if test "x$enable_libdl" = "xno" ; then
enable_libdl="0"
else
enable_libdl="1"
fi
jemalloc_version=`cat "${objroot}VERSION"`
jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'`
jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'`
jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'`
jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'`
jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'`
AC_SUBST([jemalloc_version])
AC_SUBST([jemalloc_version_major])
AC_SUBST([jemalloc_version_minor])
AC_SUBST([jemalloc_version_bugfix])
AC_SUBST([jemalloc_version_nrev])
AC_SUBST([jemalloc_version_gid])
],
[enable_libdl="1"]
)
AC_SUBST([libdl])
dnl ============================================================================
dnl Configure pthreads.
@@ -1503,20 +1641,26 @@ if test "x$abi" != "xpecoff" ; then
AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])])
dnl Some systems may embed pthreads functionality in libc; check for libpthread
dnl first, but try libc too before failing.
AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -lpthread)],
AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -pthread)],
[AC_SEARCH_LIBS([pthread_create], , ,
AC_MSG_ERROR([libpthread is missing]))])
wrap_syms="${wrap_syms} pthread_create"
have_pthread="1"
dnl Check if we have dlsym support.
have_dlsym="1"
AC_CHECK_HEADERS([dlfcn.h],
AC_CHECK_FUNC([dlsym], [],
[AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], [have_dlsym="0"])]),
[have_dlsym="0"])
if test "x$have_dlsym" = "x1" ; then
AC_DEFINE([JEMALLOC_HAVE_DLSYM], [ ])
dnl Check if we have dlsym support.
if test "x$enable_libdl" = "x1" ; then
have_dlsym="1"
AC_CHECK_HEADERS([dlfcn.h],
AC_CHECK_FUNC([dlsym], [],
[AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], [have_dlsym="0"])]),
[have_dlsym="0"])
if test "x$have_dlsym" = "x1" ; then
AC_DEFINE([JEMALLOC_HAVE_DLSYM], [ ])
fi
else
have_dlsym="0"
fi
JE_COMPILABLE([pthread_atfork(3)], [
#include <pthread.h>
], [
@@ -1780,6 +1924,19 @@ JE_COMPILABLE([GCC __atomic atomics], [
], [je_cv_gcc_atomic_atomics])
if test "x${je_cv_gcc_atomic_atomics}" = "xyes" ; then
AC_DEFINE([JEMALLOC_GCC_ATOMIC_ATOMICS])
dnl check for 8-bit atomic support
JE_COMPILABLE([GCC 8-bit __atomic atomics], [
], [
unsigned char x = 0;
int val = 1;
int y = __atomic_fetch_add(&x, val, __ATOMIC_RELAXED);
int after_add = (int)x;
return after_add == 1;
], [je_cv_gcc_u8_atomic_atomics])
if test "x${je_cv_gcc_u8_atomic_atomics}" = "xyes" ; then
AC_DEFINE([JEMALLOC_GCC_U8_ATOMIC_ATOMICS])
fi
fi
dnl ============================================================================
@@ -1794,12 +1951,24 @@ JE_COMPILABLE([GCC __sync atomics], [
], [je_cv_gcc_sync_atomics])
if test "x${je_cv_gcc_sync_atomics}" = "xyes" ; then
AC_DEFINE([JEMALLOC_GCC_SYNC_ATOMICS])
dnl check for 8-bit atomic support
JE_COMPILABLE([GCC 8-bit __sync atomics], [
], [
unsigned char x = 0;
int before_add = __sync_fetch_and_add(&x, 1);
int after_add = (int)x;
return (before_add == 0) && (after_add == 1);
], [je_cv_gcc_u8_sync_atomics])
if test "x${je_cv_gcc_u8_sync_atomics}" = "xyes" ; then
AC_DEFINE([JEMALLOC_GCC_U8_SYNC_ATOMICS])
fi
fi
dnl ============================================================================
dnl Check for atomic(3) operations as provided on Darwin.
dnl We need this not for the atomic operations (which are provided above), but
dnl rather for the OSSpinLock type it exposes.
dnl rather for the OS_unfair_lock type it exposes.
JE_COMPILABLE([Darwin OSAtomic*()], [
#include <libkern/OSAtomic.h>
@@ -1870,7 +2039,7 @@ if test "x${je_cv_madvise}" = "xyes" ; then
if test "x${je_cv_madv_dontdump}" = "xyes" ; then
AC_DEFINE([JEMALLOC_MADVISE_DONTDUMP], [ ])
fi
dnl Check for madvise(..., MADV_[NO]HUGEPAGE).
JE_COMPILABLE([madvise(..., MADV_[[NO]]HUGEPAGE)], [
#include <sys/mman.h>
@@ -1889,40 +2058,6 @@ case "${host_cpu}" in
esac
fi
dnl ============================================================================
dnl Check whether __sync_{add,sub}_and_fetch() are available despite
dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined.
AC_DEFUN([JE_SYNC_COMPARE_AND_SWAP_CHECK],[
AC_CACHE_CHECK([whether to force $1-bit __sync_{add,sub}_and_fetch()],
[je_cv_sync_compare_and_swap_$2],
[AC_LINK_IFELSE([AC_LANG_PROGRAM([
#include <stdint.h>
],
[
#ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2
{
uint$1_t x$1 = 0;
__sync_add_and_fetch(&x$1, 42);
__sync_sub_and_fetch(&x$1, 1);
}
#else
#error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 is defined, no need to force
#endif
])],
[je_cv_sync_compare_and_swap_$2=yes],
[je_cv_sync_compare_and_swap_$2=no])])
if test "x${je_cv_sync_compare_and_swap_$2}" = "xyes" ; then
AC_DEFINE([JE_FORCE_SYNC_COMPARE_AND_SWAP_$2], [ ])
fi
])
if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then
JE_SYNC_COMPARE_AND_SWAP_CHECK(32, 4)
JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8)
fi
dnl ============================================================================
dnl Check for __builtin_clz() and __builtin_clzl().
@@ -1965,21 +2100,6 @@ if test "x${je_cv_os_unfair_lock}" = "xyes" ; then
AC_DEFINE([JEMALLOC_OS_UNFAIR_LOCK], [ ])
fi
dnl ============================================================================
dnl Check for spinlock(3) operations as provided on Darwin.
JE_COMPILABLE([Darwin OSSpin*()], [
#include <libkern/OSAtomic.h>
#include <inttypes.h>
], [
OSSpinLock lock = 0;
OSSpinLockLock(&lock);
OSSpinLockUnlock(&lock);
], [je_cv_osspin])
if test "x${je_cv_osspin}" = "xyes" ; then
AC_DEFINE([JEMALLOC_OSSPIN], [ ])
fi
dnl ============================================================================
dnl Darwin-related configuration.
@@ -2032,9 +2152,7 @@ fi
dnl ============================================================================
dnl Enable background threads if possible.
if test "x${have_pthread}" = "x1" -a "x${have_dlsym}" = "x1" \
-a "x${je_cv_os_unfair_lock}" != "xyes" \
-a "x${je_cv_osspin}" != "xyes" ; then
if test "x${have_pthread}" = "x1" -a "x${je_cv_os_unfair_lock}" != "xyes" ; then
AC_DEFINE([JEMALLOC_BACKGROUND_THREAD])
fi
@@ -2175,16 +2293,6 @@ AC_CONFIG_COMMANDS([include/jemalloc/internal/public_unnamespace.h], [
srcdir="${srcdir}"
objroot="${objroot}"
])
AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [
mkdir -p "${objroot}include/jemalloc/internal"
"${SHELL}" "${srcdir}/include/jemalloc/internal/size_classes.sh" "${LG_QUANTA}" 3 "${LG_PAGE_SIZES}" 2 > "${objroot}include/jemalloc/internal/size_classes.h"
], [
SHELL="${SHELL}"
srcdir="${srcdir}"
objroot="${objroot}"
LG_QUANTA="${LG_QUANTA}"
LG_PAGE_SIZES="${LG_PAGE_SIZES}"
])
AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_protos_jet.h], [
mkdir -p "${objroot}include/jemalloc"
cat "${srcdir}/include/jemalloc/jemalloc_protos.h.in" | sed -e 's/@je_@/jet_/g' > "${objroot}include/jemalloc/jemalloc_protos_jet.h"
@@ -2277,9 +2385,13 @@ AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE])
AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}])
AC_MSG_RESULT([install_suffix : ${install_suffix}])
AC_MSG_RESULT([malloc_conf : ${config_malloc_conf}])
AC_MSG_RESULT([documentation : ${enable_doc}])
AC_MSG_RESULT([shared libs : ${enable_shared}])
AC_MSG_RESULT([static libs : ${enable_static}])
AC_MSG_RESULT([autogen : ${enable_autogen}])
AC_MSG_RESULT([debug : ${enable_debug}])
AC_MSG_RESULT([stats : ${enable_stats}])
AC_MSG_RESULT([experimetal_smallocx : ${enable_experimental_smallocx}])
AC_MSG_RESULT([prof : ${enable_prof}])
AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}])
AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}])
deps/jemalloc/doc/jemalloc.xml.in
Vendored
+156 -22
View File
@@ -424,7 +424,7 @@ for (i = 0; i < nbins; i++) {
called repeatedly. General information that never changes during
execution can be omitted by specifying <quote>g</quote> as a character
within the <parameter>opts</parameter> string. Note that
<function>malloc_message()</function> uses the
<function>malloc_stats_print()</function> uses the
<function>mallctl*()</function> functions internally, so inconsistent
statistics can be reported if multiple threads use these functions
simultaneously. If <option>--enable-stats</option> is specified during
@@ -433,10 +433,11 @@ for (i = 0; i < nbins; i++) {
arena statistics, respectively; <quote>b</quote> and <quote>l</quote> can
be specified to omit per size class statistics for bins and large objects,
respectively; <quote>x</quote> can be specified to omit all mutex
statistics. Unrecognized characters are silently ignored. Note that
thread caching may prevent some statistics from being completely up to
date, since extra locking would be required to merge counters that track
thread cache operations.</para>
statistics; <quote>e</quote> can be used to omit extent statistics.
Unrecognized characters are silently ignored. Note that thread caching
may prevent some statistics from being completely up to date, since extra
locking would be required to merge counters that track thread cache
operations.</para>
<para>The <function>malloc_usable_size()</function> function
returns the usable size of the allocation pointed to by
@@ -903,6 +904,23 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
</para></listitem>
</varlistentry>
<varlistentry id="opt.confirm_conf">
<term>
<mallctl>opt.confirm_conf</mallctl>
(<type>bool</type>)
<literal>r-</literal>
</term>
<listitem><para>Confirm-runtime-options-when-program-starts
enabled/disabled. If true, the string specified via
<option>--with-malloc-conf</option>, the string pointed to by the
global variable <varname>malloc_conf</varname>, the <quote>name</quote>
of the file referenced by the symbolic link named
<filename class="symlink">/etc/malloc.conf</filename>, and the value of
the environment variable <envar>MALLOC_CONF</envar>, will be printed in
order. Then, each option being set will be individually printed. This
option is disabled by default.</para></listitem>
</varlistentry>
<varlistentry id="opt.abort_conf">
<term>
<mallctl>opt.abort_conf</mallctl>
@@ -943,16 +961,19 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
<citerefentry><refentrytitle>munmap</refentrytitle>
<manvolnum>2</manvolnum></citerefentry> or equivalent (see <link
linkend="stats.retained">stats.retained</link> for related details).
This option is disabled by default unless discarding virtual memory is
known to trigger
platform-specific performance problems, e.g. for [64-bit] Linux, which
has a quirk in its virtual memory allocation algorithm that causes
semi-permanent VM map holes under normal jemalloc operation. Although
<citerefentry><refentrytitle>munmap</refentrytitle>
<manvolnum>2</manvolnum></citerefentry> causes issues on 32-bit Linux as
well, retaining virtual memory for 32-bit Linux is disabled by default
due to the practical possibility of address space exhaustion.
</para></listitem>
It also makes jemalloc use <citerefentry>
<refentrytitle>mmap</refentrytitle><manvolnum>2</manvolnum>
</citerefentry> or equivalent in a more greedy way, mapping larger
chunks in one go. This option is disabled by default unless discarding
virtual memory is known to trigger platform-specific performance
problems, namely 1) for [64-bit] Linux, which has a quirk in its virtual
memory allocation algorithm that causes semi-permanent VM map holes
under normal jemalloc operation; and 2) for [64-bit] Windows, which
disallows split / merged regions with
<parameter><constant>MEM_RELEASE</constant></parameter>. Although the
same issues may present on 32-bit platforms as well, retaining virtual
memory for 32-bit Linux and Windows is disabled by default due to the
practical possibility of address space exhaustion. </para></listitem>
</varlistentry>
<varlistentry id="opt.dss">
@@ -988,6 +1009,24 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
number of CPUs, or one if there is a single CPU.</para></listitem>
</varlistentry>
<varlistentry id="opt.oversize_threshold">
<term>
<mallctl>opt.oversize_threshold</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
</term>
<listitem><para>The threshold in bytes of which requests are considered
oversize. Allocation requests with greater sizes are fulfilled from a
dedicated arena (automatically managed, however not within
<literal>narenas</literal>), in order to reduce fragmentation by not
mixing huge allocations with small ones. In addition, the decay API
guarantees on the extents greater than the specified threshold may be
overridden. Note that requests with arena index specified via
<constant>MALLOCX_ARENA</constant>, or threads associated with explicit
arenas will not be considered. The default threshold is 8MiB. Values
not within large size classes disables this feature.</para></listitem>
</varlistentry>
<varlistentry id="opt.percpu_arena">
<term>
<mallctl>opt.percpu_arena</mallctl>
@@ -1009,7 +1048,7 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
<varlistentry id="opt.background_thread">
<term>
<mallctl>opt.background_thread</mallctl>
(<type>const bool</type>)
(<type>bool</type>)
<literal>r-</literal>
</term>
<listitem><para>Internal background worker threads enabled/disabled.
@@ -1024,7 +1063,7 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
<varlistentry id="opt.max_background_threads">
<term>
<mallctl>opt.max_background_threads</mallctl>
(<type>const size_t</type>)
(<type>size_t</type>)
<literal>r-</literal>
</term>
<listitem><para>Maximum number of background threads that will be created
@@ -1055,7 +1094,11 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
linkend="arena.i.dirty_decay_ms"><mallctl>arena.&lt;i&gt;.dirty_decay_ms</mallctl></link>
for related dynamic control options. See <link
linkend="opt.muzzy_decay_ms"><mallctl>opt.muzzy_decay_ms</mallctl></link>
for a description of muzzy pages.</para></listitem>
for a description of muzzy pages.for a description of muzzy pages. Note
that when the <link
linkend="opt.oversize_threshold"><mallctl>oversize_threshold</mallctl></link>
feature is enabled, the arenas reserved for oversize requests may have
its own default decay settings.</para></listitem>
</varlistentry>
<varlistentry id="opt.muzzy_decay_ms">
@@ -1763,10 +1806,11 @@ malloc_conf = "xmalloc:true";]]></programlisting>
to control allocation for arenas explicitly created via <link
linkend="arenas.create"><mallctl>arenas.create</mallctl></link> such
that all extents originate from an application-supplied extent allocator
(by specifying the custom extent hook functions during arena creation),
but the automatically created arenas will have already created extents
prior to the application having an opportunity to take over extent
allocation.</para>
(by specifying the custom extent hook functions during arena creation).
However, the API guarantees for the automatically created arenas may be
relaxed -- hooks set there may be called in a "best effort" fashion; in
addition there may be extents created prior to the application having an
opportunity to take over extent allocation.</para>
<programlisting language="C"><![CDATA[
typedef extent_hooks_s extent_hooks_t;
@@ -2593,6 +2637,17 @@ struct extent_hooks_s {
details.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.extent_avail">
<term>
<mallctl>stats.arenas.&lt;i&gt;.extent_avail</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Number of allocated (but unused) extent structs in this
arena.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.base">
<term>
<mallctl>stats.arenas.&lt;i&gt;.base</mallctl>
@@ -2760,6 +2815,28 @@ struct extent_hooks_s {
all bin size classes.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.small.nfills">
<term>
<mallctl>stats.arenas.&lt;i&gt;.small.nfills</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Cumulative number of tcache fills by all small size
classes.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.small.nflushes">
<term>
<mallctl>stats.arenas.&lt;i&gt;.small.nflushes</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Cumulative number of tcache flushes by all small size
classes.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.large.allocated">
<term>
<mallctl>stats.arenas.&lt;i&gt;.large.allocated</mallctl>
@@ -2810,6 +2887,28 @@ struct extent_hooks_s {
all large size classes.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.large.nfills">
<term>
<mallctl>stats.arenas.&lt;i&gt;.large.nfills</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Cumulative number of tcache fills by all large size
classes.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.large.nflushes">
<term>
<mallctl>stats.arenas.&lt;i&gt;.large.nflushes</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Cumulative number of tcache flushes by all large size
classes.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.bins.j.nmalloc">
<term>
<mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nmalloc</mallctl>
@@ -2909,6 +3008,17 @@ struct extent_hooks_s {
<listitem><para>Current number of slabs.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.bins.j.nonfull_slabs">
<term>
<mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nonfull_slabs</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Current number of nonfull slabs.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.bins.mutex">
<term>
<mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.mutex.{counter}</mallctl>
@@ -2922,6 +3032,30 @@ struct extent_hooks_s {
counters</link>.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.extents.n">
<term>
<mallctl>stats.arenas.&lt;i&gt;.extents.&lt;j&gt;.n{extent_type}</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para> Number of extents of the given type in this arena in
the bucket corresponding to page size index &lt;j&gt;. The extent type
is one of dirty, muzzy, or retained.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.extents.bytes">
<term>
<mallctl>stats.arenas.&lt;i&gt;.extents.&lt;j&gt;.{extent_type}_bytes</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para> Sum of the bytes managed by extents of the given type
in this arena in the bucket corresponding to page size index &lt;j&gt;.
The extent type is one of dirty, muzzy, or retained.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.lextents.j.nmalloc">
<term>
<mallctl>stats.arenas.&lt;i&gt;.lextents.&lt;j&gt;.nmalloc</mallctl>
deps/jemalloc/include/jemalloc/internal/arena_externs.h
Vendored
+18 -8
View File
@@ -3,8 +3,8 @@
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/hook.h"
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/stats.h"
extern ssize_t opt_dirty_decay_ms;
@@ -16,13 +16,17 @@ extern const char *percpu_arena_mode_names[];
extern const uint64_t h_steps[SMOOTHSTEP_NSTEPS];
extern malloc_mutex_t arenas_lock;
extern size_t opt_oversize_threshold;
extern size_t oversize_threshold;
void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena,
unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms,
ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
bin_stats_t *bstats, arena_stats_large_t *lstats);
bin_stats_t *bstats, arena_stats_large_t *lstats,
arena_stats_extents_t *estats);
void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent);
#ifdef JEMALLOC_JET
@@ -56,16 +60,17 @@ void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
szind_t ind, bool zero);
void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, tcache_t *tcache);
void arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize);
void arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize);
void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
bool slow_path);
void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, void *ptr);
void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
szind_t binind, extent_t *extent, void *ptr);
void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero);
size_t extra, bool zero, size_t *newsize);
void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, tcache_t *tcache);
size_t size, size_t alignment, bool zero, tcache_t *tcache,
hook_ralloc_args_t *hook_args);
dss_prec_t arena_dss_prec_get(arena_t *arena);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
ssize_t arena_dirty_decay_ms_default_get(void);
@@ -79,7 +84,12 @@ void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal);
size_t arena_extent_sn_next(arena_t *arena);
arena_t *arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
void arena_boot(void);
bool arena_init_huge(void);
bool arena_is_huge(unsigned arena_ind);
arena_t *arena_choose_huge(tsd_t *tsd);
bin_t *arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind,
unsigned *binshard);
void arena_boot(sc_data_t *sc_data);
void arena_prefork0(tsdn_t *tsdn, arena_t *arena);
void arena_prefork1(tsdn_t *tsdn, arena_t *arena);
void arena_prefork2(tsdn_t *tsdn, arena_t *arena);
deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h
Vendored
+115 -42
View File
@@ -4,10 +4,36 @@
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/ticker.h"
JEMALLOC_ALWAYS_INLINE bool
arena_has_default_hooks(arena_t *arena) {
return (extent_hooks_get(arena) == &extent_hooks_default);
}
JEMALLOC_ALWAYS_INLINE arena_t *
arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
if (arena != NULL) {
return arena;
}
/*
* For huge allocations, use the dedicated huge arena if both are true:
* 1) is using auto arena selection (i.e. arena == NULL), and 2) the
* thread is not assigned to a manual arena.
*/
if (unlikely(size >= oversize_threshold)) {
arena_t *tsd_arena = tsd_arena_get(tsd);
if (tsd_arena == NULL || arena_is_auto(tsd_arena)) {
return arena_choose_huge(tsd);
}
}
return arena_choose(tsd, NULL);
}
JEMALLOC_ALWAYS_INLINE prof_tctx_t *
arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) {
cassert(config_prof);
@@ -28,7 +54,7 @@ arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) {
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize,
arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) {
cassert(config_prof);
assert(ptr != NULL);
@@ -47,7 +73,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize,
}
static inline void
arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) {
arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) {
cassert(config_prof);
assert(ptr != NULL);
@@ -57,6 +83,32 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) {
large_prof_tctx_reset(tsdn, extent);
}
JEMALLOC_ALWAYS_INLINE nstime_t
arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr,
alloc_ctx_t *alloc_ctx) {
cassert(config_prof);
assert(ptr != NULL);
extent_t *extent = iealloc(tsdn, ptr);
/*
* Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're
* sure we have a sampled allocation.
*/
assert(!extent_slab_get(extent));
return large_prof_alloc_time_get(extent);
}
JEMALLOC_ALWAYS_INLINE void
arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx,
nstime_t t) {
cassert(config_prof);
assert(ptr != NULL);
extent_t *extent = iealloc(tsdn, ptr);
assert(!extent_slab_get(extent));
large_prof_alloc_time_set(extent, t);
}
JEMALLOC_ALWAYS_INLINE void
arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
tsd_t *tsd;
@@ -83,14 +135,33 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
arena_decay_ticks(tsdn, arena, 1);
}
/* Purge a single extent to retained / unmapped directly. */
JEMALLOC_ALWAYS_INLINE void
arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_t *extent) {
size_t extent_size = extent_size_get(extent);
extent_dalloc_wrapper(tsdn, arena,
r_extent_hooks, extent);
if (config_stats) {
/* Update stats accordingly. */
arena_stats_lock(tsdn, &arena->stats);
arena_stats_add_u64(tsdn, &arena->stats,
&arena->decay_dirty.stats->nmadvise, 1);
arena_stats_add_u64(tsdn, &arena->stats,
&arena->decay_dirty.stats->purged, extent_size >> LG_PAGE);
arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped,
extent_size);
arena_stats_unlock(tsdn, &arena->stats);
}
}
JEMALLOC_ALWAYS_INLINE void *
arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
tcache_t *tcache, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(size != 0);
if (likely(tcache != NULL)) {
if (likely(size <= SMALL_MAXCLASS)) {
if (likely(size <= SC_SMALL_MAXCLASS)) {
return tcache_alloc_small(tsdn_tsd(tsdn), arena,
tcache, size, ind, zero, slow_path);
}
@@ -119,7 +190,7 @@ arena_salloc(tsdn_t *tsdn, const void *ptr) {
szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true);
assert(szind != NSIZES);
assert(szind != SC_NSIZES);
return sz_index2size(szind);
}
@@ -152,11 +223,21 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
/* Only slab members should be looked up via interior pointers. */
assert(extent_addr_get(extent) == ptr || extent_slab_get(extent));
assert(szind != NSIZES);
assert(szind != SC_NSIZES);
return sz_index2size(szind);
}
static inline void
arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind) {
if (config_prof && unlikely(szind < SC_NBINS)) {
arena_dalloc_promoted(tsdn, ptr, NULL, true);
} else {
extent_t *extent = iealloc(tsdn, ptr);
large_dalloc(tsdn, extent);
}
}
static inline void
arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
assert(ptr != NULL);
@@ -173,13 +254,28 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
rtree_ctx, (uintptr_t)ptr, true);
assert(szind == extent_szind_get(extent));
assert(szind < NSIZES);
assert(szind < SC_NSIZES);
assert(slab == extent_slab_get(extent));
}
if (likely(slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
arena_dalloc_large_no_tcache(tsdn, ptr, szind);
}
}
JEMALLOC_ALWAYS_INLINE void
arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
bool slow_path) {
if (szind < nhbins) {
if (config_prof && unlikely(szind < SC_NBINS)) {
arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
} else {
tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind,
slow_path);
}
} else {
extent_t *extent = iealloc(tsdn, ptr);
large_dalloc(tsdn, extent);
@@ -203,7 +299,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
if (alloc_ctx != NULL) {
szind = alloc_ctx->szind;
slab = alloc_ctx->slab;
assert(szind != NSIZES);
assert(szind != SC_NSIZES);
} else {
rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
@@ -215,7 +311,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
rtree_ctx, (uintptr_t)ptr, true);
assert(szind == extent_szind_get(extent));
assert(szind < NSIZES);
assert(szind < SC_NSIZES);
assert(slab == extent_slab_get(extent));
}
@@ -224,25 +320,14 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
slow_path);
} else {
if (szind < nhbins) {
if (config_prof && unlikely(szind < NBINS)) {
arena_dalloc_promoted(tsdn, ptr, tcache,
slow_path);
} else {
tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
szind, slow_path);
}
} else {
extent_t *extent = iealloc(tsdn, ptr);
large_dalloc(tsdn, extent);
}
arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path);
}
}
static inline void
arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
assert(ptr != NULL);
assert(size <= LARGE_MAXCLASS);
assert(size <= SC_LARGE_MAXCLASS);
szind_t szind;
bool slab;
@@ -252,7 +337,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
* object, so base szind and slab on the given size.
*/
szind = sz_size2index(size);
slab = (szind < NBINS);
slab = (szind < SC_NBINS);
}
if ((config_prof && opt_prof) || config_debug) {
@@ -264,7 +349,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
(uintptr_t)ptr, true, &szind, &slab);
assert(szind == sz_size2index(size));
assert((config_prof && opt_prof) || slab == (szind < NBINS));
assert((config_prof && opt_prof) || slab == (szind < SC_NBINS));
if (config_debug) {
extent_t *extent = rtree_extent_read(tsdn,
@@ -278,8 +363,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
extent_t *extent = iealloc(tsdn, ptr);
large_dalloc(tsdn, extent);
arena_dalloc_large_no_tcache(tsdn, ptr, szind);
}
}
@@ -288,7 +372,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
alloc_ctx_t *alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
assert(size <= LARGE_MAXCLASS);
assert(size <= SC_LARGE_MAXCLASS);
if (unlikely(tcache == NULL)) {
arena_sdalloc_no_tcache(tsdn, ptr, size);
@@ -297,7 +381,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
szind_t szind;
bool slab;
UNUSED alloc_ctx_t local_ctx;
alloc_ctx_t local_ctx;
if (config_prof && opt_prof) {
if (alloc_ctx == NULL) {
/* Uncommon case and should be a static check. */
@@ -318,7 +402,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
* object, so base szind and slab on the given size.
*/
szind = sz_size2index(size);
slab = (szind < NBINS);
slab = (szind < SC_NBINS);
}
if (config_debug) {
@@ -336,18 +420,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
slow_path);
} else {
if (szind < nhbins) {
if (config_prof && unlikely(szind < NBINS)) {
arena_dalloc_promoted(tsdn, ptr, tcache,
slow_path);
} else {
tcache_dalloc_large(tsdn_tsd(tsdn),
tcache, ptr, szind, slow_path);
}
} else {
extent_t *extent = iealloc(tsdn, ptr);
large_dalloc(tsdn, extent);
}
arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path);
}
}
deps/jemalloc/include/jemalloc/internal/arena_stats.h
Vendored
+49 -15
View File
@@ -4,7 +4,9 @@
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mutex_prof.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/sc.h"
JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
/*
* In those architectures that support 64-bit atomics, we use atomic updates for
@@ -33,6 +35,13 @@ struct arena_stats_large_s {
* periodically merges into this counter.
*/
arena_stats_u64_t nrequests; /* Partially derived. */
/*
* Number of tcache fills / flushes for large (similarly, periodically
* merged). Note that there is no large tcache batch-fill currently
* (i.e. only fill 1 at a time); however flush may be batched.
*/
arena_stats_u64_t nfills; /* Partially derived. */
arena_stats_u64_t nflushes; /* Partially derived. */
/* Current number of allocations of this size class. */
size_t curlextents; /* Derived. */
@@ -48,6 +57,22 @@ struct arena_stats_decay_s {
arena_stats_u64_t purged;
};
typedef struct arena_stats_extents_s arena_stats_extents_t;
struct arena_stats_extents_s {
/*
* Stats for a given index in the range [0, SC_NPSIZES] in an extents_t.
* We track both bytes and # of extents: two extents in the same bucket
* may have different sizes if adjacent size classes differ by more than
* a page, so bytes cannot always be derived from # of extents.
*/
atomic_zu_t ndirty;
atomic_zu_t dirty_bytes;
atomic_zu_t nmuzzy;
atomic_zu_t muzzy_bytes;
atomic_zu_t nretained;
atomic_zu_t retained_bytes;
};
/*
* Arena stats. Note that fields marked "derived" are not directly maintained
* within the arena code; rather their values are derived during stats merge
@@ -69,6 +94,9 @@ struct arena_stats_s {
*/
atomic_zu_t retained; /* Derived. */
/* Number of extent_t structs allocated by base, but not being used. */
atomic_zu_t extent_avail;
arena_stats_decay_t decay_dirty;
arena_stats_decay_t decay_muzzy;
@@ -80,22 +108,27 @@ struct arena_stats_s {
atomic_zu_t allocated_large; /* Derived. */
arena_stats_u64_t nmalloc_large; /* Derived. */
arena_stats_u64_t ndalloc_large; /* Derived. */
arena_stats_u64_t nfills_large; /* Derived. */
arena_stats_u64_t nflushes_large; /* Derived. */
arena_stats_u64_t nrequests_large; /* Derived. */
/* VM space had to be leaked (undocumented). Normally 0. */
atomic_zu_t abandoned_vm;
/* Number of bytes cached in tcache associated with this arena. */
atomic_zu_t tcache_bytes; /* Derived. */
mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes];
/* One element for each large size class. */
arena_stats_large_t lstats[NSIZES - NBINS];
arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
/* Arena uptime. */
nstime_t uptime;
};
static inline bool
arena_stats_init(UNUSED tsdn_t *tsdn, arena_stats_t *arena_stats) {
arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) {
if (config_debug) {
for (size_t i = 0; i < sizeof(arena_stats_t); i++) {
assert(((char *)arena_stats)[i] == 0);
@@ -147,11 +180,11 @@ arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
#endif
}
UNUSED static inline void
static inline void
arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats,
arena_stats_u64_t *p, uint64_t x) {
#ifdef JEMALLOC_ATOMIC_U64
UNUSED uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED);
uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED);
assert(r - x <= r);
#else
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
@@ -176,7 +209,8 @@ arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) {
}
static inline size_t
arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) {
arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
atomic_zu_t *p) {
#ifdef JEMALLOC_ATOMIC_U64
return atomic_load_zu(p, ATOMIC_RELAXED);
#else
@@ -186,8 +220,8 @@ arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) {
}
static inline void
arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p,
size_t x) {
arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
atomic_zu_t *p, size_t x) {
#ifdef JEMALLOC_ATOMIC_U64
atomic_fetch_add_zu(p, x, ATOMIC_RELAXED);
#else
@@ -198,10 +232,10 @@ arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p,
}
static inline void
arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p,
size_t x) {
arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats,
atomic_zu_t *p, size_t x) {
#ifdef JEMALLOC_ATOMIC_U64
UNUSED size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED);
size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED);
assert(r - x <= r);
#else
malloc_mutex_assert_owner(tsdn, &arena_stats->mtx);
@@ -218,11 +252,12 @@ arena_stats_accum_zu(atomic_zu_t *dst, size_t src) {
}
static inline void
arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
arena_stats_large_flush_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
szind_t szind, uint64_t nrequests) {
arena_stats_lock(tsdn, arena_stats);
arena_stats_add_u64(tsdn, arena_stats, &arena_stats->lstats[szind -
NBINS].nrequests, nrequests);
arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS];
arena_stats_add_u64(tsdn, arena_stats, &lstats->nrequests, nrequests);
arena_stats_add_u64(tsdn, arena_stats, &lstats->nflushes, 1);
arena_stats_unlock(tsdn, arena_stats);
}
@@ -233,5 +268,4 @@ arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) {
arena_stats_unlock(tsdn, arena_stats);
}
#endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */
deps/jemalloc/include/jemalloc/internal/arena_structs_b.h
Vendored
+6 -3
View File
@@ -10,7 +10,7 @@
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/smoothstep.h"
#include "jemalloc/internal/ticker.h"
@@ -90,6 +90,9 @@ struct arena_s {
*/
atomic_u_t nthreads[2];
/* Next bin shard for binding new threads. Synchronization: atomic. */
atomic_u_t binshard_next;
/*
* When percpu_arena is enabled, to amortize the cost of reading /
* updating the current CPU id, track the most recent thread accessing
@@ -113,7 +116,6 @@ struct arena_s {
/* Synchronization: internal. */
prof_accum_t prof_accum;
uint64_t prof_accumbytes;
/*
* PRNG state for cache index randomization of large allocation base
@@ -196,6 +198,7 @@ struct arena_s {
* Synchronization: extent_avail_mtx.
*/
extent_tree_t extent_avail;
atomic_zu_t extent_avail_cnt;
malloc_mutex_t extent_avail_mtx;
/*
@@ -203,7 +206,7 @@ struct arena_s {
*
* Synchronization: internal.
*/
bin_t bins[NBINS];
bins_t bins[SC_NBINS];
/*
* Base allocator, from which arena metadata are allocated.
deps/jemalloc/include/jemalloc/internal/arena_types.h
Vendored
+10 -2
View File
@@ -1,13 +1,15 @@
#ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H
#define JEMALLOC_INTERNAL_ARENA_TYPES_H
#include "jemalloc/internal/sc.h"
/* Maximum number of regions in one slab. */
#define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN)
#define LG_SLAB_MAXREGS (LG_PAGE - SC_LG_TINY_MIN)
#define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS)
/* Default decay times in milliseconds. */
#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000)
#define MUZZY_DECAY_MS_DEFAULT ZD(10 * 1000)
#define MUZZY_DECAY_MS_DEFAULT (0)
/* Number of event ticks between time checks. */
#define DECAY_NTICKS_PER_UPDATE 1000
@@ -40,4 +42,10 @@ typedef enum {
#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base)
#define PERCPU_ARENA_DEFAULT percpu_arena_disabled
/*
* When allocation_size >= oversize_threshold, use the dedicated huge arena
* (unless have explicitly spicified arena index). 0 disables the feature.
*/
#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20)
#endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */
deps/jemalloc/include/jemalloc/internal/atomic.h
Vendored
+10 -1
View File
@@ -1,12 +1,19 @@
#ifndef JEMALLOC_INTERNAL_ATOMIC_H
#define JEMALLOC_INTERNAL_ATOMIC_H
#define ATOMIC_INLINE static inline
#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
#define JEMALLOC_U8_ATOMICS
#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS)
# include "jemalloc/internal/atomic_gcc_atomic.h"
# if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS)
# undef JEMALLOC_U8_ATOMICS
# endif
#elif defined(JEMALLOC_GCC_SYNC_ATOMICS)
# include "jemalloc/internal/atomic_gcc_sync.h"
# if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS)
# undef JEMALLOC_U8_ATOMICS
# endif
#elif defined(_MSC_VER)
# include "jemalloc/internal/atomic_msvc.h"
#elif defined(JEMALLOC_C11_ATOMICS)
@@ -66,6 +73,8 @@ JEMALLOC_GENERATE_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR)
JEMALLOC_GENERATE_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR)
JEMALLOC_GENERATE_INT_ATOMICS(uint8_t, u8, 0)
JEMALLOC_GENERATE_INT_ATOMICS(uint32_t, u32, 2)
#ifdef JEMALLOC_ATOMIC_U64
deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h
Vendored
+4 -2
View File
@@ -67,7 +67,8 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
UNUSED type *expected, type desired, \
atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
return __atomic_compare_exchange(&a->repr, expected, &desired, \
true, atomic_enum_to_builtin(success_mo), \
@@ -76,7 +77,8 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
UNUSED type *expected, type desired, \
atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
return __atomic_compare_exchange(&a->repr, expected, &desired, \
false, \
deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h
Vendored
+11 -7
View File
@@ -27,8 +27,10 @@ atomic_fence(atomic_memory_order_t mo) {
asm volatile("" ::: "memory");
# if defined(__i386__) || defined(__x86_64__)
/* This is implicit on x86. */
# elif defined(__ppc__)
# elif defined(__ppc64__)
asm volatile("lwsync");
# elif defined(__ppc__)
asm volatile("sync");
# elif defined(__sparc__) && defined(__arch64__)
if (mo == atomic_memory_order_acquire) {
asm volatile("membar #LoadLoad | #LoadStore");
@@ -113,8 +115,8 @@ atomic_store_##short_type(atomic_##short_type##_t *a, \
} \
\
ATOMIC_INLINE type \
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
atomic_memory_order_t mo) { \
/* \
* Because of FreeBSD, we care about gcc 4.2, which doesn't have\
* an atomic exchange builtin. We fake it with a CAS loop. \
@@ -129,8 +131,9 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
\
ATOMIC_INLINE bool \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
type *expected, type desired, \
atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
type prev = __sync_val_compare_and_swap(&a->repr, *expected, \
desired); \
if (prev == *expected) { \
@@ -142,8 +145,9 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
} \
ATOMIC_INLINE bool \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
type *expected, type desired, atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
type *expected, type desired, \
atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \
type prev = __sync_val_compare_and_swap(&a->repr, *expected, \
desired); \
if (prev == *expected) { \
deps/jemalloc/include/jemalloc/internal/background_thread_externs.h
Vendored
-1
View File
@@ -8,7 +8,6 @@ extern atomic_b_t background_thread_enabled_state;
extern size_t n_background_threads;
extern size_t max_background_threads;
extern background_thread_info_t *background_thread_info;
extern bool can_enable_background_thread;
bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
bool background_threads_enable(tsd_t *tsd);
deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h
Vendored
+6 -1
View File
@@ -15,7 +15,12 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) {
JEMALLOC_ALWAYS_INLINE background_thread_info_t *
arena_background_thread_info_get(arena_t *arena) {
unsigned arena_ind = arena_ind_get(arena);
return &background_thread_info[arena_ind % ncpus];
return &background_thread_info[arena_ind % max_background_threads];
}
JEMALLOC_ALWAYS_INLINE background_thread_info_t *
background_thread_info_get(size_t ind) {
return &background_thread_info[ind % max_background_threads];
}
JEMALLOC_ALWAYS_INLINE uint64_t
deps/jemalloc/include/jemalloc/internal/background_thread_structs.h
Vendored
+1
View File
@@ -9,6 +9,7 @@
#define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX
#define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT
#define DEFAULT_NUM_BACKGROUND_THREAD 4
typedef enum {
background_thread_stopped,
deps/jemalloc/include/jemalloc/internal/base_structs.h
Vendored
+2 -2
View File
@@ -3,7 +3,7 @@
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/sc.h"
/* Embedded at the beginning of every block of base-managed virtual memory. */
struct base_block_s {
@@ -46,7 +46,7 @@ struct base_s {
base_block_t *blocks;
/* Heap of extents that track unused trailing space within blocks. */
extent_heap_t avail[NSIZES];
extent_heap_t avail[SC_NSIZES];
/* Stats, only maintained if config_stats. */
size_t allocated;
deps/jemalloc/include/jemalloc/internal/bin.h
Vendored
+21 -4
View File
@@ -1,10 +1,12 @@
#ifndef JEMALLOC_INTERNAL_BIN_H
#define JEMALLOC_INTERNAL_BIN_H
#include "jemalloc/internal/bin_stats.h"
#include "jemalloc/internal/bin_types.h"
#include "jemalloc/internal/extent_types.h"
#include "jemalloc/internal/extent_structs.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/bin_stats.h"
#include "jemalloc/internal/sc.h"
/*
* A bin contains a set of extents that are currently being used for slab
@@ -41,6 +43,9 @@ struct bin_info_s {
/* Total number of regions in a slab for this bin's size class. */
uint32_t nregs;
/* Number of sharded bins in each arena for this size class. */
uint32_t n_shards;
/*
* Metadata used to manipulate bitmaps for slabs associated with this
* bin.
@@ -48,8 +53,7 @@ struct bin_info_s {
bitmap_info_t bitmap_info;
};
extern const bin_info_t bin_infos[NBINS];
extern bin_info_t bin_infos[SC_NBINS];
typedef struct bin_s bin_t;
struct bin_s {
@@ -78,6 +82,18 @@ struct bin_s {
bin_stats_t stats;
};
/* A set of sharded bins of the same size class. */
typedef struct bins_s bins_t;
struct bins_s {
/* Sharded bins. Dynamically sized. */
bin_t *bin_shards;
};
void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]);
bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size,
size_t end_size, size_t nshards);
void bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]);
/* Initializes a bin to empty. Returns true on error. */
bool bin_init(bin_t *bin);
@@ -90,7 +106,7 @@ void bin_postfork_child(tsdn_t *tsdn, bin_t *bin);
static inline void
bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) {
malloc_mutex_lock(tsdn, &bin->lock);
malloc_mutex_prof_read(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
dst_bin_stats->nmalloc += bin->stats.nmalloc;
dst_bin_stats->ndalloc += bin->stats.ndalloc;
dst_bin_stats->nrequests += bin->stats.nrequests;
@@ -100,6 +116,7 @@ bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) {
dst_bin_stats->nslabs += bin->stats.nslabs;
dst_bin_stats->reslabs += bin->stats.reslabs;
dst_bin_stats->curslabs += bin->stats.curslabs;
dst_bin_stats->nonfull_slabs += bin->stats.nonfull_slabs;
malloc_mutex_unlock(tsdn, &bin->lock);
}
deps/jemalloc/include/jemalloc/internal/bin_stats.h
Vendored
+3
View File
@@ -45,6 +45,9 @@ struct bin_stats_s {
/* Current number of slabs in this bin. */
size_t curslabs;
/* Current size of nonfull slabs heap in this bin. */
size_t nonfull_slabs;
mutex_prof_data_t mutex_data;
};
deps/jemalloc/include/jemalloc/internal/bin_types.h
Vendored
+17
View File
@@ -0,0 +1,17 @@
#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H
#define JEMALLOC_INTERNAL_BIN_TYPES_H
#include "jemalloc/internal/sc.h"
#define BIN_SHARDS_MAX (1 << EXTENT_BITS_BINSHARD_WIDTH)
#define N_BIN_SHARDS_DEFAULT 1
/* Used in TSD static initializer only. Real init in arena_bind(). */
#define TSD_BINSHARDS_ZERO_INITIALIZER {{UINT8_MAX}}
typedef struct tsd_binshards_s tsd_binshards_t;
struct tsd_binshards_s {
uint8_t binshard[SC_NBINS];
};
#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */
deps/jemalloc/include/jemalloc/internal/bit_util.h
Vendored
+74
View File
@@ -27,6 +27,25 @@ ffs_u(unsigned bitmap) {
return JEMALLOC_INTERNAL_FFS(bitmap);
}
#ifdef JEMALLOC_INTERNAL_POPCOUNTL
BIT_UTIL_INLINE unsigned
popcount_lu(unsigned long bitmap) {
return JEMALLOC_INTERNAL_POPCOUNTL(bitmap);
}
#endif
/*
* Clears first unset bit in bitmap, and returns
* place of bit. bitmap *must not* be 0.
*/
BIT_UTIL_INLINE size_t
cfs_lu(unsigned long* bitmap) {
size_t bit = ffs_lu(*bitmap) - 1;
*bitmap ^= ZU(1) << bit;
return bit;
}
BIT_UTIL_INLINE unsigned
ffs_zu(size_t bitmap) {
#if LG_SIZEOF_PTR == LG_SIZEOF_INT
@@ -63,6 +82,22 @@ ffs_u32(uint32_t bitmap) {
BIT_UTIL_INLINE uint64_t
pow2_ceil_u64(uint64_t x) {
#if (defined(__amd64__) || defined(__x86_64__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ))
if(unlikely(x <= 1)) {
return x;
}
size_t msb_on_index;
#if (defined(__amd64__) || defined(__x86_64__))
asm ("bsrq %1, %0"
: "=r"(msb_on_index) // Outputs.
: "r"(x-1) // Inputs.
);
#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
msb_on_index = (63 ^ __builtin_clzll(x - 1));
#endif
assert(msb_on_index < 63);
return 1ULL << (msb_on_index + 1);
#else
x--;
x |= x >> 1;
x |= x >> 2;
@@ -72,10 +107,27 @@ pow2_ceil_u64(uint64_t x) {
x |= x >> 32;
x++;
return x;
#endif
}
BIT_UTIL_INLINE uint32_t
pow2_ceil_u32(uint32_t x) {
#if ((defined(__i386__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) && (!defined(__s390__)))
if(unlikely(x <= 1)) {
return x;
}
size_t msb_on_index;
#if (defined(__i386__))
asm ("bsr %1, %0"
: "=r"(msb_on_index) // Outputs.
: "r"(x-1) // Inputs.
);
#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ))
msb_on_index = (31 ^ __builtin_clz(x - 1));
#endif
assert(msb_on_index < 31);
return 1U << (msb_on_index + 1);
#else
x--;
x |= x >> 1;
x |= x >> 2;
@@ -84,6 +136,7 @@ pow2_ceil_u32(uint32_t x) {
x |= x >> 16;
x++;
return x;
#endif
}
/* Compute the smallest power of 2 that is >= x. */
@@ -160,6 +213,27 @@ lg_floor(size_t x) {
}
#endif
BIT_UTIL_INLINE unsigned
lg_ceil(size_t x) {
return lg_floor(x) + ((x & (x - 1)) == 0 ? 0 : 1);
}
#undef BIT_UTIL_INLINE
/* A compile-time version of lg_floor and lg_ceil. */
#define LG_FLOOR_1(x) 0
#define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1))
#define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2))
#define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4))
#define LG_FLOOR_16(x) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8))
#define LG_FLOOR_32(x) (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16))
#define LG_FLOOR_64(x) (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32))
#if LG_SIZEOF_PTR == 2
# define LG_FLOOR(x) LG_FLOOR_32((x))
#else
# define LG_FLOOR(x) LG_FLOOR_64((x))
#endif
#define LG_CEIL(x) (LG_FLOOR(x) + (((x) & ((x) - 1)) == 0 ? 0 : 1))
#endif /* JEMALLOC_INTERNAL_BIT_UTIL_H */
deps/jemalloc/include/jemalloc/internal/bitmap.h
Vendored
+3 -3
View File
@@ -3,18 +3,18 @@
#include "jemalloc/internal/arena_types.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/sc.h"
typedef unsigned long bitmap_t;
#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG
/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
#if LG_SLAB_MAXREGS > LG_CEIL_NSIZES
#if LG_SLAB_MAXREGS > LG_CEIL(SC_NSIZES)
/* Maximum bitmap bit count is determined by maximum regions per slab. */
# define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS
#else
/* Maximum bitmap bit count is determined by number of extent size classes. */
# define LG_BITMAP_MAXBITS LG_CEIL_NSIZES
# define LG_BITMAP_MAXBITS LG_CEIL(SC_NSIZES)
#endif
#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS)
deps/jemalloc/include/jemalloc/internal/cache_bin.h
Vendored
+27 -10
View File
@@ -88,11 +88,21 @@ JEMALLOC_ALWAYS_INLINE void *
cache_bin_alloc_easy(cache_bin_t *bin, bool *success) {
void *ret;
if (unlikely(bin->ncached == 0)) {
bin->low_water = -1;
*success = false;
return NULL;
bin->ncached--;
/*
* Check for both bin->ncached == 0 and ncached < low_water
* in a single branch.
*/
if (unlikely(bin->ncached <= bin->low_water)) {
bin->low_water = bin->ncached;
if (bin->ncached == -1) {
bin->ncached = 0;
*success = false;
return NULL;
}
}
/*
* success (instead of ret) should be checked upon the return of this
* function. We avoid checking (ret == NULL) because there is never a
@@ -101,14 +111,21 @@ cache_bin_alloc_easy(cache_bin_t *bin, bool *success) {
* cacheline).
*/
*success = true;
ret = *(bin->avail - bin->ncached);
bin->ncached--;
if (unlikely(bin->ncached < bin->low_water)) {
bin->low_water = bin->ncached;
}
ret = *(bin->avail - (bin->ncached + 1));
return ret;
}
JEMALLOC_ALWAYS_INLINE bool
cache_bin_dalloc_easy(cache_bin_t *bin, cache_bin_info_t *bin_info, void *ptr) {
if (unlikely(bin->ncached == bin_info->ncached_max)) {
return false;
}
assert(bin->ncached < bin_info->ncached_max);
bin->ncached++;
*(bin->avail - bin->ncached) = ptr;
return true;
}
#endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */
deps/jemalloc/include/jemalloc/internal/ctl.h
Vendored
+6 -3
View File
@@ -5,7 +5,7 @@
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex_prof.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/stats.h"
/* Maximum ctl tree depth. */
@@ -39,9 +39,12 @@ typedef struct ctl_arena_stats_s {
uint64_t nmalloc_small;
uint64_t ndalloc_small;
uint64_t nrequests_small;
uint64_t nfills_small;
uint64_t nflushes_small;
bin_stats_t bstats[NBINS];
arena_stats_large_t lstats[NSIZES - NBINS];
bin_stats_t bstats[SC_NBINS];
arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
arena_stats_extents_t estats[SC_NPSIZES];
} ctl_arena_stats_t;
typedef struct ctl_stats_s {
deps/jemalloc/include/jemalloc/internal/emitter.h
Vendored
+244 -193
View File
@@ -45,7 +45,9 @@ struct emitter_col_s {
int int_val;
unsigned unsigned_val;
uint32_t uint32_val;
uint32_t uint32_t_val;
uint64_t uint64_val;
uint64_t uint64_t_val;
size_t size_val;
ssize_t ssize_val;
const char *str_val;
@@ -60,17 +62,6 @@ struct emitter_row_s {
ql_head(emitter_col_t) cols;
};
static inline void
emitter_row_init(emitter_row_t *row) {
ql_new(&row->cols);
}
static inline void
emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
ql_elm_new(col, link);
ql_tail_insert(&row->cols, col, link);
}
typedef struct emitter_s emitter_t;
struct emitter_s {
emitter_output_t output;
@@ -80,18 +71,10 @@ struct emitter_s {
int nesting_depth;
/* True if we've already emitted a value at the given depth. */
bool item_at_depth;
/* True if we emitted a key and will emit corresponding value next. */
bool emitted_key;
};
static inline void
emitter_init(emitter_t *emitter, emitter_output_t emitter_output,
void (*write_cb)(void *, const char *), void *cbopaque) {
emitter->output = emitter_output;
emitter->write_cb = write_cb;
emitter->cbopaque = cbopaque;
emitter->item_at_depth = false;
emitter->nesting_depth = 0;
}
/* Internal convenience function. Write to the emitter the given string. */
JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void
@@ -103,22 +86,11 @@ emitter_printf(emitter_t *emitter, const char *format, ...) {
va_end(ap);
}
/* Write to the emitter the given string, but only in table mode. */
JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void
emitter_table_printf(emitter_t *emitter, const char *format, ...) {
if (emitter->output == emitter_output_table) {
va_list ap;
va_start(ap, format);
malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap);
va_end(ap);
}
}
static inline void
static inline const char * JEMALLOC_FORMAT_ARG(3)
emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier,
emitter_justify_t justify, int width) {
size_t written;
fmt_specifier++;
if (justify == emitter_justify_none) {
written = malloc_snprintf(out_fmt, out_size,
"%%%s", fmt_specifier);
@@ -131,6 +103,7 @@ emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier,
}
/* Only happens in case of bad format string, which *we* choose. */
assert(written < out_size);
return out_fmt;
}
/*
@@ -156,26 +129,27 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
char buf[BUF_SIZE];
#define EMIT_SIMPLE(type, format) \
emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width); \
emitter_printf(emitter, fmt, *(const type *)value); \
emitter_printf(emitter, \
emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width), \
*(const type *)value);
switch (value_type) {
case emitter_type_bool:
emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width);
emitter_printf(emitter, fmt, *(const bool *)value ?
"true" : "false");
emitter_printf(emitter,
emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width),
*(const bool *)value ? "true" : "false");
break;
case emitter_type_int:
EMIT_SIMPLE(int, "d")
EMIT_SIMPLE(int, "%d")
break;
case emitter_type_unsigned:
EMIT_SIMPLE(unsigned, "u")
EMIT_SIMPLE(unsigned, "%u")
break;
case emitter_type_ssize:
EMIT_SIMPLE(ssize_t, "zd")
EMIT_SIMPLE(ssize_t, "%zd")
break;
case emitter_type_size:
EMIT_SIMPLE(size_t, "zu")
EMIT_SIMPLE(size_t, "%zu")
break;
case emitter_type_string:
str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"",
@@ -185,17 +159,17 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
* anywhere near the fmt size.
*/
assert(str_written < BUF_SIZE);
emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width);
emitter_printf(emitter, fmt, buf);
emitter_printf(emitter,
emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), buf);
break;
case emitter_type_uint32:
EMIT_SIMPLE(uint32_t, FMTu32)
EMIT_SIMPLE(uint32_t, "%" FMTu32)
break;
case emitter_type_uint64:
EMIT_SIMPLE(uint64_t, FMTu64)
EMIT_SIMPLE(uint64_t, "%" FMTu64)
break;
case emitter_type_title:
EMIT_SIMPLE(char *const, "s");
EMIT_SIMPLE(char *const, "%s");
break;
default:
unreachable();
@@ -235,47 +209,143 @@ emitter_indent(emitter_t *emitter) {
static inline void
emitter_json_key_prefix(emitter_t *emitter) {
if (emitter->emitted_key) {
emitter->emitted_key = false;
return;
}
emitter_printf(emitter, "%s\n", emitter->item_at_depth ? "," : "");
emitter_indent(emitter);
}
static inline void
emitter_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth == 0);
emitter_printf(emitter, "{");
emitter_nest_inc(emitter);
} else {
// tabular init
emitter_printf(emitter, "%s", "");
}
}
/******************************************************************************/
/* Public functions for emitter_t. */
static inline void
emitter_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth == 1);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n}\n");
}
emitter_init(emitter_t *emitter, emitter_output_t emitter_output,
void (*write_cb)(void *, const char *), void *cbopaque) {
emitter->output = emitter_output;
emitter->write_cb = write_cb;
emitter->cbopaque = cbopaque;
emitter->item_at_depth = false;
emitter->emitted_key = false;
emitter->nesting_depth = 0;
}
/*
* Note emits a different kv pair as well, but only in table mode. Omits the
* note if table_note_key is NULL.
/******************************************************************************/
/* JSON public API. */
/*
* Emits a key (e.g. as appears in an object). The next json entity emitted will
* be the corresponding value.
*/
static inline void
emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_json_key(emitter_t *emitter, const char *json_key) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": ", json_key);
emitter->emitted_key = true;
}
}
static inline void
emitter_json_value(emitter_t *emitter, emitter_type_t value_type,
const void *value) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value);
emitter->item_at_depth = true;
}
}
/* Shorthand for calling emitter_json_key and then emitter_json_value. */
static inline void
emitter_json_kv(emitter_t *emitter, const char *json_key,
emitter_type_t value_type, const void *value) {
emitter_json_key(emitter, json_key);
emitter_json_value(emitter, value_type, value);
}
static inline void
emitter_json_array_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "[");
emitter_nest_inc(emitter);
}
}
/* Shorthand for calling emitter_json_key and then emitter_json_array_begin. */
static inline void
emitter_json_array_kv_begin(emitter_t *emitter, const char *json_key) {
emitter_json_key(emitter, json_key);
emitter_json_array_begin(emitter);
}
static inline void
emitter_json_array_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n");
emitter_indent(emitter);
emitter_printf(emitter, "]");
}
}
static inline void
emitter_json_object_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "{");
emitter_nest_inc(emitter);
}
}
/* Shorthand for calling emitter_json_key and then emitter_json_object_begin. */
static inline void
emitter_json_object_kv_begin(emitter_t *emitter, const char *json_key) {
emitter_json_key(emitter, json_key);
emitter_json_object_begin(emitter);
}
static inline void
emitter_json_object_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n");
emitter_indent(emitter);
emitter_printf(emitter, "}");
}
}
/******************************************************************************/
/* Table public API. */
static inline void
emitter_table_dict_begin(emitter_t *emitter, const char *table_key) {
if (emitter->output == emitter_output_table) {
emitter_indent(emitter);
emitter_printf(emitter, "%s\n", table_key);
emitter_nest_inc(emitter);
}
}
static inline void
emitter_table_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_table) {
emitter_nest_dec(emitter);
}
}
static inline void
emitter_table_kv_note(emitter_t *emitter, const char *table_key,
emitter_type_t value_type, const void *value,
const char *table_note_key, emitter_type_t table_note_value_type,
const void *table_note_value) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": ", json_key);
emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value);
} else {
if (emitter->output == emitter_output_table) {
emitter_indent(emitter);
emitter_printf(emitter, "%s: ", table_key);
emitter_print_value(emitter, emitter_justify_none, -1,
@@ -292,130 +362,22 @@ emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key,
}
static inline void
emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_table_kv(emitter_t *emitter, const char *table_key,
emitter_type_t value_type, const void *value) {
emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL,
emitter_table_kv_note(emitter, table_key, value_type, value, NULL,
emitter_type_bool, NULL);
}
static inline void
emitter_json_kv(emitter_t *emitter, const char *json_key,
emitter_type_t value_type, const void *value) {
if (emitter->output == emitter_output_json) {
emitter_kv(emitter, json_key, NULL, value_type, value);
}
}
/* Write to the emitter the given string, but only in table mode. */
JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void
emitter_table_kv(emitter_t *emitter, const char *table_key,
emitter_type_t value_type, const void *value) {
emitter_table_printf(emitter_t *emitter, const char *format, ...) {
if (emitter->output == emitter_output_table) {
emitter_kv(emitter, NULL, table_key, value_type, value);
}
}
static inline void
emitter_dict_begin(emitter_t *emitter, const char *json_key,
const char *table_header) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": {", json_key);
emitter_nest_inc(emitter);
} else {
emitter_indent(emitter);
emitter_printf(emitter, "%s\n", table_header);
emitter_nest_inc(emitter);
}
}
static inline void
emitter_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n");
emitter_indent(emitter);
emitter_printf(emitter, "}");
} else {
emitter_nest_dec(emitter);
}
}
static inline void
emitter_json_dict_begin(emitter_t *emitter, const char *json_key) {
if (emitter->output == emitter_output_json) {
emitter_dict_begin(emitter, json_key, NULL);
}
}
static inline void
emitter_json_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
emitter_dict_end(emitter);
}
}
static inline void
emitter_table_dict_begin(emitter_t *emitter, const char *table_key) {
if (emitter->output == emitter_output_table) {
emitter_dict_begin(emitter, NULL, table_key);
}
}
static inline void
emitter_table_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_table) {
emitter_dict_end(emitter);
}
}
static inline void
emitter_json_arr_begin(emitter_t *emitter, const char *json_key) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "\"%s\": [", json_key);
emitter_nest_inc(emitter);
}
}
static inline void
emitter_json_arr_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n");
emitter_indent(emitter);
emitter_printf(emitter, "]");
}
}
static inline void
emitter_json_arr_obj_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_printf(emitter, "{");
emitter_nest_inc(emitter);
}
}
static inline void
emitter_json_arr_obj_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth > 0);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n");
emitter_indent(emitter);
emitter_printf(emitter, "}");
}
}
static inline void
emitter_json_arr_value(emitter_t *emitter, emitter_type_t value_type,
const void *value) {
if (emitter->output == emitter_output_json) {
emitter_json_key_prefix(emitter);
emitter_print_value(emitter, emitter_justify_none, -1,
value_type, value);
va_list ap;
va_start(ap, format);
malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap);
va_end(ap);
}
}
@@ -432,4 +394,93 @@ emitter_table_row(emitter_t *emitter, emitter_row_t *row) {
emitter_table_printf(emitter, "\n");
}
static inline void
emitter_row_init(emitter_row_t *row) {
ql_new(&row->cols);
}
static inline void
emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
ql_elm_new(col, link);
ql_tail_insert(&row->cols, col, link);
}
/******************************************************************************/
/*
* Generalized public API. Emits using either JSON or table, according to
* settings in the emitter_t. */
/*
* Note emits a different kv pair as well, but only in table mode. Omits the
* note if table_note_key is NULL.
*/
static inline void
emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_type_t value_type, const void *value,
const char *table_note_key, emitter_type_t table_note_value_type,
const void *table_note_value) {
if (emitter->output == emitter_output_json) {
emitter_json_key(emitter, json_key);
emitter_json_value(emitter, value_type, value);
} else {
emitter_table_kv_note(emitter, table_key, value_type, value,
table_note_key, table_note_value_type, table_note_value);
}
emitter->item_at_depth = true;
}
static inline void
emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_type_t value_type, const void *value) {
emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL,
emitter_type_bool, NULL);
}
static inline void
emitter_dict_begin(emitter_t *emitter, const char *json_key,
const char *table_header) {
if (emitter->output == emitter_output_json) {
emitter_json_key(emitter, json_key);
emitter_json_object_begin(emitter);
} else {
emitter_table_dict_begin(emitter, table_header);
}
}
static inline void
emitter_dict_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
emitter_json_object_end(emitter);
} else {
emitter_table_dict_end(emitter);
}
}
static inline void
emitter_begin(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth == 0);
emitter_printf(emitter, "{");
emitter_nest_inc(emitter);
} else {
/*
* This guarantees that we always call write_cb at least once.
* This is useful if some invariant is established by each call
* to write_cb, but doesn't hold initially: e.g., some buffer
* holds a null-terminated string.
*/
emitter_printf(emitter, "%s", "");
}
}
static inline void
emitter_end(emitter_t *emitter) {
if (emitter->output == emitter_output_json) {
assert(emitter->nesting_depth == 1);
emitter_nest_dec(emitter);
emitter_printf(emitter, "\n}\n");
}
}
#endif /* JEMALLOC_INTERNAL_EMITTER_H */
deps/jemalloc/include/jemalloc/internal/extent_externs.h
Vendored
+11 -1
View File
@@ -24,13 +24,17 @@ size_t extent_size_quantize_floor(size_t size);
size_t extent_size_quantize_ceil(size_t size);
#endif
rb_proto(, extent_avail_, extent_tree_t, extent_t)
ph_proto(, extent_avail_, extent_tree_t, extent_t)
ph_proto(, extent_heap_, extent_heap_t, extent_t)
bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,
bool delay_coalesce);
extent_state_t extents_state_get(const extents_t *extents);
size_t extents_npages_get(extents_t *extents);
/* Get the number of extents in the given page size index. */
size_t extents_nextents_get(extents_t *extents, pszind_t ind);
/* Get the sum total bytes of the extents in the given page size index. */
size_t extents_nbytes_get(extents_t *extents, pszind_t ind);
extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr,
size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
@@ -70,4 +74,10 @@ bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
bool extent_boot(void);
void extent_util_stats_get(tsdn_t *tsdn, const void *ptr,
size_t *nfree, size_t *nregs, size_t *size);
void extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
size_t *nfree, size_t *nregs, size_t *size,
size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr);
#endif /* JEMALLOC_INTERNAL_EXTENT_EXTERNS_H */
deps/jemalloc/include/jemalloc/internal/extent_inlines.h
Vendored
+83 -15
View File
@@ -6,6 +6,7 @@
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
static inline void
@@ -34,18 +35,19 @@ extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
(uintptr_t)extent2);
}
static inline arena_t *
extent_arena_get(const extent_t *extent) {
static inline unsigned
extent_arena_ind_get(const extent_t *extent) {
unsigned arena_ind = (unsigned)((extent->e_bits &
EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT);
/*
* The following check is omitted because we should never actually read
* a NULL arena pointer.
*/
if (false && arena_ind >= MALLOCX_ARENA_LIMIT) {
return NULL;
}
assert(arena_ind < MALLOCX_ARENA_LIMIT);
return arena_ind;
}
static inline arena_t *
extent_arena_get(const extent_t *extent) {
unsigned arena_ind = extent_arena_ind_get(extent);
return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE);
}
@@ -53,14 +55,14 @@ static inline szind_t
extent_szind_get_maybe_invalid(const extent_t *extent) {
szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >>
EXTENT_BITS_SZIND_SHIFT);
assert(szind <= NSIZES);
assert(szind <= SC_NSIZES);
return szind;
}
static inline szind_t
extent_szind_get(const extent_t *extent) {
szind_t szind = extent_szind_get_maybe_invalid(extent);
assert(szind < NSIZES); /* Never call when "invalid". */
assert(szind < SC_NSIZES); /* Never call when "invalid". */
return szind;
}
@@ -69,6 +71,14 @@ extent_usize_get(const extent_t *extent) {
return sz_index2size(extent_szind_get(extent));
}
static inline unsigned
extent_binshard_get(const extent_t *extent) {
unsigned binshard = (unsigned)((extent->e_bits &
EXTENT_BITS_BINSHARD_MASK) >> EXTENT_BITS_BINSHARD_SHIFT);
assert(binshard < bin_infos[extent_szind_get(extent)].n_shards);
return binshard;
}
static inline size_t
extent_sn_get(const extent_t *extent) {
return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >>
@@ -176,6 +186,11 @@ extent_prof_tctx_get(const extent_t *extent) {
ATOMIC_ACQUIRE);
}
static inline nstime_t
extent_prof_alloc_time_get(const extent_t *extent) {
return extent->e_alloc_time;
}
static inline void
extent_arena_set(extent_t *extent, arena_t *arena) {
unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U <<
@@ -184,13 +199,21 @@ extent_arena_set(extent_t *extent, arena_t *arena) {
((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT);
}
static inline void
extent_binshard_set(extent_t *extent, unsigned binshard) {
/* The assertion assumes szind is set already. */
assert(binshard < bin_infos[extent_szind_get(extent)].n_shards);
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_BINSHARD_MASK) |
((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT);
}
static inline void
extent_addr_set(extent_t *extent, void *addr) {
extent->e_addr = addr;
}
static inline void
extent_addr_randomize(UNUSED tsdn_t *tsdn, extent_t *extent, size_t alignment) {
extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) {
assert(extent_base_get(extent) == extent_addr_get(extent));
if (alignment < PAGE) {
@@ -234,7 +257,7 @@ extent_bsize_set(extent_t *extent, size_t bsize) {
static inline void
extent_szind_set(extent_t *extent, szind_t szind) {
assert(szind <= NSIZES); /* NSIZES means "invalid". */
assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) |
((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT);
}
@@ -246,6 +269,16 @@ extent_nfree_set(extent_t *extent, unsigned nfree) {
((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT);
}
static inline void
extent_nfree_binshard_set(extent_t *extent, unsigned nfree, unsigned binshard) {
/* The assertion assumes szind is set already. */
assert(binshard < bin_infos[extent_szind_get(extent)].n_shards);
extent->e_bits = (extent->e_bits &
(~EXTENT_BITS_NFREE_MASK & ~EXTENT_BITS_BINSHARD_MASK)) |
((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT) |
((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT);
}
static inline void
extent_nfree_inc(extent_t *extent) {
assert(extent_slab_get(extent));
@@ -258,6 +291,12 @@ extent_nfree_dec(extent_t *extent) {
extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
}
static inline void
extent_nfree_sub(extent_t *extent, uint64_t n) {
assert(extent_slab_get(extent));
extent->e_bits -= (n << EXTENT_BITS_NFREE_SHIFT);
}
static inline void
extent_sn_set(extent_t *extent, size_t sn) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) |
@@ -299,10 +338,35 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) {
atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE);
}
static inline void
extent_prof_alloc_time_set(extent_t *extent, nstime_t t) {
nstime_copy(&extent->e_alloc_time, &t);
}
static inline bool
extent_is_head_get(extent_t *extent) {
if (maps_coalesce) {
not_reached();
}
return (bool)((extent->e_bits & EXTENT_BITS_IS_HEAD_MASK) >>
EXTENT_BITS_IS_HEAD_SHIFT);
}
static inline void
extent_is_head_set(extent_t *extent, bool is_head) {
if (maps_coalesce) {
not_reached();
}
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_IS_HEAD_MASK) |
((uint64_t)is_head << EXTENT_BITS_IS_HEAD_SHIFT);
}
static inline void
extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed,
bool committed, bool dumpable) {
bool committed, bool dumpable, extent_head_state_t is_head) {
assert(addr == PAGE_ADDR2BASE(addr) || !slab);
extent_arena_set(extent, arena);
@@ -316,6 +380,10 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
extent_committed_set(extent, committed);
extent_dumpable_set(extent, dumpable);
ql_elm_new(extent, ql_link);
if (!maps_coalesce) {
extent_is_head_set(extent, (is_head == EXTENT_IS_HEAD) ? true :
false);
}
if (config_prof) {
extent_prof_tctx_set(extent, NULL);
}
@@ -327,7 +395,7 @@ extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) {
extent_addr_set(extent, addr);
extent_bsize_set(extent, bsize);
extent_slab_set(extent, false);
extent_szind_set(extent, NSIZES);
extent_szind_set(extent, SC_NSIZES);
extent_sn_set(extent, sn);
extent_state_set(extent, extent_state_active);
extent_zeroed_set(extent, true);
deps/jemalloc/include/jemalloc/internal/extent_structs.h
Vendored
+48 -11
View File
@@ -2,11 +2,12 @@
#define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/ql.h"
#include "jemalloc/internal/ph.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/sc.h"
typedef enum {
extent_state_active = 0,
@@ -28,9 +29,10 @@ struct extent_s {
* t: state
* i: szind
* f: nfree
* s: bin_shard
* n: sn
*
* nnnnnnnn ... nnnnffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa
* nnnnnnnn ... nnnnnnss ssssffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa
*
* arena_ind: Arena from which this extent came, or all 1 bits if
* unassociated.
@@ -75,6 +77,8 @@ struct extent_s {
*
* nfree: Number of free regions in slab.
*
* bin_shard: the shard of the bin from which this extent came.
*
* sn: Serial number (potentially non-unique).
*
* Serial numbers may wrap around if !opt_retain, but as long as
@@ -112,7 +116,7 @@ struct extent_s {
#define EXTENT_BITS_STATE_SHIFT (EXTENT_BITS_ZEROED_WIDTH + EXTENT_BITS_ZEROED_SHIFT)
#define EXTENT_BITS_STATE_MASK MASK(EXTENT_BITS_STATE_WIDTH, EXTENT_BITS_STATE_SHIFT)
#define EXTENT_BITS_SZIND_WIDTH LG_CEIL_NSIZES
#define EXTENT_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES)
#define EXTENT_BITS_SZIND_SHIFT (EXTENT_BITS_STATE_WIDTH + EXTENT_BITS_STATE_SHIFT)
#define EXTENT_BITS_SZIND_MASK MASK(EXTENT_BITS_SZIND_WIDTH, EXTENT_BITS_SZIND_SHIFT)
@@ -120,7 +124,15 @@ struct extent_s {
#define EXTENT_BITS_NFREE_SHIFT (EXTENT_BITS_SZIND_WIDTH + EXTENT_BITS_SZIND_SHIFT)
#define EXTENT_BITS_NFREE_MASK MASK(EXTENT_BITS_NFREE_WIDTH, EXTENT_BITS_NFREE_SHIFT)
#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT)
#define EXTENT_BITS_BINSHARD_WIDTH 6
#define EXTENT_BITS_BINSHARD_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT)
#define EXTENT_BITS_BINSHARD_MASK MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT)
#define EXTENT_BITS_IS_HEAD_WIDTH 1
#define EXTENT_BITS_IS_HEAD_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT)
#define EXTENT_BITS_IS_HEAD_MASK MASK(EXTENT_BITS_IS_HEAD_WIDTH, EXTENT_BITS_IS_HEAD_SHIFT)
#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_IS_HEAD_WIDTH + EXTENT_BITS_IS_HEAD_SHIFT)
#define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT)
/* Pointer to the extent that this structure is responsible for. */
@@ -160,11 +172,13 @@ struct extent_s {
/* Small region slab metadata. */
arena_slab_data_t e_slab_data;
/*
* Profile counters, used for large objects. Points to a
* prof_tctx_t.
*/
atomic_p_t e_prof_tctx;
/* Profiling data, used for large objects. */
struct {
/* Time when this was allocated. */
nstime_t e_alloc_time;
/* Points to a prof_tctx_t. */
atomic_p_t e_prof_tctx;
};
};
};
typedef ql_head(extent_t) extent_list_t;
@@ -180,14 +194,16 @@ struct extents_s {
*
* Synchronization: mtx.
*/
extent_heap_t heaps[NPSIZES+1];
extent_heap_t heaps[SC_NPSIZES + 1];
atomic_zu_t nextents[SC_NPSIZES + 1];
atomic_zu_t nbytes[SC_NPSIZES + 1];
/*
* Bitmap for which set bits correspond to non-empty heaps.
*
* Synchronization: mtx.
*/
bitmap_t bitmap[BITMAP_GROUPS(NPSIZES+1)];
bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)];
/*
* LRU of all extents in heaps.
@@ -216,4 +232,25 @@ struct extents_s {
bool delay_coalesce;
};
/*
* The following two structs are for experimental purposes. See
* experimental_utilization_query_ctl and
* experimental_utilization_batch_query_ctl in src/ctl.c.
*/
struct extent_util_stats_s {
size_t nfree;
size_t nregs;
size_t size;
};
struct extent_util_stats_verbose_s {
void *slabcur_addr;
size_t nfree;
size_t nregs;
size_t size;
size_t bin_nfree;
size_t bin_nregs;
};
#endif /* JEMALLOC_INTERNAL_EXTENT_STRUCTS_H */
deps/jemalloc/include/jemalloc/internal/extent_types.h
Vendored
+8 -2
View File
@@ -4,9 +4,10 @@
typedef struct extent_s extent_t;
typedef struct extents_s extents_t;
#define EXTENT_HOOKS_INITIALIZER NULL
typedef struct extent_util_stats_s extent_util_stats_t;
typedef struct extent_util_stats_verbose_s extent_util_stats_verbose_t;
#define EXTENT_GROW_MAX_PIND (NPSIZES - 1)
#define EXTENT_HOOKS_INITIALIZER NULL
/*
* When reuse (and split) an active extent, (1U << opt_lg_extent_max_active_fit)
@@ -14,4 +15,9 @@ typedef struct extents_s extents_t;
*/
#define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6
typedef enum {
EXTENT_NOT_HEAD,
EXTENT_IS_HEAD /* Only relevant for Windows && opt.retain. */
} extent_head_state_t;
#endif /* JEMALLOC_INTERNAL_EXTENT_TYPES_H */
deps/jemalloc/include/jemalloc/internal/hash.h
Vendored
+33 -32
View File
@@ -104,8 +104,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
uint32_t k1 = 0;
switch (len & 3) {
case 3: k1 ^= tail[2] << 16;
case 2: k1 ^= tail[1] << 8;
case 3: k1 ^= tail[2] << 16; JEMALLOC_FALLTHROUGH
case 2: k1 ^= tail[1] << 8; JEMALLOC_FALLTHROUGH
case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15);
k1 *= c2; h1 ^= k1;
}
@@ -119,7 +119,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
return h1;
}
UNUSED static inline void
static inline void
hash_x86_128(const void *key, const int len, uint32_t seed,
uint64_t r_out[2]) {
const uint8_t * data = (const uint8_t *) key;
@@ -177,28 +177,29 @@ hash_x86_128(const void *key, const int len, uint32_t seed,
uint32_t k4 = 0;
switch (len & 15) {
case 15: k4 ^= tail[14] << 16;
case 14: k4 ^= tail[13] << 8;
case 15: k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH
case 14: k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH
case 13: k4 ^= tail[12] << 0;
k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
case 12: k3 ^= tail[11] << 24;
case 11: k3 ^= tail[10] << 16;
case 10: k3 ^= tail[ 9] << 8;
JEMALLOC_FALLTHROUGH
case 12: k3 ^= tail[11] << 24; JEMALLOC_FALLTHROUGH
case 11: k3 ^= tail[10] << 16; JEMALLOC_FALLTHROUGH
case 10: k3 ^= tail[ 9] << 8; JEMALLOC_FALLTHROUGH
case 9: k3 ^= tail[ 8] << 0;
k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
case 8: k2 ^= tail[ 7] << 24;
case 7: k2 ^= tail[ 6] << 16;
case 6: k2 ^= tail[ 5] << 8;
JEMALLOC_FALLTHROUGH
case 8: k2 ^= tail[ 7] << 24; JEMALLOC_FALLTHROUGH
case 7: k2 ^= tail[ 6] << 16; JEMALLOC_FALLTHROUGH
case 6: k2 ^= tail[ 5] << 8; JEMALLOC_FALLTHROUGH
case 5: k2 ^= tail[ 4] << 0;
k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
case 4: k1 ^= tail[ 3] << 24;
case 3: k1 ^= tail[ 2] << 16;
case 2: k1 ^= tail[ 1] << 8;
JEMALLOC_FALLTHROUGH
case 4: k1 ^= tail[ 3] << 24; JEMALLOC_FALLTHROUGH
case 3: k1 ^= tail[ 2] << 16; JEMALLOC_FALLTHROUGH
case 2: k1 ^= tail[ 1] << 8; JEMALLOC_FALLTHROUGH
case 1: k1 ^= tail[ 0] << 0;
k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
JEMALLOC_FALLTHROUGH
}
}
@@ -220,7 +221,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed,
r_out[1] = (((uint64_t) h4) << 32) | h3;
}
UNUSED static inline void
static inline void
hash_x64_128(const void *key, const int len, const uint32_t seed,
uint64_t r_out[2]) {
const uint8_t *data = (const uint8_t *) key;
@@ -260,22 +261,22 @@ hash_x64_128(const void *key, const int len, const uint32_t seed,
uint64_t k2 = 0;
switch (len & 15) {
case 15: k2 ^= ((uint64_t)(tail[14])) << 48; /* falls through */
case 14: k2 ^= ((uint64_t)(tail[13])) << 40; /* falls through */
case 13: k2 ^= ((uint64_t)(tail[12])) << 32; /* falls through */
case 12: k2 ^= ((uint64_t)(tail[11])) << 24; /* falls through */
case 11: k2 ^= ((uint64_t)(tail[10])) << 16; /* falls through */
case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; /* falls through */
case 15: k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH
case 14: k2 ^= ((uint64_t)(tail[13])) << 40; JEMALLOC_FALLTHROUGH
case 13: k2 ^= ((uint64_t)(tail[12])) << 32; JEMALLOC_FALLTHROUGH
case 12: k2 ^= ((uint64_t)(tail[11])) << 24; JEMALLOC_FALLTHROUGH
case 11: k2 ^= ((uint64_t)(tail[10])) << 16; JEMALLOC_FALLTHROUGH
case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; JEMALLOC_FALLTHROUGH
case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0;
k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
/* falls through */
case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; /* falls through */
case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; /* falls through */
case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; /* falls through */
case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; /* falls through */
case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; /* falls through */
case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; /* falls through */
case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; /* falls through */
JEMALLOC_FALLTHROUGH
case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; JEMALLOC_FALLTHROUGH
case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; JEMALLOC_FALLTHROUGH
case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; JEMALLOC_FALLTHROUGH
case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; JEMALLOC_FALLTHROUGH
case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; JEMALLOC_FALLTHROUGH
case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; JEMALLOC_FALLTHROUGH
case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; JEMALLOC_FALLTHROUGH
case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0;
k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
}
deps/jemalloc/include/jemalloc/internal/hook.h
Vendored
+163
View File
@@ -0,0 +1,163 @@
#ifndef JEMALLOC_INTERNAL_HOOK_H
#define JEMALLOC_INTERNAL_HOOK_H
#include "jemalloc/internal/tsd.h"
/*
* This API is *extremely* experimental, and may get ripped out, changed in API-
* and ABI-incompatible ways, be insufficiently or incorrectly documented, etc.
*
* It allows hooking the stateful parts of the API to see changes as they
* happen.
*
* Allocation hooks are called after the allocation is done, free hooks are
* called before the free is done, and expand hooks are called after the
* allocation is expanded.
*
* For realloc and rallocx, if the expansion happens in place, the expansion
* hook is called. If it is moved, then the alloc hook is called on the new
* location, and then the free hook is called on the old location (i.e. both
* hooks are invoked in between the alloc and the dalloc).
*
* If we return NULL from OOM, then usize might not be trustworthy. Calling
* realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0)
* only calls the free hook. (Calling realloc(NULL, 0) is treated as malloc(0),
* and only calls the alloc hook).
*
* Reentrancy:
* Reentrancy is guarded against from within the hook implementation. If you
* call allocator functions from within a hook, the hooks will not be invoked
* again.
* Threading:
* The installation of a hook synchronizes with all its uses. If you can
* prove the installation of a hook happens-before a jemalloc entry point,
* then the hook will get invoked (unless there's a racing removal).
*
* Hook insertion appears to be atomic at a per-thread level (i.e. if a thread
* allocates and has the alloc hook invoked, then a subsequent free on the
* same thread will also have the free hook invoked).
*
* The *removal* of a hook does *not* block until all threads are done with
* the hook. Hook authors have to be resilient to this, and need some
* out-of-band mechanism for cleaning up any dynamically allocated memory
* associated with their hook.
* Ordering:
* Order of hook execution is unspecified, and may be different than insertion
* order.
*/
#define HOOK_MAX 4
enum hook_alloc_e {
hook_alloc_malloc,
hook_alloc_posix_memalign,
hook_alloc_aligned_alloc,
hook_alloc_calloc,
hook_alloc_memalign,
hook_alloc_valloc,
hook_alloc_mallocx,
/* The reallocating functions have both alloc and dalloc variants */
hook_alloc_realloc,
hook_alloc_rallocx,
};
/*
* We put the enum typedef after the enum, since this file may get included by
* jemalloc_cpp.cpp, and C++ disallows enum forward declarations.
*/
typedef enum hook_alloc_e hook_alloc_t;
enum hook_dalloc_e {
hook_dalloc_free,
hook_dalloc_dallocx,
hook_dalloc_sdallocx,
/*
* The dalloc halves of reallocation (not called if in-place expansion
* happens).
*/
hook_dalloc_realloc,
hook_dalloc_rallocx,
};
typedef enum hook_dalloc_e hook_dalloc_t;
enum hook_expand_e {
hook_expand_realloc,
hook_expand_rallocx,
hook_expand_xallocx,
};
typedef enum hook_expand_e hook_expand_t;
typedef void (*hook_alloc)(
void *extra, hook_alloc_t type, void *result, uintptr_t result_raw,
uintptr_t args_raw[3]);
typedef void (*hook_dalloc)(
void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
typedef void (*hook_expand)(
void *extra, hook_expand_t type, void *address, size_t old_usize,
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
typedef struct hooks_s hooks_t;
struct hooks_s {
hook_alloc alloc_hook;
hook_dalloc dalloc_hook;
hook_expand expand_hook;
void *extra;
};
/*
* Begin implementation details; everything above this point might one day live
* in a public API. Everything below this point never will.
*/
/*
* The realloc pathways haven't gotten any refactoring love in a while, and it's
* fairly difficult to pass information from the entry point to the hooks. We
* put the informaiton the hooks will need into a struct to encapsulate
* everything.
*
* Much of these pathways are force-inlined, so that the compiler can avoid
* materializing this struct until we hit an extern arena function. For fairly
* goofy reasons, *many* of the realloc paths hit an extern arena function.
* These paths are cold enough that it doesn't matter; eventually, we should
* rewrite the realloc code to make the expand-in-place and the
* free-then-realloc paths more orthogonal, at which point we don't need to
* spread the hook logic all over the place.
*/
typedef struct hook_ralloc_args_s hook_ralloc_args_t;
struct hook_ralloc_args_s {
/* I.e. as opposed to rallocx. */
bool is_realloc;
/*
* The expand hook takes 4 arguments, even if only 3 are actually used;
* we add an extra one in case the user decides to memcpy without
* looking too closely at the hooked function.
*/
uintptr_t args[4];
};
/*
* Returns an opaque handle to be used when removing the hook. NULL means that
* we couldn't install the hook.
*/
bool hook_boot();
void *hook_install(tsdn_t *tsdn, hooks_t *hooks);
/* Uninstalls the hook with the handle previously returned from hook_install. */
void hook_remove(tsdn_t *tsdn, void *opaque);
/* Hooks */
void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
uintptr_t args_raw[3]);
void hook_invoke_dalloc(hook_dalloc_t type, void *address,
uintptr_t args_raw[3]);
void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
#endif /* JEMALLOC_INTERNAL_HOOK_H */
deps/jemalloc/include/jemalloc/internal/hooks.h
Vendored
-19
View File
@@ -1,19 +0,0 @@
#ifndef JEMALLOC_INTERNAL_HOOKS_H
#define JEMALLOC_INTERNAL_HOOKS_H
extern JEMALLOC_EXPORT void (*hooks_arena_new_hook)();
extern JEMALLOC_EXPORT void (*hooks_libc_hook)();
#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn)
#define open JEMALLOC_HOOK(open, hooks_libc_hook)
#define read JEMALLOC_HOOK(read, hooks_libc_hook)
#define write JEMALLOC_HOOK(write, hooks_libc_hook)
#define readlink JEMALLOC_HOOK(readlink, hooks_libc_hook)
#define close JEMALLOC_HOOK(close, hooks_libc_hook)
#define creat JEMALLOC_HOOK(creat, hooks_libc_hook)
#define secure_getenv JEMALLOC_HOOK(secure_getenv, hooks_libc_hook)
/* Note that this is undef'd and re-define'd in src/prof.c. */
#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook)
#endif /* JEMALLOC_INTERNAL_HOOKS_H */
deps/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h
Vendored
+3
View File
@@ -31,6 +31,9 @@
# include <sys/uio.h>
# endif
# include <pthread.h>
# ifdef __FreeBSD__
# include <pthread_np.h>
# endif
# include <signal.h>
# ifdef JEMALLOC_OS_UNFAIR_LOCK
# include <os/lock.h>
deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in
Vendored
+22 -22
View File
@@ -48,25 +48,13 @@
/* Defined if GCC __atomic atomics are available. */
#undef JEMALLOC_GCC_ATOMIC_ATOMICS
/* and the 8-bit variant support. */
#undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS
/* Defined if GCC __sync atomics are available. */
#undef JEMALLOC_GCC_SYNC_ATOMICS
/*
* Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and
* __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite
* __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the
* functions are defined in libgcc instead of being inlines).
*/
#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4
/*
* Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and
* __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite
* __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the
* functions are defined in libgcc instead of being inlines).
*/
#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8
/* and the 8-bit variant support. */
#undef JEMALLOC_GCC_U8_SYNC_ATOMICS
/*
* Defined if __builtin_clz() and __builtin_clzl() are available.
@@ -78,12 +66,6 @@
*/
#undef JEMALLOC_OS_UNFAIR_LOCK
/*
* Defined if OSSpin*() functions are available, as provided by Darwin, and
* documented in the spinlock(3) manual page.
*/
#undef JEMALLOC_OSSPIN
/* Defined if syscall(2) is usable. */
#undef JEMALLOC_USE_SYSCALL
@@ -153,6 +135,9 @@
/* JEMALLOC_STATS enables statistics calculation. */
#undef JEMALLOC_STATS
/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */
#undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API
/* JEMALLOC_PROF enables allocation profiling. */
#undef JEMALLOC_PROF
@@ -233,6 +218,12 @@
#undef JEMALLOC_INTERNAL_FFSL
#undef JEMALLOC_INTERNAL_FFS
/*
* popcount*() functions to use for bitmapping.
*/
#undef JEMALLOC_INTERNAL_POPCOUNTL
#undef JEMALLOC_INTERNAL_POPCOUNT
/*
* If defined, explicitly attempt to more uniformly distribute large allocation
* pointer alignments across all cache indices.
@@ -245,6 +236,12 @@
*/
#undef JEMALLOC_LOG
/*
* If defined, use readlinkat() (instead of readlink()) to follow
* /etc/malloc_conf.
*/
#undef JEMALLOC_READLINKAT
/*
* Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
*/
@@ -363,4 +360,7 @@
*/
#undef JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE
/* Performs additional safety checks when defined. */
#undef JEMALLOC_OPT_SAFETY_CHECKS
#endif /* JEMALLOC_INTERNAL_DEFS_H_ */
deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h
Vendored
+5 -1
View File
@@ -2,7 +2,6 @@
#define JEMALLOC_INTERNAL_EXTERNS_H
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/tsd_types.h"
/* TSD checks this to set thread local slow state accordingly. */
@@ -11,6 +10,7 @@ extern bool malloc_slow;
/* Run-time options. */
extern bool opt_abort;
extern bool opt_abort_conf;
extern bool opt_confirm_conf;
extern const char *opt_junk;
extern bool opt_junk_alloc;
extern bool opt_junk_free;
@@ -25,6 +25,9 @@ extern unsigned ncpus;
/* Number of arenas used for automatic multiplexing of threads and arenas. */
extern unsigned narenas_auto;
/* Base index for manual arenas. */
extern unsigned manual_arena_base;
/*
* Arenas that are used to service external requests. Not all elements of the
* arenas array are necessarily used; arenas are created lazily as needed.
@@ -49,5 +52,6 @@ void jemalloc_prefork(void);
void jemalloc_postfork_parent(void);
void jemalloc_postfork_child(void);
bool malloc_initialized(void);
void je_sdallocx_noflags(void *ptr, size_t size);
#endif /* JEMALLOC_INTERNAL_EXTERNS_H */
deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h
Vendored
+8 -6
View File
@@ -4,13 +4,15 @@
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/size_classes.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/ticker.h"
JEMALLOC_ALWAYS_INLINE malloc_cpuid_t
malloc_getcpu(void) {
assert(have_percpu_arena);
#if defined(JEMALLOC_HAVE_SCHED_GETCPU)
#if defined(_WIN32)
return GetCurrentProcessorNumber();
#elif defined(JEMALLOC_HAVE_SCHED_GETCPU)
return (malloc_cpuid_t)sched_getcpu();
#else
not_reached();
@@ -108,14 +110,14 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) {
JEMALLOC_ALWAYS_INLINE cache_bin_t *
tcache_small_bin_get(tcache_t *tcache, szind_t binind) {
assert(binind < NBINS);
assert(binind < SC_NBINS);
return &tcache->bins_small[binind];
}
JEMALLOC_ALWAYS_INLINE cache_bin_t *
tcache_large_bin_get(tcache_t *tcache, szind_t binind) {
assert(binind >= NBINS &&binind < nhbins);
return &tcache->bins_large[binind - NBINS];
assert(binind >= SC_NBINS &&binind < nhbins);
return &tcache->bins_large[binind - SC_NBINS];
}
JEMALLOC_ALWAYS_INLINE bool
@@ -156,7 +158,7 @@ pre_reentrancy(tsd_t *tsd, arena_t *arena) {
if (fast) {
/* Prepare slow path for reentrancy. */
tsd_slow_update(tsd);
assert(tsd->state == tsd_state_nominal_slow);
assert(tsd_state_get(tsd) == tsd_state_nominal_slow);
}
}

Some files were not shown because too many files have changed in this diff Show More